diff --git a/.github/workflows/cpp-model-images.yml b/.github/workflows/cpp-model-images.yml new file mode 100644 index 0000000..66bcfd1 --- /dev/null +++ b/.github/workflows/cpp-model-images.yml @@ -0,0 +1,92 @@ +name: C++ Rockchip model images + +on: + pull_request: + paths: + - 'src/cpp/**' + - '.github/workflows/cpp-model-images.yml' + push: + branches: [main] + paths: + - 'src/cpp/**' + - '.github/workflows/cpp-model-images.yml' + workflow_dispatch: + inputs: + model: + description: Catalog model ID (empty builds the full catalog) + type: string + required: false + +permissions: + contents: read + +concurrency: + group: cpp-model-images-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + images: + runs-on: ubuntu-24.04-arm + timeout-minutes: 120 + permissions: + contents: read + packages: write + defaults: + run: + working-directory: src/cpp + env: + RUNTIME_IMAGE: rockchip-cv-runtime:${{ github.sha }} + PUBLISH: ${{ github.ref == 'refs/heads/main' && github.event_name != 'pull_request' }} + MODEL_ID: ${{ inputs.model }} + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + lfs: false + sparse-checkout: src/cpp + - uses: actions/setup-python@v6 + with: + python-version: '3.12' + - name: Validate model catalog + run: | + python -m pip install 'PyYAML==6.0.3' + python tools/models.py validate + - uses: docker/login-action@v4 + if: env.PUBLISH == 'true' + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + - name: Build or reuse the shared runtime + shell: bash + run: | + namespace="ghcr.io/${GITHUB_REPOSITORY,,}" + remote_runtime="$namespace/runtime:$GITHUB_SHA" + echo "IMAGE_NAMESPACE=$namespace" >> "$GITHUB_ENV" + if [[ "$PUBLISH" == true ]]; then + if output=$(docker pull --platform linux/arm64 "$remote_runtime" 2>&1); then + echo "$output" + docker image tag "$remote_runtime" "$RUNTIME_IMAGE" + exit 0 + fi + echo "$output" + if ! grep -Eqi 'manifest unknown|not found|name unknown' <<< "$output"; then + exit 1 + fi + fi + docker build --builder default --platform linux/arm64 --target runtime \ + --label "org.opencontainers.image.source=https://github.com/$GITHUB_REPOSITORY" \ + --label "org.opencontainers.image.revision=$GITHUB_SHA" \ + --tag "$RUNTIME_IMAGE" . + if [[ "$PUBLISH" == true ]]; then + docker image tag "$RUNTIME_IMAGE" "$remote_runtime" + docker push "$remote_runtime" + fi + - name: Build and verify model packages + shell: bash + run: | + args=() + if [[ -n "$MODEL_ID" ]]; then args+=(--model "$MODEL_ID"); fi + if [[ "$PUBLISH" == true ]]; then args+=(--publish); fi + python tools/build_images.py --runtime "$RUNTIME_IMAGE" --repository "$IMAGE_NAMESPACE" \ + --revision "$GITHUB_SHA" "${args[@]}" diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index bdf8296..b340f1c 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -51,6 +51,9 @@ on: - '!docker/rv182x-rk3588/**' tags: ['v*.*.*'] pull_request: + paths-ignore: + - 'src/cpp/**' + - '.github/workflows/cpp-model-images.yml' workflow_dispatch: inputs: platform: @@ -84,11 +87,6 @@ jobs: - rk3576_ppocr steps: - - name: Checkout Code - uses: actions/checkout@v6 - with: - lfs: true - - name: Check if build is needed id: check_changes run: | @@ -107,6 +105,15 @@ jobs: echo "build=true" >> $GITHUB_OUTPUT fi + - name: Checkout Code + if: steps.check_changes.outputs.build == 'true' + uses: actions/checkout@v6 + with: + lfs: true + sparse-checkout: | + src/${{ matrix.platform }} + docker + - name: Prepare Image Name if: steps.check_changes.outputs.build == 'true' run: | diff --git a/src/cpp/.dockerignore b/src/cpp/.dockerignore new file mode 100644 index 0000000..536dd3f --- /dev/null +++ b/src/cpp/.dockerignore @@ -0,0 +1,18 @@ +.git +.venv +__pycache__ +*.pyc +**/__pycache__ +**/*.pyc +**/*.so +build +media +.local +*.log +._* +**/._* +.DS_Store +**/.DS_Store +.env +.env.* +.model-package/ diff --git a/src/cpp/.gitignore b/src/cpp/.gitignore new file mode 100644 index 0000000..3454741 --- /dev/null +++ b/src/cpp/.gitignore @@ -0,0 +1,13 @@ +__pycache__/ +*.pyc +*.so +build/ +.venv/ +media/ +.env +.env.* +.local/ +.DS_Store +*.log +._* +.model-package/ diff --git a/src/cpp/CMakeLists.txt b/src/cpp/CMakeLists.txt new file mode 100644 index 0000000..aa6f154 --- /dev/null +++ b/src/cpp/CMakeLists.txt @@ -0,0 +1,35 @@ +cmake_minimum_required(VERSION 3.18) +project(rk_cpp_api LANGUAGES CXX) +set(CMAKE_CXX_STANDARD 17) +find_package(pybind11 CONFIG REQUIRED) +find_package(PkgConfig REQUIRED) +find_package(Threads REQUIRED) +find_package(nlohmann_json REQUIRED) +pkg_check_modules(AV REQUIRED IMPORTED_TARGET libavformat libavcodec libavutil libavdevice) +set(ROCKCHIP_SDK /opt/rockchip CACHE PATH "Rockchip SDK prefix") +find_path(RGA_INCLUDE im2d.hpp HINTS ${ROCKCHIP_SDK}/rga/include REQUIRED) +find_path(RKNN_INCLUDE rknn_api.h HINTS ${ROCKCHIP_SDK}/rknn/include REQUIRED) +find_path(MPP_INCLUDE rk_mpi.h HINTS ${ROCKCHIP_SDK}/mpp/include PATH_SUFFIXES rockchip REQUIRED) +find_library(RGA_LIB rga HINTS ${ROCKCHIP_SDK}/rga/lib REQUIRED) +find_library(RKNN_LIB rknnrt HINTS ${ROCKCHIP_SDK}/rknn/lib REQUIRED) +find_library(MPP_LIB rockchip_mpp HINTS ${ROCKCHIP_SDK}/mpp/lib REQUIRED) +pybind11_add_module(_native + native/bindings.cpp + native/engine/geometry.cpp + native/engine/postprocess.cpp + native/engine/frame.cpp + native/engine/frame_queue.cpp + native/engine/dma_buffer.cpp + native/engine/decoder.cpp + native/engine/preprocess.cpp + native/engine/task_model.cpp + native/engine/task_decode.cpp + native/engine/task_dense.cpp + native/engine/task_json.cpp + native/engine/native_output.cpp + native/engine/detector.cpp + native/engine/detector_pool.cpp + native/engine/jpeg_preview.cpp) +target_include_directories(_native PRIVATE native ${RGA_INCLUDE} ${RKNN_INCLUDE} ${MPP_INCLUDE}) +target_link_libraries(_native PRIVATE PkgConfig::AV Threads::Threads nlohmann_json::nlohmann_json ${RGA_LIB} ${RKNN_LIB} ${MPP_LIB}) +install(TARGETS _native LIBRARY DESTINATION rkapi) diff --git a/src/cpp/Dockerfile b/src/cpp/Dockerfile new file mode 100644 index 0000000..a55e50a --- /dev/null +++ b/src/cpp/Dockerfile @@ -0,0 +1,58 @@ +FROM ubuntu:26.04 AS build +ENV DEBIAN_FRONTEND=noninteractive +ARG UBUNTU_MIRROR=http://ports.ubuntu.com/ubuntu-ports +RUN sed -i "s|http://ports.ubuntu.com/ubuntu-ports/|${UBUNTU_MIRROR}/|g" /etc/apt/sources.list.d/ubuntu.sources \ + && apt-get update && apt-get install -y --no-install-recommends \ + build-essential cmake pkg-config git curl ca-certificates \ + python3-dev python3-yaml pybind11-dev nlohmann-json3-dev libdrm-dev \ + libavformat-dev libavcodec-dev libavutil-dev libavdevice-dev \ + && rm -rf /var/lib/apt/lists/* +COPY tools/setup-sdk.sh /tmp/setup-sdk.sh +RUN bash /tmp/setup-sdk.sh +WORKDIR /source +COPY CMakeLists.txt ./ +COPY native ./native +RUN cmake -S . -B /build -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/opt/rkapi \ + && cmake --build /build --parallel 4 && cmake --install /build + +FROM ubuntu:26.04 AS runtime +ARG UBUNTU_MIRROR=http://ports.ubuntu.com/ubuntu-ports +ENV DEBIAN_FRONTEND=noninteractive \ + PYTHONUNBUFFERED=1 \ + OPENCV_IO_MAX_IMAGE_PIXELS=67108864 \ + LD_LIBRARY_PATH=/opt/rockchip/rga/lib:/opt/rockchip/rknn/lib:/opt/rockchip/mpp/lib \ + PATH=/opt/venv/bin:$PATH +RUN sed -i "s|http://ports.ubuntu.com/ubuntu-ports/|${UBUNTU_MIRROR}/|g" /etc/apt/sources.list.d/ubuntu.sources \ + && apt-get update && apt-get install -y --no-install-recommends \ + python3-venv libavdevice62 libavformat62 libavcodec62 libavutil60 libdrm2 ca-certificates \ + && rm -rf /var/lib/apt/lists/* \ + && python3 -m venv /opt/venv +COPY requirements.txt /tmp/requirements.txt +ARG PIP_INDEX_URL=https://pypi.org/simple +RUN pip install --no-cache-dir --only-binary=:all: --index-url "$PIP_INDEX_URL" -r /tmp/requirements.txt +COPY --from=build /opt/rockchip/rga/lib/librga.so /opt/rockchip/rga/lib/ +COPY --from=build /opt/rockchip/rknn/lib/librknnrt.so /opt/rockchip/rknn/lib/ +COPY --from=build /opt/rockchip/mpp/lib/librockchip_mpp.so.0 /opt/rockchip/mpp/lib/ +RUN ln -s librockchip_mpp.so.0 /opt/rockchip/mpp/lib/librockchip_mpp.so.1 \ + && ln -s librockchip_mpp.so.1 /opt/rockchip/mpp/lib/librockchip_mpp.so +COPY --from=build /opt/rkapi /opt/rkapi +COPY rkapi /opt/rkapi/rkapi +WORKDIR /opt/rkapi +ENV RK_PACKAGE=/opt/model/package.json +LABEL org.opencontainers.image.title="Rockchip CV API" \ + org.opencontainers.image.description="FastAPI service backed by the Rockchip C++ RGA/DMA/RKNN engine" +EXPOSE 8000 +HEALTHCHECK --interval=30s --timeout=3s --start-period=60s \ + CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=2)" || exit 1 +CMD ["uvicorn", "rkapi.app:app", "--host", "0.0.0.0", "--port", "8000", "--workers", "1", "--limit-concurrency", "16"] + +# The catalog selects weights and adapter; no Dockerfile edit per new model. +FROM build AS package +COPY rkapi /source/rkapi +COPY tools/models.py /source/tools/models.py +COPY models /source/models +ARG MODEL_ID +RUN python3 tools/models.py prepare --model "$MODEL_ID" --output /model-package + +FROM runtime AS model +COPY --from=package /model-package/ /opt/model/ diff --git a/src/cpp/Dockerfile.model b/src/cpp/Dockerfile.model new file mode 100644 index 0000000..9c5cb2b --- /dev/null +++ b/src/cpp/Dockerfile.model @@ -0,0 +1,3 @@ +ARG RUNTIME_IMAGE=rockchip-cv-runtime:local +FROM ${RUNTIME_IMAGE} +COPY package/ /opt/model/ diff --git a/src/cpp/README.md b/src/cpp/README.md new file mode 100644 index 0000000..c4ae712 --- /dev/null +++ b/src/cpp/README.md @@ -0,0 +1,316 @@ +# Rockchip CV API for RK3576 and RK3588 + +**Run computer vision on RK3576 and RK3588. Build your app with HTTP.** + +Start a Docker container on your Rockchip board and send an image to +`POST /api/predict`. Each image includes the model, RKNN userspace runtime, +and a C++ engine that uses the NPU for inference and RGA for preprocessing. +Your app can call it from any machine with an HTTP client. + +[Quick start](#quick-start) · [Models](#models) · [API](#api) · [Video](#video) · [Build and publish](#build-and-publish) + +## Quick start + +You need an **RK3576 or RK3588 board** running ARM64 Linux with Docker and +working NPU, RGA, and MPP drivers supplied by the board OS. + +**1. Start YOLO26 object detection on the board** + +```sh +CHIP=rk3588 # Use rk3576 on an RK3576 board. +IMAGE="ghcr.io/seeed-projects/recomputer-rk-cv/yolo26n:${CHIP}-int8" + +docker run -d --name cv-api --restart unless-stopped \ + --privileged -p 8000:8000 \ + -v /proc/device-tree/compatible:/proc/device-tree/compatible:ro \ + "$IMAGE" +``` + +Pick a model and tag from the [model catalog](#models), +or [build the same image locally](#build-locally). Use a different container +name and host port, such as `-p 8001:8000`, to serve a second model. + +**2. Check the model and send an image** + +Run this on the board or replace `localhost` with the board's IP address. + +```sh +API=http://localhost:8000 + +curl -fsS "$API/health" +curl -fsS "$API/api/metadata" +curl -fsS "$API/api/predict" -F "file=@image.jpg" +``` + +Call `/api/metadata` when your client connects to discover the model's task, +labels, and result format. See [Read model metadata](#read-model-metadata) for +the response fields and raster decoding instructions. + +Use a local JPEG or PNG for `image.jpg`. Open `http://:8000/docs` +for interactive requests. + +
+Example detection response + +```json +{ + "success": true, + "source": "image", + "predictions": [ + { + "class": "person", + "class_id": 0, + "confidence": 0.92, + "box": {"x1": 100, "y1": 200, "x2": 300, "y2": 500} + } + ], + "image": {"width": 1280, "height": 720} +} +``` + +This excerpt shows bounding boxes in original image pixels. A frame with +no detections returns an empty `predictions` array. Responses also include +the loaded model and timing fields. + +
+ +### Python client + +Install `requests` in your application environment with +`python -m pip install requests`. This example selects the result field from +the model metadata and prints boxes for the quick-start detection model. + +```python +import requests + +api = "http://localhost:8000" # Use the board's IP when calling remotely. + +metadata_response = requests.get(f"{api}/api/metadata", timeout=10) +metadata_response.raise_for_status() +metadata = metadata_response.json() + +with open("image.jpg", "rb") as image: + response = requests.post( + f"{api}/api/predict", + files={"file": image}, + data={"conf": 0.3}, + timeout=30, + ) + +response.raise_for_status() +result = response.json()[metadata["output"]["field"]] +if metadata["model"]["task"] == "detection": + for item in result: + print(item["class"], item["confidence"], item["box"]) +``` + +## Models + +YOLO26 ships in sizes `n`, `s`, and `m` for RK3576 and RK3588, with +INT8 and FP16 builds for all seven tasks. Each image serves one model. +The catalog contains 84 variants across 21 image names. + +All images built from the same code revision share the same ARM64 operating +system, dependencies, C++ engine, HTTP API, and browser code. Only the final +filesystem layer contains model-specific files. The common entrypoint loads +the model manifest bundled at `/opt/model/package.json`. + +Image names below use the prefix `ghcr.io/seeed-projects/recomputer-rk-cv/`. + +| Task | Example RK3588 image | Prediction result | +| --- | --- | --- | +| Object detection | `yolo26n:rk3588-int8` | `predictions` with boxes and scores | +| Instance segmentation | `yolo26n-seg:rk3588-int8` | `predictions[].mask` | +| Pose estimation | `yolo26n-pose:rk3588-int8` | `predictions[].keypoints` as `[x, y, confidence]` | +| Oriented boxes | `yolo26n-obb:rk3588-int8` | `predictions[].polygon` with four corners | +| Classification | `yolo26n-cls:rk3588-int8` | `classification` with ranked classes | +| Semantic segmentation | `yolo26n-sem:rk3588-int8` | `semantic` raster and class labels | +| Depth estimation | `yolo26n-depth:rk3588-int8` | `depth` raster | + +Replace `rk3588` with `rk3576` for the other board, or `int8` with `fp16` +to change precision. Run `python3 tools/models.py list` for all image tags +and build IDs in the [model catalog](models/). + +
+Raster format + +Masks, semantic maps, and depth maps have `width`, `height`, `box`, and +`runs`. The box locates the raster in the source image. Runs alternate value +and count in row-major order. Semantic values are class IDs. Recover depth +samples with `offset + value * scale`. The value `65536` marks invalid depth. + +
+ +## API + +| Method | Endpoint | Purpose | +| --- | --- | --- | +| `GET` | `/health` | Check model and runtime status | +| `GET` | `/api/models` | Identify the model loaded in this container | +| `GET` | `/api/metadata` | Discover the model's labels, input size, and prediction format | +| `POST` | `/api/predict` | Predict an image, a video frame, or the latest live frame | +| `GET`, `POST` | `/api/config` | Read or update default confidence and NMS thresholds | +| `GET` | `/api/video_feed` | Stream the configured source as annotated MJPEG | +| `GET` | `/` | Watch live object detection and adjust thresholds | +| `GET` | `/docs`, `/openapi.json` | Browse or export the API schema | + +Send an image as multipart `file` or a clip as `video`. For a clip, set +`timestamp` in seconds to select one frame. It defaults to `0`. +A POST with no upload predicts the latest frame from `RK_SOURCE`. + +```sh +curl -fsS "$API/api/predict" -F "video=@clip.mp4" -F "timestamp=1.5" +``` + +Optional multipart fields `conf` and `iou` override thresholds for one +request. Their initial defaults are `0.25` and `0.45`. To update service +defaults, POST JSON with `obj_thresh` and `nms_thresh` to `/api/config`. +NMS runs in C++ when enabled for the export. The compatibility route +`/api/models/{model_id}/predict` accepts the loaded ID, family, or a +configured compatibility alias. + +### Read model metadata + +```sh +curl -fsS "$API/api/metadata" +``` + +The JSON response includes `schema_version` (currently `1`), `model` (including +its task), `model_input` (the network width and height), `class_count`, `labels`, +and `output`. The index into `labels` is the zero-based `class_id`. The labels +reflect `RK_CLASSES` when set. If the label list is empty, use the class ID +directly. Depth has zero classes. + +Read `prediction[metadata["output"]["field"]]` from the prediction response to +get the task's main result. When `output.item_field` is present, read that key +from each item in the result array. `output.point_fields` gives the order of +values in each keypoint or polygon vertex. Coordinates use source-image pixels. +Use the prediction's `image.width` and `image.height` for the frame size. + +For raster results, expand each `[value, count]` pair in the flat `runs` array, +then reshape the values into `height` rows of `width` samples. The raster's +`box` locates it in the source image. Mask values are `0` or `1`, semantic +values are class IDs, and depth samples use `offset + value * scale`. Skip the +sentinel given by `output.raster.invalid_value` in the metadata before applying +that formula. + +
+Latency and configuration + +The response body reports stage timings in milliseconds, including upload +storage, decoding, context wait, DMA upload, RGA preprocessing, +`rknn_run_ms`, `output_get_ms`, and postprocessing. `nms_ms`, when present, +is part of `postprocess_ms`. The `Server-Timing` header reports request +parsing, receive wait, response encoding, and total server time before +response headers. Several timings overlap. Measure the full request on +the client to include transport and response delivery. + +| Environment variable | Default | Purpose | +| --- | --- | --- | +| `RK_NMS` | Model setting | Enable or disable server NMS with `on` or `off` | +| `RK_SOURCE` | Unset | RTSP URL, mounted video file, or H.264 V4L2 device | +| `RK_LOOP_VIDEO` | `0` | Set to `1` for the browser-rendered local MP4 detection demo | +| `RK_WORKERS` | `0` | Automatic live worker count, or `1` for a single context | +| `RK_MAX_UPLOAD_MB` | `128` | Maximum upload size in MB | +| `RK_CLASSES` | Packaged labels | Override labels with one class per line | +| `RK_PACKAGE` | `/opt/model/package.json` | Model package manifest path | + +Use `docker logs cv-api` to investigate startup failures. Check that the +image tag matches the board and its kernel drivers are available. +HTTP 503 on a live request means the source is unavailable or has yet to +provide a usable frame. + +
+ +## Video + +For continuous prediction, add `-e RK_SOURCE=rtsp://192.168.1.10/live` to +the Docker run command. For a local video, pass +`-v "$PWD/media:/media:ro" -e RK_SOURCE=/media/input.mp4`. +An H.264 V4L2 camera can use `-e RK_SOURCE=/dev/video0`. + +Continuous H.264/H.265 sources use MPP hardware decoding. Once a frame is +ready, open `http://:8000` to watch the annotated feed and +adjust confidence and NMS IoU. Threshold changes also update the container's +API defaults. Use a detection image such as `yolo26n:rk3576-int8` for the live preview. + +For smooth playback of a mounted H.264 MP4 with an object detection model, +also pass `-e RK_LOOP_VIDEO=1`. The board loops the file and publishes +timestamped detection results. The browser plays the original MP4 and draws +boxes on a canvas, so playback can run at the video's frame rate even when +inference produces fewer results. The MP4 should use landscape or portrait +dimensions encoded directly, without a rotation tag. + +With a YOLO26 split scores-and-boxes model, the browser applies confidence +filtering and per-class NMS locally. The board sends every valid decoded +proposal with its highest-scoring class. It applies no confidence cutoff, +top-k limit, or NMS to this stream. The preview controls affect the current +tab. The prediction API and legacy MJPEG feed use the container's API settings. + +The browser reads `GET /api/demo/candidates` as a binary stream. Each +little-endian frame contains a `uint32` payload size in bytes, `uint32` source +width, `uint32` source height, `uint32` video loop index, `float64` media +timestamp in seconds, and `uint64` frame sequence. The payload contains +six `float32` values per proposal in the order `x1, y1, x2, y2, confidence, +class_id`. Box coordinates are in source-image pixels. Use the label list +from `GET /api/metadata`. + +Call `curl -fsS -X POST "$API/api/predict"` for the latest result or use +`/api/video_feed` directly for the 10 FPS MJPEG preview. Local files follow +media timestamps from container startup and retain the last result at EOF +when looping is disabled. + +## Build and publish + +### Build locally + +Run these commands in `src/cpp` on an ARM64 Docker host. Inference requires +the matching Rockchip board. + +```sh +python3 tools/models.py list + +docker build --platform linux/arm64 --target model \ + --build-arg MODEL_ID=yolo26n-rk3588-int8 \ + -t yolo26n:rk3588-int8 . +``` + +Set `IMAGE=yolo26n:rk3588-int8` in the quick-start command to run the local +image. To build several model images from exactly the same local runtime, use +the same packaging script as CI. Leave out `--model` to build the full catalog. + +```sh +docker build --platform linux/arm64 --target runtime -t rockchip-cv-runtime:local . +python3 -m pip install PyYAML +python3 tools/build_images.py --runtime rockchip-cv-runtime:local \ + --repository rockchip-cv-local --model yolo26n-rk3588-int8 +``` + +### GitHub Actions + +The [C++ Rockchip model images workflow](../../.github/workflows/cpp-model-images.yml) +uses a native ARM64 runner with a sparse checkout of `src/cpp`. Pull requests +build and check the full catalog. Changes to this directory or workflow on +`main` publish all 84 variants to GHCR. A manual run accepts one catalog ID +or the full catalog when left empty. Manual runs on `main` publish images. + +The workflow saves one shared runtime per source commit under +`runtime:`. Each model uses it as the base, verifies the model +package and native module imports, and checks that its layer list matches the +runtime plus exactly one model layer. Each model is published with a +`-` tag and a `--` tag. + +Each manifest points to an immutable Hugging Face revision. The build +downloads the RKNN model and its metadata, verifies their SHA-256 checksums, +and checks the metadata against the task, chip, class list, and tensor +contract. The repository stores the manifests without the weights. + +## Credits + +This implementation builds on [rockchip-cv-api](https://github.com/Hanzo-Huang/rockchip-cv-api). +The C++ inference engine draws from [rockchip-yolo-cpp](https://github.com/Hanzo-Huang/rockchip-yolo-cpp). +The HTTP interface follows [Seeed's Rockchip CV API](https://github.com/Seeed-Projects/reComputer-RK-CV/tree/main/src/rk3588_yolo11). +The model packages use weights and conversion metadata from +[HanzoHuang/YOLO26-RKNN2](https://huggingface.co/HanzoHuang/YOLO26-RKNN2). +Model manifests and metadata record provenance, license information, and the +SHA-256 checksums used during packaging. diff --git a/src/cpp/models/yolo26/classify/m/rk3576/fp16/model.json b/src/cpp/models/yolo26/classify/m/rk3576/fp16/model.json new file mode 100644 index 0000000..27f3d57 --- /dev/null +++ b/src/cpp/models/yolo26/classify/m/rk3576/fp16/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26m-cls-rk3576-fp16", + "name": "yolo26m-cls", + "family": "yolo26", + "task": "classify", + "backend": "yolo", + "chip": "rk3576", + "size": "m", + "precision": "fp16", + "tag": "rk3576-yolo26m-cls-fp16", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/classify/yolo26m-cls/rk3576/fp16/model.rknn", + "sha256": "77e8f6e31a676eaf5e24babb27d91c882637da842c95603a14697fe4e41f1653", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/classify/yolo26m-cls/rk3576/fp16/metadata.yaml", + "metadata_sha256": "522b3c3033e4101920b80682e00996263dd7d7c8909ea533b46e8efd7a4071bc", + "classes": 1000, + "end2end": false, + "coordinates": "pixels", + "preprocess": "center_crop", + "dense_resize": "none", + "nms": true, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/classify/m/rk3576/int8/model.json b/src/cpp/models/yolo26/classify/m/rk3576/int8/model.json new file mode 100644 index 0000000..4f9e2b3 --- /dev/null +++ b/src/cpp/models/yolo26/classify/m/rk3576/int8/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26m-cls-rk3576-int8", + "name": "yolo26m-cls", + "family": "yolo26", + "task": "classify", + "backend": "yolo", + "chip": "rk3576", + "size": "m", + "precision": "int8", + "tag": "rk3576-yolo26m-cls-int8", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/classify/yolo26m-cls/rk3576/int8/model.rknn", + "sha256": "337c2a5b2ba2e9a8e1e206c1aa4a2749c51b2d24ae0bd3578a1a5dea924768a2", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/classify/yolo26m-cls/rk3576/int8/metadata.yaml", + "metadata_sha256": "b3fb53da0f1482abe42d7e5588202929bde447f6bcb28ee5c8f0722d61f47aa6", + "classes": 1000, + "end2end": false, + "coordinates": "pixels", + "preprocess": "center_crop", + "dense_resize": "none", + "nms": true, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/classify/m/rk3588/fp16/model.json b/src/cpp/models/yolo26/classify/m/rk3588/fp16/model.json new file mode 100644 index 0000000..835ecc8 --- /dev/null +++ b/src/cpp/models/yolo26/classify/m/rk3588/fp16/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26m-cls-rk3588-fp16", + "name": "yolo26m-cls", + "family": "yolo26", + "task": "classify", + "backend": "yolo", + "chip": "rk3588", + "size": "m", + "precision": "fp16", + "tag": "rk3588-yolo26m-cls-fp16", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/classify/yolo26m-cls/rk3588/fp16/model.rknn", + "sha256": "1ef605c33a231775d54a13cceaf6ae9b978ebf1fa0e0fe726bd1bd54ab4e630a", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/classify/yolo26m-cls/rk3588/fp16/metadata.yaml", + "metadata_sha256": "c2e33cd0905d82793b488849a708ca70212bd89b2c12b311bb7ff7b9b98755e1", + "classes": 1000, + "end2end": false, + "coordinates": "pixels", + "preprocess": "center_crop", + "dense_resize": "none", + "nms": true, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/classify/m/rk3588/int8/model.json b/src/cpp/models/yolo26/classify/m/rk3588/int8/model.json new file mode 100644 index 0000000..f38e5d1 --- /dev/null +++ b/src/cpp/models/yolo26/classify/m/rk3588/int8/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26m-cls-rk3588-int8", + "name": "yolo26m-cls", + "family": "yolo26", + "task": "classify", + "backend": "yolo", + "chip": "rk3588", + "size": "m", + "precision": "int8", + "tag": "rk3588-yolo26m-cls-int8", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/classify/yolo26m-cls/rk3588/int8/model.rknn", + "sha256": "2a57e6c2bdc040ee1face87a7a7e0cb724de65a029f41fdd2f28edaaed8da0d8", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/classify/yolo26m-cls/rk3588/int8/metadata.yaml", + "metadata_sha256": "43445cc99832d4279f1c9cdfca1b1d0d6cb647c8344921a246c9a4bca7ebae2a", + "classes": 1000, + "end2end": false, + "coordinates": "pixels", + "preprocess": "center_crop", + "dense_resize": "none", + "nms": true, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/classify/n/rk3576/fp16/model.json b/src/cpp/models/yolo26/classify/n/rk3576/fp16/model.json new file mode 100644 index 0000000..c600344 --- /dev/null +++ b/src/cpp/models/yolo26/classify/n/rk3576/fp16/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26n-cls-rk3576-fp16", + "name": "yolo26n-cls", + "family": "yolo26", + "task": "classify", + "backend": "yolo", + "chip": "rk3576", + "size": "n", + "precision": "fp16", + "tag": "rk3576-yolo26n-cls-fp16", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/classify/yolo26n-cls/rk3576/fp16/model.rknn", + "sha256": "e8a6fd7ec034a339ec2089b15ea05d7a082dc0ac08cd8ca6fc8eaacfcd37e939", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/classify/yolo26n-cls/rk3576/fp16/metadata.yaml", + "metadata_sha256": "858202830265bbb50835e28286c95af075fc9f33e96f534c0916c27a93318259", + "classes": 1000, + "end2end": false, + "coordinates": "pixels", + "preprocess": "center_crop", + "dense_resize": "none", + "nms": true, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/classify/n/rk3576/int8/model.json b/src/cpp/models/yolo26/classify/n/rk3576/int8/model.json new file mode 100644 index 0000000..1707a3e --- /dev/null +++ b/src/cpp/models/yolo26/classify/n/rk3576/int8/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26n-cls-rk3576-int8", + "name": "yolo26n-cls", + "family": "yolo26", + "task": "classify", + "backend": "yolo", + "chip": "rk3576", + "size": "n", + "precision": "int8", + "tag": "rk3576-yolo26n-cls-int8", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/classify/yolo26n-cls/rk3576/int8/model.rknn", + "sha256": "2246169c2ae0d0ecb45889609e92d3699ff93c00029e6d335ffbe08fc8291588", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/classify/yolo26n-cls/rk3576/int8/metadata.yaml", + "metadata_sha256": "1b9519bf1da463d25a301efae2b5665e8381a2039fa6f530a460fedb0f944bc4", + "classes": 1000, + "end2end": false, + "coordinates": "pixels", + "preprocess": "center_crop", + "dense_resize": "none", + "nms": true, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/classify/n/rk3588/fp16/model.json b/src/cpp/models/yolo26/classify/n/rk3588/fp16/model.json new file mode 100644 index 0000000..0e0f263 --- /dev/null +++ b/src/cpp/models/yolo26/classify/n/rk3588/fp16/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26n-cls-rk3588-fp16", + "name": "yolo26n-cls", + "family": "yolo26", + "task": "classify", + "backend": "yolo", + "chip": "rk3588", + "size": "n", + "precision": "fp16", + "tag": "rk3588-yolo26n-cls-fp16", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/classify/yolo26n-cls/rk3588/fp16/model.rknn", + "sha256": "e80f9b970676ee7ea40bf718273f1256188a955a6eb0590508e06732fef8559c", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/classify/yolo26n-cls/rk3588/fp16/metadata.yaml", + "metadata_sha256": "d3df4fa395b13eda73332a78658b53a2046b68de241b50c5ad8fdcda85c4676c", + "classes": 1000, + "end2end": false, + "coordinates": "pixels", + "preprocess": "center_crop", + "dense_resize": "none", + "nms": true, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/classify/n/rk3588/int8/model.json b/src/cpp/models/yolo26/classify/n/rk3588/int8/model.json new file mode 100644 index 0000000..34e060c --- /dev/null +++ b/src/cpp/models/yolo26/classify/n/rk3588/int8/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26n-cls-rk3588-int8", + "name": "yolo26n-cls", + "family": "yolo26", + "task": "classify", + "backend": "yolo", + "chip": "rk3588", + "size": "n", + "precision": "int8", + "tag": "rk3588-yolo26n-cls-int8", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/classify/yolo26n-cls/rk3588/int8/model.rknn", + "sha256": "8a673461aa95d731501e7d62663f298c9cbb2c85b5a5a15baaf51e3bc9e21cb0", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/classify/yolo26n-cls/rk3588/int8/metadata.yaml", + "metadata_sha256": "a98335ef98f59a85d97d41a7eada0bf6c95ffad5e6f3236b1c23789dad8ffa60", + "classes": 1000, + "end2end": false, + "coordinates": "pixels", + "preprocess": "center_crop", + "dense_resize": "none", + "nms": true, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/classify/s/rk3576/fp16/model.json b/src/cpp/models/yolo26/classify/s/rk3576/fp16/model.json new file mode 100644 index 0000000..c120b73 --- /dev/null +++ b/src/cpp/models/yolo26/classify/s/rk3576/fp16/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26s-cls-rk3576-fp16", + "name": "yolo26s-cls", + "family": "yolo26", + "task": "classify", + "backend": "yolo", + "chip": "rk3576", + "size": "s", + "precision": "fp16", + "tag": "rk3576-yolo26s-cls-fp16", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/classify/yolo26s-cls/rk3576/fp16/model.rknn", + "sha256": "7d669952be0662b32d1a5b9e6e9c33e2e5013567613abb6a613e20ff44f0b197", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/classify/yolo26s-cls/rk3576/fp16/metadata.yaml", + "metadata_sha256": "57bc7da0bc1b91a5a9ef61dd52d93410a88337d0f016ca02b5b3d97e8a23f27e", + "classes": 1000, + "end2end": false, + "coordinates": "pixels", + "preprocess": "center_crop", + "dense_resize": "none", + "nms": true, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/classify/s/rk3576/int8/model.json b/src/cpp/models/yolo26/classify/s/rk3576/int8/model.json new file mode 100644 index 0000000..6544a70 --- /dev/null +++ b/src/cpp/models/yolo26/classify/s/rk3576/int8/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26s-cls-rk3576-int8", + "name": "yolo26s-cls", + "family": "yolo26", + "task": "classify", + "backend": "yolo", + "chip": "rk3576", + "size": "s", + "precision": "int8", + "tag": "rk3576-yolo26s-cls-int8", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/classify/yolo26s-cls/rk3576/int8/model.rknn", + "sha256": "3594ab133779b1d4f5e753979bf09e572e3857075abf9c21560d33cfd7d514f0", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/classify/yolo26s-cls/rk3576/int8/metadata.yaml", + "metadata_sha256": "85b56747ddca68610ef705866bcac01b3bd6611a8f0cc030c51a0ecdfefab18e", + "classes": 1000, + "end2end": false, + "coordinates": "pixels", + "preprocess": "center_crop", + "dense_resize": "none", + "nms": true, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/classify/s/rk3588/fp16/model.json b/src/cpp/models/yolo26/classify/s/rk3588/fp16/model.json new file mode 100644 index 0000000..d2e966e --- /dev/null +++ b/src/cpp/models/yolo26/classify/s/rk3588/fp16/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26s-cls-rk3588-fp16", + "name": "yolo26s-cls", + "family": "yolo26", + "task": "classify", + "backend": "yolo", + "chip": "rk3588", + "size": "s", + "precision": "fp16", + "tag": "rk3588-yolo26s-cls-fp16", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/classify/yolo26s-cls/rk3588/fp16/model.rknn", + "sha256": "171211b6e95c37aa9229008a061886cb38eeb0b7f26958dca9156c03db0d0c5b", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/classify/yolo26s-cls/rk3588/fp16/metadata.yaml", + "metadata_sha256": "dffae5982c6a095e330eb18055d52f3c7c4dbed0085565c95942b855af6f7a9d", + "classes": 1000, + "end2end": false, + "coordinates": "pixels", + "preprocess": "center_crop", + "dense_resize": "none", + "nms": true, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/classify/s/rk3588/int8/model.json b/src/cpp/models/yolo26/classify/s/rk3588/int8/model.json new file mode 100644 index 0000000..cfd8ee4 --- /dev/null +++ b/src/cpp/models/yolo26/classify/s/rk3588/int8/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26s-cls-rk3588-int8", + "name": "yolo26s-cls", + "family": "yolo26", + "task": "classify", + "backend": "yolo", + "chip": "rk3588", + "size": "s", + "precision": "int8", + "tag": "rk3588-yolo26s-cls-int8", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/classify/yolo26s-cls/rk3588/int8/model.rknn", + "sha256": "4aeba888e18b62f760a64bbc47abbc8aea0d584d03d0e00e059a6a046f9e31bf", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/classify/yolo26s-cls/rk3588/int8/metadata.yaml", + "metadata_sha256": "1e94741d916849e80d055a0039ac39dede6c678b476a6a63de7bbf27d31f13cc", + "classes": 1000, + "end2end": false, + "coordinates": "pixels", + "preprocess": "center_crop", + "dense_resize": "none", + "nms": true, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/depth/m/rk3576/fp16/model.json b/src/cpp/models/yolo26/depth/m/rk3576/fp16/model.json new file mode 100644 index 0000000..ac5c028 --- /dev/null +++ b/src/cpp/models/yolo26/depth/m/rk3576/fp16/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26m-depth-rk3576-fp16", + "name": "yolo26m-depth", + "family": "yolo26", + "task": "depth", + "backend": "yolo", + "chip": "rk3576", + "size": "m", + "precision": "fp16", + "tag": "rk3576-yolo26m-depth-fp16", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/depth/yolo26m-depth/rk3576/fp16/model.rknn", + "sha256": "43e118eae21100f8faa197c5b2f247c2e674292a65f7d22f8321705c2e8f7012", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/depth/yolo26m-depth/rk3576/fp16/metadata.yaml", + "metadata_sha256": "ec36feeedea446887f3dd38ffdfaa4d8f1382188e10eb2bdba16ee68c096eff3", + "classes": 1, + "end2end": false, + "coordinates": "pixels", + "preprocess": "letterbox", + "dense_resize": "align_corners", + "nms": true, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/depth/m/rk3576/int8/model.json b/src/cpp/models/yolo26/depth/m/rk3576/int8/model.json new file mode 100644 index 0000000..f34c49a --- /dev/null +++ b/src/cpp/models/yolo26/depth/m/rk3576/int8/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26m-depth-rk3576-int8", + "name": "yolo26m-depth", + "family": "yolo26", + "task": "depth", + "backend": "yolo", + "chip": "rk3576", + "size": "m", + "precision": "int8", + "tag": "rk3576-yolo26m-depth-int8", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/depth/yolo26m-depth/rk3576/int8/model.rknn", + "sha256": "7c4cccc2c45fa0664a850314ba71b80d4a8f4fde5b176fe4e68472132c6769ce", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/depth/yolo26m-depth/rk3576/int8/metadata.yaml", + "metadata_sha256": "dc2d5f5aa7d6ce0478555484e8b3f325d465a51ef8440e080c4d606d77e8fa87", + "classes": 1, + "end2end": false, + "coordinates": "pixels", + "preprocess": "letterbox", + "dense_resize": "align_corners", + "nms": true, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/depth/m/rk3588/fp16/model.json b/src/cpp/models/yolo26/depth/m/rk3588/fp16/model.json new file mode 100644 index 0000000..fefb3a5 --- /dev/null +++ b/src/cpp/models/yolo26/depth/m/rk3588/fp16/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26m-depth-rk3588-fp16", + "name": "yolo26m-depth", + "family": "yolo26", + "task": "depth", + "backend": "yolo", + "chip": "rk3588", + "size": "m", + "precision": "fp16", + "tag": "rk3588-yolo26m-depth-fp16", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/depth/yolo26m-depth/rk3588/fp16/model.rknn", + "sha256": "2eacb7ddec8e08b47902b96bdfa56865ecb0deb89b06569a83ef21c22f8952a5", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/depth/yolo26m-depth/rk3588/fp16/metadata.yaml", + "metadata_sha256": "5165c24bfd35769778234cd78366887ad8d945a12839c2392ba67e927c9f3fb7", + "classes": 1, + "end2end": false, + "coordinates": "pixels", + "preprocess": "letterbox", + "dense_resize": "align_corners", + "nms": true, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/depth/m/rk3588/int8/model.json b/src/cpp/models/yolo26/depth/m/rk3588/int8/model.json new file mode 100644 index 0000000..1efd582 --- /dev/null +++ b/src/cpp/models/yolo26/depth/m/rk3588/int8/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26m-depth-rk3588-int8", + "name": "yolo26m-depth", + "family": "yolo26", + "task": "depth", + "backend": "yolo", + "chip": "rk3588", + "size": "m", + "precision": "int8", + "tag": "rk3588-yolo26m-depth-int8", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/depth/yolo26m-depth/rk3588/int8/model.rknn", + "sha256": "d355b594f54ac27eeba98fd63926df00c527b82b81f16ac217831cb19bbab54f", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/depth/yolo26m-depth/rk3588/int8/metadata.yaml", + "metadata_sha256": "119a8444aceda7dd1ada774b607531afd8cec44279018196fbf31f2b441b6f04", + "classes": 1, + "end2end": false, + "coordinates": "pixels", + "preprocess": "letterbox", + "dense_resize": "align_corners", + "nms": true, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/depth/n/rk3576/fp16/model.json b/src/cpp/models/yolo26/depth/n/rk3576/fp16/model.json new file mode 100644 index 0000000..a1d1625 --- /dev/null +++ b/src/cpp/models/yolo26/depth/n/rk3576/fp16/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26n-depth-rk3576-fp16", + "name": "yolo26n-depth", + "family": "yolo26", + "task": "depth", + "backend": "yolo", + "chip": "rk3576", + "size": "n", + "precision": "fp16", + "tag": "rk3576-yolo26n-depth-fp16", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/depth/yolo26n-depth/rk3576/fp16/model.rknn", + "sha256": "e251acb8820843aa6bc6c67a57052749f42365a02e9120a356052e7951c3165b", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/depth/yolo26n-depth/rk3576/fp16/metadata.yaml", + "metadata_sha256": "a7750943445e5a459ef68aa594e4d93edfe7e6d3bc31302857ad0e645e43fdfd", + "classes": 1, + "end2end": false, + "coordinates": "pixels", + "preprocess": "letterbox", + "dense_resize": "align_corners", + "nms": true, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/depth/n/rk3576/int8/model.json b/src/cpp/models/yolo26/depth/n/rk3576/int8/model.json new file mode 100644 index 0000000..761e20b --- /dev/null +++ b/src/cpp/models/yolo26/depth/n/rk3576/int8/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26n-depth-rk3576-int8", + "name": "yolo26n-depth", + "family": "yolo26", + "task": "depth", + "backend": "yolo", + "chip": "rk3576", + "size": "n", + "precision": "int8", + "tag": "rk3576-yolo26n-depth-int8", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/depth/yolo26n-depth/rk3576/int8/model.rknn", + "sha256": "59db3499b632f5014a79b8649453a8d1fb94d9407d24daf60a2c4b1425adb91b", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/depth/yolo26n-depth/rk3576/int8/metadata.yaml", + "metadata_sha256": "938a275390637f61ddb346c96a22accd3150d5c0377dced26fb7df91284ab3ff", + "classes": 1, + "end2end": false, + "coordinates": "pixels", + "preprocess": "letterbox", + "dense_resize": "align_corners", + "nms": true, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/depth/n/rk3588/fp16/model.json b/src/cpp/models/yolo26/depth/n/rk3588/fp16/model.json new file mode 100644 index 0000000..db4557f --- /dev/null +++ b/src/cpp/models/yolo26/depth/n/rk3588/fp16/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26n-depth-rk3588-fp16", + "name": "yolo26n-depth", + "family": "yolo26", + "task": "depth", + "backend": "yolo", + "chip": "rk3588", + "size": "n", + "precision": "fp16", + "tag": "rk3588-yolo26n-depth-fp16", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/depth/yolo26n-depth/rk3588/fp16/model.rknn", + "sha256": "6908c54185a65c003f17a87ab93a9d19d719ec4d3aaf99a0493498598a9cecfc", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/depth/yolo26n-depth/rk3588/fp16/metadata.yaml", + "metadata_sha256": "d137a75e8fc9d5cf930fc49ff0b60e1158df6316f87b9093c11802242c7fa113", + "classes": 1, + "end2end": false, + "coordinates": "pixels", + "preprocess": "letterbox", + "dense_resize": "align_corners", + "nms": true, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/depth/n/rk3588/int8/model.json b/src/cpp/models/yolo26/depth/n/rk3588/int8/model.json new file mode 100644 index 0000000..89fff31 --- /dev/null +++ b/src/cpp/models/yolo26/depth/n/rk3588/int8/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26n-depth-rk3588-int8", + "name": "yolo26n-depth", + "family": "yolo26", + "task": "depth", + "backend": "yolo", + "chip": "rk3588", + "size": "n", + "precision": "int8", + "tag": "rk3588-yolo26n-depth-int8", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/depth/yolo26n-depth/rk3588/int8/model.rknn", + "sha256": "3c9bac9aec7440cbbbcb99982b018fc7fdaee55ec0f4c2eeb552ceb82830229d", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/depth/yolo26n-depth/rk3588/int8/metadata.yaml", + "metadata_sha256": "ca28df20e92bd34f4cccfadadc47210110e05a564d15280dced003af9a414c11", + "classes": 1, + "end2end": false, + "coordinates": "pixels", + "preprocess": "letterbox", + "dense_resize": "align_corners", + "nms": true, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/depth/s/rk3576/fp16/model.json b/src/cpp/models/yolo26/depth/s/rk3576/fp16/model.json new file mode 100644 index 0000000..54f1faf --- /dev/null +++ b/src/cpp/models/yolo26/depth/s/rk3576/fp16/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26s-depth-rk3576-fp16", + "name": "yolo26s-depth", + "family": "yolo26", + "task": "depth", + "backend": "yolo", + "chip": "rk3576", + "size": "s", + "precision": "fp16", + "tag": "rk3576-yolo26s-depth-fp16", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/depth/yolo26s-depth/rk3576/fp16/model.rknn", + "sha256": "5fec20d3f468b3e6bde8f38192e53fdf9a70edc4c92d9405a1058d3804be1371", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/depth/yolo26s-depth/rk3576/fp16/metadata.yaml", + "metadata_sha256": "ca29161494815cccd9cd631e37a182ea91f92e969c80317aeb69ddd40c3a41cb", + "classes": 1, + "end2end": false, + "coordinates": "pixels", + "preprocess": "letterbox", + "dense_resize": "align_corners", + "nms": true, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/depth/s/rk3576/int8/model.json b/src/cpp/models/yolo26/depth/s/rk3576/int8/model.json new file mode 100644 index 0000000..b102df1 --- /dev/null +++ b/src/cpp/models/yolo26/depth/s/rk3576/int8/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26s-depth-rk3576-int8", + "name": "yolo26s-depth", + "family": "yolo26", + "task": "depth", + "backend": "yolo", + "chip": "rk3576", + "size": "s", + "precision": "int8", + "tag": "rk3576-yolo26s-depth-int8", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/depth/yolo26s-depth/rk3576/int8/model.rknn", + "sha256": "5301bef795f11afd512523c0b1076fa930bdeab561730a35f229f7f31f0ff72f", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/depth/yolo26s-depth/rk3576/int8/metadata.yaml", + "metadata_sha256": "de36d8e9bd3c38eff34daca999094d38af4366aaf1fbf8932b2f39280eac5e1e", + "classes": 1, + "end2end": false, + "coordinates": "pixels", + "preprocess": "letterbox", + "dense_resize": "align_corners", + "nms": true, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/depth/s/rk3588/fp16/model.json b/src/cpp/models/yolo26/depth/s/rk3588/fp16/model.json new file mode 100644 index 0000000..9575e98 --- /dev/null +++ b/src/cpp/models/yolo26/depth/s/rk3588/fp16/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26s-depth-rk3588-fp16", + "name": "yolo26s-depth", + "family": "yolo26", + "task": "depth", + "backend": "yolo", + "chip": "rk3588", + "size": "s", + "precision": "fp16", + "tag": "rk3588-yolo26s-depth-fp16", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/depth/yolo26s-depth/rk3588/fp16/model.rknn", + "sha256": "8d15dc2730dcbb8511723628cfbbe2df39541dc21a2c22eeef348a23b6ed2969", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/depth/yolo26s-depth/rk3588/fp16/metadata.yaml", + "metadata_sha256": "cde86f4f51ad1d030f826b9e8d262ebe0008ca6856cedab5464fa75cd543c45d", + "classes": 1, + "end2end": false, + "coordinates": "pixels", + "preprocess": "letterbox", + "dense_resize": "align_corners", + "nms": true, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/depth/s/rk3588/int8/model.json b/src/cpp/models/yolo26/depth/s/rk3588/int8/model.json new file mode 100644 index 0000000..7e7ba15 --- /dev/null +++ b/src/cpp/models/yolo26/depth/s/rk3588/int8/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26s-depth-rk3588-int8", + "name": "yolo26s-depth", + "family": "yolo26", + "task": "depth", + "backend": "yolo", + "chip": "rk3588", + "size": "s", + "precision": "int8", + "tag": "rk3588-yolo26s-depth-int8", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/depth/yolo26s-depth/rk3588/int8/model.rknn", + "sha256": "f09220d0f539f55f31c792d9ea61b0c09b0ec1ef0b09ad35bb2d0c3264db0899", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/depth/yolo26s-depth/rk3588/int8/metadata.yaml", + "metadata_sha256": "9bca789f45cea8efd35214759070143b642e2a3c6d93cf4fe15ceff2087a4180", + "classes": 1, + "end2end": false, + "coordinates": "pixels", + "preprocess": "letterbox", + "dense_resize": "align_corners", + "nms": true, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/detect/m/rk3576/fp16/model.json b/src/cpp/models/yolo26/detect/m/rk3576/fp16/model.json new file mode 100644 index 0000000..3332026 --- /dev/null +++ b/src/cpp/models/yolo26/detect/m/rk3576/fp16/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26m-rk3576-fp16", + "name": "yolo26m", + "family": "yolo26", + "task": "detection", + "backend": "yolo", + "chip": "rk3576", + "size": "m", + "precision": "fp16", + "tag": "rk3576-yolo26m-fp16", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/detect/yolo26m/rk3576/fp16/model.rknn", + "sha256": "add1df2b69f0037252e0d4bdbfd6817097034ad64b4f32d68e78a61499d8b13a", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/detect/yolo26m/rk3576/fp16/metadata.yaml", + "metadata_sha256": "0f282035d6b18f737a88b9472a4d1f27c0ad151697e04e8768eb952aa8394b5d", + "classes": 80, + "end2end": true, + "coordinates": "pixels", + "preprocess": "letterbox", + "dense_resize": "none", + "nms": false, + "conversion": "model-zoo-official" +} diff --git a/src/cpp/models/yolo26/detect/m/rk3576/int8/model.json b/src/cpp/models/yolo26/detect/m/rk3576/int8/model.json new file mode 100644 index 0000000..871fce8 --- /dev/null +++ b/src/cpp/models/yolo26/detect/m/rk3576/int8/model.json @@ -0,0 +1,28 @@ +{ + "id": "yolo26m-rk3576-int8", + "name": "yolo26m", + "family": "yolo26", + "task": "detection", + "backend": "yolo", + "chip": "rk3576", + "size": "m", + "precision": "int8", + "tag": "rk3576-yolo26m-int8", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/detect/yolo26m/rk3576/int8/model.rknn", + "sha256": "5ba8c4ec6c19cd8fd596cf5e41b9b9df318d3b61fdaaba0b37a34cb4821d5f4b", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/detect/yolo26m/rk3576/int8/metadata.yaml", + "metadata_sha256": "0f282035d6b18f737a88b9472a4d1f27c0ad151697e04e8768eb952aa8394b5d", + "classes": 80, + "end2end": true, + "coordinates": "pixels", + "preprocess": "letterbox", + "dense_resize": "none", + "nms": false, + "conversion": "model-zoo-official", + "aliases": [ + "rk3576-yolo26m" + ], + "id_aliases": [ + "yolo26m-rk3576" + ] +} diff --git a/src/cpp/models/yolo26/detect/m/rk3588/fp16/model.json b/src/cpp/models/yolo26/detect/m/rk3588/fp16/model.json new file mode 100644 index 0000000..8eb9ef1 --- /dev/null +++ b/src/cpp/models/yolo26/detect/m/rk3588/fp16/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26m-rk3588-fp16", + "name": "yolo26m", + "family": "yolo26", + "task": "detection", + "backend": "yolo", + "chip": "rk3588", + "size": "m", + "precision": "fp16", + "tag": "rk3588-yolo26m-fp16", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/detect/yolo26m/rk3588/fp16/model.rknn", + "sha256": "335effbcef144156ca643366123617a52a8c5e0488d633a0148fd82df2058204", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/detect/yolo26m/rk3588/fp16/metadata.yaml", + "metadata_sha256": "d515ed0121ea22c805dff01cd81fef9405ba61773b1b6dab2fa12472ce7d5097", + "classes": 80, + "end2end": true, + "coordinates": "pixels", + "preprocess": "letterbox", + "dense_resize": "none", + "nms": false, + "conversion": "model-zoo-official" +} diff --git a/src/cpp/models/yolo26/detect/m/rk3588/int8/model.json b/src/cpp/models/yolo26/detect/m/rk3588/int8/model.json new file mode 100644 index 0000000..445c60a --- /dev/null +++ b/src/cpp/models/yolo26/detect/m/rk3588/int8/model.json @@ -0,0 +1,28 @@ +{ + "id": "yolo26m-rk3588-int8", + "name": "yolo26m", + "family": "yolo26", + "task": "detection", + "backend": "yolo", + "chip": "rk3588", + "size": "m", + "precision": "int8", + "tag": "rk3588-yolo26m-int8", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/detect/yolo26m/rk3588/int8/model.rknn", + "sha256": "0b0960a0c81a83901715023cfb52d730d8b38394b0c72681a9f068e885b7aa25", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/detect/yolo26m/rk3588/int8/metadata.yaml", + "metadata_sha256": "d515ed0121ea22c805dff01cd81fef9405ba61773b1b6dab2fa12472ce7d5097", + "classes": 80, + "end2end": true, + "coordinates": "pixels", + "preprocess": "letterbox", + "dense_resize": "none", + "nms": false, + "conversion": "model-zoo-official", + "aliases": [ + "rk3588-yolo26m" + ], + "id_aliases": [ + "yolo26m-rk3588" + ] +} diff --git a/src/cpp/models/yolo26/detect/n/rk3576/fp16/model.json b/src/cpp/models/yolo26/detect/n/rk3576/fp16/model.json new file mode 100644 index 0000000..b87414a --- /dev/null +++ b/src/cpp/models/yolo26/detect/n/rk3576/fp16/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26n-rk3576-fp16", + "name": "yolo26n", + "family": "yolo26", + "task": "detection", + "backend": "yolo", + "chip": "rk3576", + "size": "n", + "precision": "fp16", + "tag": "rk3576-yolo26n-fp16", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/detect/yolo26n/rk3576/fp16/model.rknn", + "sha256": "fae694be4de95a4d904c8f42312c556bfe3f23b9852c783a43dacce73d770e30", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/detect/yolo26n/rk3576/fp16/metadata.yaml", + "metadata_sha256": "0f282035d6b18f737a88b9472a4d1f27c0ad151697e04e8768eb952aa8394b5d", + "classes": 80, + "end2end": true, + "coordinates": "pixels", + "preprocess": "letterbox", + "dense_resize": "none", + "nms": false, + "conversion": "model-zoo-official" +} diff --git a/src/cpp/models/yolo26/detect/n/rk3576/int8/model.json b/src/cpp/models/yolo26/detect/n/rk3576/int8/model.json new file mode 100644 index 0000000..1992ff4 --- /dev/null +++ b/src/cpp/models/yolo26/detect/n/rk3576/int8/model.json @@ -0,0 +1,29 @@ +{ + "id": "yolo26n-rk3576-int8", + "name": "yolo26n", + "family": "yolo26", + "task": "detection", + "backend": "yolo", + "chip": "rk3576", + "size": "n", + "precision": "int8", + "tag": "rk3576-yolo26n-int8", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/detect/yolo26n/rk3576/int8/model.rknn", + "sha256": "c071d829795e3733fe7876ca0576323df3febeff6fa11cd771249d68a7b23461", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/detect/yolo26n/rk3576/int8/metadata.yaml", + "metadata_sha256": "0f282035d6b18f737a88b9472a4d1f27c0ad151697e04e8768eb952aa8394b5d", + "classes": 80, + "end2end": true, + "coordinates": "pixels", + "preprocess": "letterbox", + "dense_resize": "none", + "nms": false, + "conversion": "model-zoo-official", + "aliases": [ + "rk3576-yolo26n", + "rk3576-yolo26" + ], + "id_aliases": [ + "yolo26n-rk3576" + ] +} diff --git a/src/cpp/models/yolo26/detect/n/rk3588/fp16/model.json b/src/cpp/models/yolo26/detect/n/rk3588/fp16/model.json new file mode 100644 index 0000000..2840a08 --- /dev/null +++ b/src/cpp/models/yolo26/detect/n/rk3588/fp16/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26n-rk3588-fp16", + "name": "yolo26n", + "family": "yolo26", + "task": "detection", + "backend": "yolo", + "chip": "rk3588", + "size": "n", + "precision": "fp16", + "tag": "rk3588-yolo26n-fp16", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/detect/yolo26n/rk3588/fp16/model.rknn", + "sha256": "bd7b18a5e68445d00f06768a550e508bc2d27c7c3bfc13a0ecc645d4fb28a1e3", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/detect/yolo26n/rk3588/fp16/metadata.yaml", + "metadata_sha256": "d515ed0121ea22c805dff01cd81fef9405ba61773b1b6dab2fa12472ce7d5097", + "classes": 80, + "end2end": true, + "coordinates": "pixels", + "preprocess": "letterbox", + "dense_resize": "none", + "nms": false, + "conversion": "model-zoo-official" +} diff --git a/src/cpp/models/yolo26/detect/n/rk3588/int8/model.json b/src/cpp/models/yolo26/detect/n/rk3588/int8/model.json new file mode 100644 index 0000000..f452a4f --- /dev/null +++ b/src/cpp/models/yolo26/detect/n/rk3588/int8/model.json @@ -0,0 +1,29 @@ +{ + "id": "yolo26n-rk3588-int8", + "name": "yolo26n", + "family": "yolo26", + "task": "detection", + "backend": "yolo", + "chip": "rk3588", + "size": "n", + "precision": "int8", + "tag": "rk3588-yolo26n-int8", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/detect/yolo26n/rk3588/int8/model.rknn", + "sha256": "8b841a96e12116d133668ae7c96739435aac028a225a37fcf23c707d5de1796e", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/detect/yolo26n/rk3588/int8/metadata.yaml", + "metadata_sha256": "d515ed0121ea22c805dff01cd81fef9405ba61773b1b6dab2fa12472ce7d5097", + "classes": 80, + "end2end": true, + "coordinates": "pixels", + "preprocess": "letterbox", + "dense_resize": "none", + "nms": false, + "conversion": "model-zoo-official", + "aliases": [ + "rk3588-yolo26n", + "rk3588-yolo26" + ], + "id_aliases": [ + "yolo26n-rk3588" + ] +} diff --git a/src/cpp/models/yolo26/detect/s/rk3576/fp16/model.json b/src/cpp/models/yolo26/detect/s/rk3576/fp16/model.json new file mode 100644 index 0000000..de5a046 --- /dev/null +++ b/src/cpp/models/yolo26/detect/s/rk3576/fp16/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26s-rk3576-fp16", + "name": "yolo26s", + "family": "yolo26", + "task": "detection", + "backend": "yolo", + "chip": "rk3576", + "size": "s", + "precision": "fp16", + "tag": "rk3576-yolo26s-fp16", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/detect/yolo26s/rk3576/fp16/model.rknn", + "sha256": "338f464092b576efce2eabe0058b85e61c94e3ffb86f37231cdd1594c4b418e1", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/detect/yolo26s/rk3576/fp16/metadata.yaml", + "metadata_sha256": "0f282035d6b18f737a88b9472a4d1f27c0ad151697e04e8768eb952aa8394b5d", + "classes": 80, + "end2end": true, + "coordinates": "pixels", + "preprocess": "letterbox", + "dense_resize": "none", + "nms": false, + "conversion": "model-zoo-official" +} diff --git a/src/cpp/models/yolo26/detect/s/rk3576/int8/model.json b/src/cpp/models/yolo26/detect/s/rk3576/int8/model.json new file mode 100644 index 0000000..6e55edd --- /dev/null +++ b/src/cpp/models/yolo26/detect/s/rk3576/int8/model.json @@ -0,0 +1,28 @@ +{ + "id": "yolo26s-rk3576-int8", + "name": "yolo26s", + "family": "yolo26", + "task": "detection", + "backend": "yolo", + "chip": "rk3576", + "size": "s", + "precision": "int8", + "tag": "rk3576-yolo26s-int8", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/detect/yolo26s/rk3576/int8/model.rknn", + "sha256": "a84fecf326d4dcfcc1e2d53bcf3053955ff87f0c865a6d9da97d7bdd233a50ee", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/detect/yolo26s/rk3576/int8/metadata.yaml", + "metadata_sha256": "0f282035d6b18f737a88b9472a4d1f27c0ad151697e04e8768eb952aa8394b5d", + "classes": 80, + "end2end": true, + "coordinates": "pixels", + "preprocess": "letterbox", + "dense_resize": "none", + "nms": false, + "conversion": "model-zoo-official", + "aliases": [ + "rk3576-yolo26s" + ], + "id_aliases": [ + "yolo26s-rk3576" + ] +} diff --git a/src/cpp/models/yolo26/detect/s/rk3588/fp16/model.json b/src/cpp/models/yolo26/detect/s/rk3588/fp16/model.json new file mode 100644 index 0000000..e0d2382 --- /dev/null +++ b/src/cpp/models/yolo26/detect/s/rk3588/fp16/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26s-rk3588-fp16", + "name": "yolo26s", + "family": "yolo26", + "task": "detection", + "backend": "yolo", + "chip": "rk3588", + "size": "s", + "precision": "fp16", + "tag": "rk3588-yolo26s-fp16", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/detect/yolo26s/rk3588/fp16/model.rknn", + "sha256": "70da427dae922b33da61de9326aa291e57a9a873f6cbd4ae183a1e2f0271a990", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/detect/yolo26s/rk3588/fp16/metadata.yaml", + "metadata_sha256": "d515ed0121ea22c805dff01cd81fef9405ba61773b1b6dab2fa12472ce7d5097", + "classes": 80, + "end2end": true, + "coordinates": "pixels", + "preprocess": "letterbox", + "dense_resize": "none", + "nms": false, + "conversion": "model-zoo-official" +} diff --git a/src/cpp/models/yolo26/detect/s/rk3588/int8/model.json b/src/cpp/models/yolo26/detect/s/rk3588/int8/model.json new file mode 100644 index 0000000..eea2c97 --- /dev/null +++ b/src/cpp/models/yolo26/detect/s/rk3588/int8/model.json @@ -0,0 +1,28 @@ +{ + "id": "yolo26s-rk3588-int8", + "name": "yolo26s", + "family": "yolo26", + "task": "detection", + "backend": "yolo", + "chip": "rk3588", + "size": "s", + "precision": "int8", + "tag": "rk3588-yolo26s-int8", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/detect/yolo26s/rk3588/int8/model.rknn", + "sha256": "dd4665e40b78cc7eb895bd5ed6380bf4ec1714c2de4a477fc8bb353fadeefdb0", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/detect/yolo26s/rk3588/int8/metadata.yaml", + "metadata_sha256": "d515ed0121ea22c805dff01cd81fef9405ba61773b1b6dab2fa12472ce7d5097", + "classes": 80, + "end2end": true, + "coordinates": "pixels", + "preprocess": "letterbox", + "dense_resize": "none", + "nms": false, + "conversion": "model-zoo-official", + "aliases": [ + "rk3588-yolo26s" + ], + "id_aliases": [ + "yolo26s-rk3588" + ] +} diff --git a/src/cpp/models/yolo26/obb/m/rk3576/fp16/model.json b/src/cpp/models/yolo26/obb/m/rk3576/fp16/model.json new file mode 100644 index 0000000..7bb9963 --- /dev/null +++ b/src/cpp/models/yolo26/obb/m/rk3576/fp16/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26m-obb-rk3576-fp16", + "name": "yolo26m-obb", + "family": "yolo26", + "task": "obb", + "backend": "yolo", + "chip": "rk3576", + "size": "m", + "precision": "fp16", + "tag": "rk3576-yolo26m-obb-fp16", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/obb/yolo26m-obb/rk3576/fp16/model.rknn", + "sha256": "3e6cfadc0879747d3fe94dcd5afc592b4d52dde628d0993a32cb9ff8d0c14566", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/obb/yolo26m-obb/rk3576/fp16/metadata.yaml", + "metadata_sha256": "4976841eb7b6bd0c19a6765d3ffbdf3ea9b2d76e6ec075d23e420061d443437e", + "classes": 15, + "end2end": true, + "coordinates": "normalized", + "preprocess": "letterbox", + "dense_resize": "none", + "nms": false, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/obb/m/rk3576/int8/model.json b/src/cpp/models/yolo26/obb/m/rk3576/int8/model.json new file mode 100644 index 0000000..9348a4b --- /dev/null +++ b/src/cpp/models/yolo26/obb/m/rk3576/int8/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26m-obb-rk3576-int8", + "name": "yolo26m-obb", + "family": "yolo26", + "task": "obb", + "backend": "yolo", + "chip": "rk3576", + "size": "m", + "precision": "int8", + "tag": "rk3576-yolo26m-obb-int8", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/obb/yolo26m-obb/rk3576/int8/model.rknn", + "sha256": "6fff9f095795a489ff3211032e673a1ac2d5b05331f5c7c5dfddd00e762af8e9", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/obb/yolo26m-obb/rk3576/int8/metadata.yaml", + "metadata_sha256": "1e12c754d9940a1829e76a3abc12aa4b77eb74e04f58d5c1b083c97da84638d2", + "classes": 15, + "end2end": true, + "coordinates": "normalized", + "preprocess": "letterbox", + "dense_resize": "none", + "nms": false, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/obb/m/rk3588/fp16/model.json b/src/cpp/models/yolo26/obb/m/rk3588/fp16/model.json new file mode 100644 index 0000000..f26072a --- /dev/null +++ b/src/cpp/models/yolo26/obb/m/rk3588/fp16/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26m-obb-rk3588-fp16", + "name": "yolo26m-obb", + "family": "yolo26", + "task": "obb", + "backend": "yolo", + "chip": "rk3588", + "size": "m", + "precision": "fp16", + "tag": "rk3588-yolo26m-obb-fp16", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/obb/yolo26m-obb/rk3588/fp16/model.rknn", + "sha256": "2c521b1795afc6f0af272cd28855e03883928abe67a6baaf0421fc9e9df8f7f2", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/obb/yolo26m-obb/rk3588/fp16/metadata.yaml", + "metadata_sha256": "d1cdcfbbc13e140998f06c00236ef031e147cb32efccd339e8043646fdc9875e", + "classes": 15, + "end2end": true, + "coordinates": "normalized", + "preprocess": "letterbox", + "dense_resize": "none", + "nms": false, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/obb/m/rk3588/int8/model.json b/src/cpp/models/yolo26/obb/m/rk3588/int8/model.json new file mode 100644 index 0000000..73182d1 --- /dev/null +++ b/src/cpp/models/yolo26/obb/m/rk3588/int8/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26m-obb-rk3588-int8", + "name": "yolo26m-obb", + "family": "yolo26", + "task": "obb", + "backend": "yolo", + "chip": "rk3588", + "size": "m", + "precision": "int8", + "tag": "rk3588-yolo26m-obb-int8", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/obb/yolo26m-obb/rk3588/int8/model.rknn", + "sha256": "9a6dde92d6c139202d59f1eb741d815fc098abaaeec6240cd303beb7920786de", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/obb/yolo26m-obb/rk3588/int8/metadata.yaml", + "metadata_sha256": "bd2b28714f86ac26bb89064f1813658e212cd0d6389277d497ab5d9c59698c84", + "classes": 15, + "end2end": true, + "coordinates": "normalized", + "preprocess": "letterbox", + "dense_resize": "none", + "nms": false, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/obb/n/rk3576/fp16/model.json b/src/cpp/models/yolo26/obb/n/rk3576/fp16/model.json new file mode 100644 index 0000000..3c364d3 --- /dev/null +++ b/src/cpp/models/yolo26/obb/n/rk3576/fp16/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26n-obb-rk3576-fp16", + "name": "yolo26n-obb", + "family": "yolo26", + "task": "obb", + "backend": "yolo", + "chip": "rk3576", + "size": "n", + "precision": "fp16", + "tag": "rk3576-yolo26n-obb-fp16", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/obb/yolo26n-obb/rk3576/fp16/model.rknn", + "sha256": "997a5cd0348092c782a147af9b62b860033509052efac89f71b222a72740e68a", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/obb/yolo26n-obb/rk3576/fp16/metadata.yaml", + "metadata_sha256": "0bf2c6c5f0a749ef312a68c58efdc8c3e9757ff93e0407cfcb23f78fb2f9391a", + "classes": 15, + "end2end": true, + "coordinates": "normalized", + "preprocess": "letterbox", + "dense_resize": "none", + "nms": false, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/obb/n/rk3576/int8/model.json b/src/cpp/models/yolo26/obb/n/rk3576/int8/model.json new file mode 100644 index 0000000..2d394bf --- /dev/null +++ b/src/cpp/models/yolo26/obb/n/rk3576/int8/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26n-obb-rk3576-int8", + "name": "yolo26n-obb", + "family": "yolo26", + "task": "obb", + "backend": "yolo", + "chip": "rk3576", + "size": "n", + "precision": "int8", + "tag": "rk3576-yolo26n-obb-int8", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/obb/yolo26n-obb/rk3576/int8/model.rknn", + "sha256": "8eda2991892ba0a4f62b3a5160bf36fa0e4152c5154df0b90c8e113971f17d10", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/obb/yolo26n-obb/rk3576/int8/metadata.yaml", + "metadata_sha256": "ab7320511c10048219d1b2b0ae99eb973a0e5c4eae113df676f78723e070f06a", + "classes": 15, + "end2end": true, + "coordinates": "normalized", + "preprocess": "letterbox", + "dense_resize": "none", + "nms": false, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/obb/n/rk3588/fp16/model.json b/src/cpp/models/yolo26/obb/n/rk3588/fp16/model.json new file mode 100644 index 0000000..3c75ca6 --- /dev/null +++ b/src/cpp/models/yolo26/obb/n/rk3588/fp16/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26n-obb-rk3588-fp16", + "name": "yolo26n-obb", + "family": "yolo26", + "task": "obb", + "backend": "yolo", + "chip": "rk3588", + "size": "n", + "precision": "fp16", + "tag": "rk3588-yolo26n-obb-fp16", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/obb/yolo26n-obb/rk3588/fp16/model.rknn", + "sha256": "82d8be18c5cc8d2f7a692a1cf7e6bd96fea43ccdc05a2006e18cc11480635be0", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/obb/yolo26n-obb/rk3588/fp16/metadata.yaml", + "metadata_sha256": "8069ac3675641f0844485cab80a9ca8b65a5b94f03285e7bfa3ca838179b73d3", + "classes": 15, + "end2end": true, + "coordinates": "normalized", + "preprocess": "letterbox", + "dense_resize": "none", + "nms": false, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/obb/n/rk3588/int8/model.json b/src/cpp/models/yolo26/obb/n/rk3588/int8/model.json new file mode 100644 index 0000000..1ef44d9 --- /dev/null +++ b/src/cpp/models/yolo26/obb/n/rk3588/int8/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26n-obb-rk3588-int8", + "name": "yolo26n-obb", + "family": "yolo26", + "task": "obb", + "backend": "yolo", + "chip": "rk3588", + "size": "n", + "precision": "int8", + "tag": "rk3588-yolo26n-obb-int8", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/obb/yolo26n-obb/rk3588/int8/model.rknn", + "sha256": "96816a0d597cd793230cf059ac26b49608a762ae09ee78a62e838af58f863149", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/obb/yolo26n-obb/rk3588/int8/metadata.yaml", + "metadata_sha256": "5334214693a1d6ac433d6d3ba5c9586dfed9d41e151687e2aa5ea9dfa0c0d2b8", + "classes": 15, + "end2end": true, + "coordinates": "normalized", + "preprocess": "letterbox", + "dense_resize": "none", + "nms": false, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/obb/s/rk3576/fp16/model.json b/src/cpp/models/yolo26/obb/s/rk3576/fp16/model.json new file mode 100644 index 0000000..37e536f --- /dev/null +++ b/src/cpp/models/yolo26/obb/s/rk3576/fp16/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26s-obb-rk3576-fp16", + "name": "yolo26s-obb", + "family": "yolo26", + "task": "obb", + "backend": "yolo", + "chip": "rk3576", + "size": "s", + "precision": "fp16", + "tag": "rk3576-yolo26s-obb-fp16", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/obb/yolo26s-obb/rk3576/fp16/model.rknn", + "sha256": "9e5d1f72d15c833d88ad24e393bd729e511cdc0dbed66e40822af99d07c62e08", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/obb/yolo26s-obb/rk3576/fp16/metadata.yaml", + "metadata_sha256": "db7848d27d57b9684113cb543992a5eff9566a18b18860ec0cf8d4602bb14ee6", + "classes": 15, + "end2end": true, + "coordinates": "normalized", + "preprocess": "letterbox", + "dense_resize": "none", + "nms": false, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/obb/s/rk3576/int8/model.json b/src/cpp/models/yolo26/obb/s/rk3576/int8/model.json new file mode 100644 index 0000000..e48c6a1 --- /dev/null +++ b/src/cpp/models/yolo26/obb/s/rk3576/int8/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26s-obb-rk3576-int8", + "name": "yolo26s-obb", + "family": "yolo26", + "task": "obb", + "backend": "yolo", + "chip": "rk3576", + "size": "s", + "precision": "int8", + "tag": "rk3576-yolo26s-obb-int8", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/obb/yolo26s-obb/rk3576/int8/model.rknn", + "sha256": "a4a7539324dc076a7162794489b6de29054fc7bb62db009fb19f148f5312bfa0", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/obb/yolo26s-obb/rk3576/int8/metadata.yaml", + "metadata_sha256": "6bc6265bc75f3eacf0427337892f86330fef0a51131b4ffd000522d94490e38d", + "classes": 15, + "end2end": true, + "coordinates": "normalized", + "preprocess": "letterbox", + "dense_resize": "none", + "nms": false, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/obb/s/rk3588/fp16/model.json b/src/cpp/models/yolo26/obb/s/rk3588/fp16/model.json new file mode 100644 index 0000000..0b05b6d --- /dev/null +++ b/src/cpp/models/yolo26/obb/s/rk3588/fp16/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26s-obb-rk3588-fp16", + "name": "yolo26s-obb", + "family": "yolo26", + "task": "obb", + "backend": "yolo", + "chip": "rk3588", + "size": "s", + "precision": "fp16", + "tag": "rk3588-yolo26s-obb-fp16", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/obb/yolo26s-obb/rk3588/fp16/model.rknn", + "sha256": "f6034c3e7f03da112fba09ba191c4ea45d7826fcd46adcb8162d0f0ef681648a", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/obb/yolo26s-obb/rk3588/fp16/metadata.yaml", + "metadata_sha256": "6b2846583f26629f80250794d478508ceca73f9574a709f0c24e9ef0c4dfac90", + "classes": 15, + "end2end": true, + "coordinates": "normalized", + "preprocess": "letterbox", + "dense_resize": "none", + "nms": false, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/obb/s/rk3588/int8/model.json b/src/cpp/models/yolo26/obb/s/rk3588/int8/model.json new file mode 100644 index 0000000..123a199 --- /dev/null +++ b/src/cpp/models/yolo26/obb/s/rk3588/int8/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26s-obb-rk3588-int8", + "name": "yolo26s-obb", + "family": "yolo26", + "task": "obb", + "backend": "yolo", + "chip": "rk3588", + "size": "s", + "precision": "int8", + "tag": "rk3588-yolo26s-obb-int8", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/obb/yolo26s-obb/rk3588/int8/model.rknn", + "sha256": "bbc14416a292bbe5e84003cc34d851169f4b6206b722c0ad9b8abc1ccefc431a", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/obb/yolo26s-obb/rk3588/int8/metadata.yaml", + "metadata_sha256": "8e005e6acd540d92909b8d0602f3463a01784838e2cf5b7718ed47063adc5752", + "classes": 15, + "end2end": true, + "coordinates": "normalized", + "preprocess": "letterbox", + "dense_resize": "none", + "nms": false, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/pose/m/rk3576/fp16/model.json b/src/cpp/models/yolo26/pose/m/rk3576/fp16/model.json new file mode 100644 index 0000000..33f5f74 --- /dev/null +++ b/src/cpp/models/yolo26/pose/m/rk3576/fp16/model.json @@ -0,0 +1,26 @@ +{ + "id": "yolo26m-pose-rk3576-fp16", + "name": "yolo26m-pose", + "family": "yolo26", + "task": "pose", + "backend": "yolo", + "chip": "rk3576", + "size": "m", + "precision": "fp16", + "tag": "rk3576-yolo26m-pose-fp16", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/pose/yolo26m-pose/rk3576/fp16/model.rknn", + "sha256": "b8ee613fc56e416f99d3e300b830d80d9356be27237eebf44abbb229d6201add", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/pose/yolo26m-pose/rk3576/fp16/metadata.yaml", + "metadata_sha256": "3bb3a1f0b95ea5dd2854a96e93e75b83775b872a05f09f50623e5360e07f935a", + "classes": 1, + "end2end": true, + "coordinates": "normalized", + "preprocess": "letterbox", + "dense_resize": "none", + "nms": false, + "conversion": "custom", + "kpt_shape": [ + 17, + 3 + ] +} diff --git a/src/cpp/models/yolo26/pose/m/rk3576/int8/model.json b/src/cpp/models/yolo26/pose/m/rk3576/int8/model.json new file mode 100644 index 0000000..1f122fa --- /dev/null +++ b/src/cpp/models/yolo26/pose/m/rk3576/int8/model.json @@ -0,0 +1,26 @@ +{ + "id": "yolo26m-pose-rk3576-int8", + "name": "yolo26m-pose", + "family": "yolo26", + "task": "pose", + "backend": "yolo", + "chip": "rk3576", + "size": "m", + "precision": "int8", + "tag": "rk3576-yolo26m-pose-int8", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/pose/yolo26m-pose/rk3576/int8/model.rknn", + "sha256": "688a919ff31bf1fbafb7c96908d1627a3fa0511af02dd919b7751c598cdd53fe", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/pose/yolo26m-pose/rk3576/int8/metadata.yaml", + "metadata_sha256": "944b7fd62541bf44f5494b5395730053f01f6d69c1c5ceab9818c8f54a51bfa7", + "classes": 1, + "end2end": true, + "coordinates": "normalized", + "preprocess": "letterbox", + "dense_resize": "none", + "nms": false, + "conversion": "custom", + "kpt_shape": [ + 17, + 3 + ] +} diff --git a/src/cpp/models/yolo26/pose/m/rk3588/fp16/model.json b/src/cpp/models/yolo26/pose/m/rk3588/fp16/model.json new file mode 100644 index 0000000..f8a7109 --- /dev/null +++ b/src/cpp/models/yolo26/pose/m/rk3588/fp16/model.json @@ -0,0 +1,26 @@ +{ + "id": "yolo26m-pose-rk3588-fp16", + "name": "yolo26m-pose", + "family": "yolo26", + "task": "pose", + "backend": "yolo", + "chip": "rk3588", + "size": "m", + "precision": "fp16", + "tag": "rk3588-yolo26m-pose-fp16", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/pose/yolo26m-pose/rk3588/fp16/model.rknn", + "sha256": "6e8dea1ad1622133d9d37e9d5d950d92058738424d2e0161440f205342a148b2", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/pose/yolo26m-pose/rk3588/fp16/metadata.yaml", + "metadata_sha256": "d59309c7f7f9ca0bcd775fea70bcb146f4a99dd89aeb657cf72579e8805272c4", + "classes": 1, + "end2end": true, + "coordinates": "normalized", + "preprocess": "letterbox", + "dense_resize": "none", + "nms": false, + "conversion": "custom", + "kpt_shape": [ + 17, + 3 + ] +} diff --git a/src/cpp/models/yolo26/pose/m/rk3588/int8/model.json b/src/cpp/models/yolo26/pose/m/rk3588/int8/model.json new file mode 100644 index 0000000..5cf9b6e --- /dev/null +++ b/src/cpp/models/yolo26/pose/m/rk3588/int8/model.json @@ -0,0 +1,26 @@ +{ + "id": "yolo26m-pose-rk3588-int8", + "name": "yolo26m-pose", + "family": "yolo26", + "task": "pose", + "backend": "yolo", + "chip": "rk3588", + "size": "m", + "precision": "int8", + "tag": "rk3588-yolo26m-pose-int8", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/pose/yolo26m-pose/rk3588/int8/model.rknn", + "sha256": "84bfe9742ebb56da29dc166ad5765d144a4e25f9a3d2b14154f41eda55e63d08", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/pose/yolo26m-pose/rk3588/int8/metadata.yaml", + "metadata_sha256": "d8d157a0ab0970a194899031d3a42bdabc8d246b52447dc810e5b14581198c92", + "classes": 1, + "end2end": true, + "coordinates": "normalized", + "preprocess": "letterbox", + "dense_resize": "none", + "nms": false, + "conversion": "custom", + "kpt_shape": [ + 17, + 3 + ] +} diff --git a/src/cpp/models/yolo26/pose/n/rk3576/fp16/model.json b/src/cpp/models/yolo26/pose/n/rk3576/fp16/model.json new file mode 100644 index 0000000..297caec --- /dev/null +++ b/src/cpp/models/yolo26/pose/n/rk3576/fp16/model.json @@ -0,0 +1,26 @@ +{ + "id": "yolo26n-pose-rk3576-fp16", + "name": "yolo26n-pose", + "family": "yolo26", + "task": "pose", + "backend": "yolo", + "chip": "rk3576", + "size": "n", + "precision": "fp16", + "tag": "rk3576-yolo26n-pose-fp16", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/pose/yolo26n-pose/rk3576/fp16/model.rknn", + "sha256": "2ccea925108d78263930fc8d89e8f6ced81fb130c04b94fb42632291e96f1576", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/pose/yolo26n-pose/rk3576/fp16/metadata.yaml", + "metadata_sha256": "259cc844bfe6cbf9b3568f3f1f1ec50cc59f72f961fa755eb7008d39e1965344", + "classes": 1, + "end2end": true, + "coordinates": "normalized", + "preprocess": "letterbox", + "dense_resize": "none", + "nms": false, + "conversion": "custom", + "kpt_shape": [ + 17, + 3 + ] +} diff --git a/src/cpp/models/yolo26/pose/n/rk3576/int8/model.json b/src/cpp/models/yolo26/pose/n/rk3576/int8/model.json new file mode 100644 index 0000000..9dfdcd0 --- /dev/null +++ b/src/cpp/models/yolo26/pose/n/rk3576/int8/model.json @@ -0,0 +1,26 @@ +{ + "id": "yolo26n-pose-rk3576-int8", + "name": "yolo26n-pose", + "family": "yolo26", + "task": "pose", + "backend": "yolo", + "chip": "rk3576", + "size": "n", + "precision": "int8", + "tag": "rk3576-yolo26n-pose-int8", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/pose/yolo26n-pose/rk3576/int8/model.rknn", + "sha256": "d4eb4b8560d99508f437fef8c587ec456ed26c947106b6b2b9343d0bcac08c81", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/pose/yolo26n-pose/rk3576/int8/metadata.yaml", + "metadata_sha256": "cb282344b0ee98a717a6a6becaf61f2e11603258cd227288e982292f09250499", + "classes": 1, + "end2end": true, + "coordinates": "normalized", + "preprocess": "letterbox", + "dense_resize": "none", + "nms": false, + "conversion": "custom", + "kpt_shape": [ + 17, + 3 + ] +} diff --git a/src/cpp/models/yolo26/pose/n/rk3588/fp16/model.json b/src/cpp/models/yolo26/pose/n/rk3588/fp16/model.json new file mode 100644 index 0000000..25c7189 --- /dev/null +++ b/src/cpp/models/yolo26/pose/n/rk3588/fp16/model.json @@ -0,0 +1,26 @@ +{ + "id": "yolo26n-pose-rk3588-fp16", + "name": "yolo26n-pose", + "family": "yolo26", + "task": "pose", + "backend": "yolo", + "chip": "rk3588", + "size": "n", + "precision": "fp16", + "tag": "rk3588-yolo26n-pose-fp16", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/pose/yolo26n-pose/rk3588/fp16/model.rknn", + "sha256": "c967414443dda8a4c5ea1760e0a94b9079f7cbcead728581688552e1acaa108e", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/pose/yolo26n-pose/rk3588/fp16/metadata.yaml", + "metadata_sha256": "8bfc44780809471169f4eca2b3f058b55f9bb94b9c01b513d084a11b6878b2dd", + "classes": 1, + "end2end": true, + "coordinates": "normalized", + "preprocess": "letterbox", + "dense_resize": "none", + "nms": false, + "conversion": "custom", + "kpt_shape": [ + 17, + 3 + ] +} diff --git a/src/cpp/models/yolo26/pose/n/rk3588/int8/model.json b/src/cpp/models/yolo26/pose/n/rk3588/int8/model.json new file mode 100644 index 0000000..e40d28e --- /dev/null +++ b/src/cpp/models/yolo26/pose/n/rk3588/int8/model.json @@ -0,0 +1,26 @@ +{ + "id": "yolo26n-pose-rk3588-int8", + "name": "yolo26n-pose", + "family": "yolo26", + "task": "pose", + "backend": "yolo", + "chip": "rk3588", + "size": "n", + "precision": "int8", + "tag": "rk3588-yolo26n-pose-int8", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/pose/yolo26n-pose/rk3588/int8/model.rknn", + "sha256": "56339efa91cafb6031bf44590b2d182c3566bde57fc83851e9b031c4c9464fef", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/pose/yolo26n-pose/rk3588/int8/metadata.yaml", + "metadata_sha256": "cef8e28ad8037bfe2ba4a0123db3e46624f3c9caa9207ab579c5b7cdfab841d3", + "classes": 1, + "end2end": true, + "coordinates": "normalized", + "preprocess": "letterbox", + "dense_resize": "none", + "nms": false, + "conversion": "custom", + "kpt_shape": [ + 17, + 3 + ] +} diff --git a/src/cpp/models/yolo26/pose/s/rk3576/fp16/model.json b/src/cpp/models/yolo26/pose/s/rk3576/fp16/model.json new file mode 100644 index 0000000..4d41261 --- /dev/null +++ b/src/cpp/models/yolo26/pose/s/rk3576/fp16/model.json @@ -0,0 +1,26 @@ +{ + "id": "yolo26s-pose-rk3576-fp16", + "name": "yolo26s-pose", + "family": "yolo26", + "task": "pose", + "backend": "yolo", + "chip": "rk3576", + "size": "s", + "precision": "fp16", + "tag": "rk3576-yolo26s-pose-fp16", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/pose/yolo26s-pose/rk3576/fp16/model.rknn", + "sha256": "87cc2a19510936f2b3abdaf3824450c2ec0a9d98148ae6360f692cb05a50e452", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/pose/yolo26s-pose/rk3576/fp16/metadata.yaml", + "metadata_sha256": "007517b495dcd3233f6b8accce4af978a6af2a00f30ed3e7cecb09ea3b90dffe", + "classes": 1, + "end2end": true, + "coordinates": "normalized", + "preprocess": "letterbox", + "dense_resize": "none", + "nms": false, + "conversion": "custom", + "kpt_shape": [ + 17, + 3 + ] +} diff --git a/src/cpp/models/yolo26/pose/s/rk3576/int8/model.json b/src/cpp/models/yolo26/pose/s/rk3576/int8/model.json new file mode 100644 index 0000000..4ef3265 --- /dev/null +++ b/src/cpp/models/yolo26/pose/s/rk3576/int8/model.json @@ -0,0 +1,26 @@ +{ + "id": "yolo26s-pose-rk3576-int8", + "name": "yolo26s-pose", + "family": "yolo26", + "task": "pose", + "backend": "yolo", + "chip": "rk3576", + "size": "s", + "precision": "int8", + "tag": "rk3576-yolo26s-pose-int8", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/pose/yolo26s-pose/rk3576/int8/model.rknn", + "sha256": "2b8852f0d3f3514211cf7c4949cc0599a8239c19034623bfc8f0ca20ad8eed58", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/pose/yolo26s-pose/rk3576/int8/metadata.yaml", + "metadata_sha256": "dd90be4f01f80543ee5930b5b59f15c135d9b90eb688c9791fd8ef79c90afd5f", + "classes": 1, + "end2end": true, + "coordinates": "normalized", + "preprocess": "letterbox", + "dense_resize": "none", + "nms": false, + "conversion": "custom", + "kpt_shape": [ + 17, + 3 + ] +} diff --git a/src/cpp/models/yolo26/pose/s/rk3588/fp16/model.json b/src/cpp/models/yolo26/pose/s/rk3588/fp16/model.json new file mode 100644 index 0000000..471f262 --- /dev/null +++ b/src/cpp/models/yolo26/pose/s/rk3588/fp16/model.json @@ -0,0 +1,26 @@ +{ + "id": "yolo26s-pose-rk3588-fp16", + "name": "yolo26s-pose", + "family": "yolo26", + "task": "pose", + "backend": "yolo", + "chip": "rk3588", + "size": "s", + "precision": "fp16", + "tag": "rk3588-yolo26s-pose-fp16", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/pose/yolo26s-pose/rk3588/fp16/model.rknn", + "sha256": "9e36b53030161d2974ede1d0b5dc2359dbaa0f63c52900341eef11728b7a810b", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/pose/yolo26s-pose/rk3588/fp16/metadata.yaml", + "metadata_sha256": "04696d7c7a809b97cd66342932d863e5759c958ca834a3dafa7377d6c92b9ea6", + "classes": 1, + "end2end": true, + "coordinates": "normalized", + "preprocess": "letterbox", + "dense_resize": "none", + "nms": false, + "conversion": "custom", + "kpt_shape": [ + 17, + 3 + ] +} diff --git a/src/cpp/models/yolo26/pose/s/rk3588/int8/model.json b/src/cpp/models/yolo26/pose/s/rk3588/int8/model.json new file mode 100644 index 0000000..8f8f8c6 --- /dev/null +++ b/src/cpp/models/yolo26/pose/s/rk3588/int8/model.json @@ -0,0 +1,26 @@ +{ + "id": "yolo26s-pose-rk3588-int8", + "name": "yolo26s-pose", + "family": "yolo26", + "task": "pose", + "backend": "yolo", + "chip": "rk3588", + "size": "s", + "precision": "int8", + "tag": "rk3588-yolo26s-pose-int8", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/pose/yolo26s-pose/rk3588/int8/model.rknn", + "sha256": "5c5693e2f194b5f6999f062184b42710866fd95a6399be0b35d8f96155c835c6", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/pose/yolo26s-pose/rk3588/int8/metadata.yaml", + "metadata_sha256": "9e8ca9eb33661c1f869f8b70c4a482c929643c4206e6bfdf5108f6c9011c8c26", + "classes": 1, + "end2end": true, + "coordinates": "normalized", + "preprocess": "letterbox", + "dense_resize": "none", + "nms": false, + "conversion": "custom", + "kpt_shape": [ + 17, + 3 + ] +} diff --git a/src/cpp/models/yolo26/segment/m/rk3576/fp16/model.json b/src/cpp/models/yolo26/segment/m/rk3576/fp16/model.json new file mode 100644 index 0000000..d6ad439 --- /dev/null +++ b/src/cpp/models/yolo26/segment/m/rk3576/fp16/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26m-seg-rk3576-fp16", + "name": "yolo26m-seg", + "family": "yolo26", + "task": "segment", + "backend": "yolo", + "chip": "rk3576", + "size": "m", + "precision": "fp16", + "tag": "rk3576-yolo26m-seg-fp16", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/segment/yolo26m-seg/rk3576/fp16/model.rknn", + "sha256": "783c9e51566d49c9fb41403ce8ce7b8bc35ba2b9aa1e133126a335c361776546", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/segment/yolo26m-seg/rk3576/fp16/metadata.yaml", + "metadata_sha256": "05e9adbf236d3ed52ffdd975858ef55d1f83a56f0e2e2104b3aa1f593e6a563a", + "classes": 80, + "end2end": true, + "coordinates": "normalized", + "preprocess": "letterbox", + "dense_resize": "none", + "nms": false, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/segment/m/rk3576/int8/model.json b/src/cpp/models/yolo26/segment/m/rk3576/int8/model.json new file mode 100644 index 0000000..b50a59c --- /dev/null +++ b/src/cpp/models/yolo26/segment/m/rk3576/int8/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26m-seg-rk3576-int8", + "name": "yolo26m-seg", + "family": "yolo26", + "task": "segment", + "backend": "yolo", + "chip": "rk3576", + "size": "m", + "precision": "int8", + "tag": "rk3576-yolo26m-seg-int8", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/segment/yolo26m-seg/rk3576/int8/model.rknn", + "sha256": "6e7e4c7e013fc14a63b6bdab200f217f8a1955ba75b0992a58dd635ba48320cc", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/segment/yolo26m-seg/rk3576/int8/metadata.yaml", + "metadata_sha256": "0cdd4fdef890db06f7fdf4aac80d368f86d7a01e7b1612272a34f623c401631c", + "classes": 80, + "end2end": true, + "coordinates": "normalized", + "preprocess": "letterbox", + "dense_resize": "none", + "nms": false, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/segment/m/rk3588/fp16/model.json b/src/cpp/models/yolo26/segment/m/rk3588/fp16/model.json new file mode 100644 index 0000000..9429f0d --- /dev/null +++ b/src/cpp/models/yolo26/segment/m/rk3588/fp16/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26m-seg-rk3588-fp16", + "name": "yolo26m-seg", + "family": "yolo26", + "task": "segment", + "backend": "yolo", + "chip": "rk3588", + "size": "m", + "precision": "fp16", + "tag": "rk3588-yolo26m-seg-fp16", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/segment/yolo26m-seg/rk3588/fp16/model.rknn", + "sha256": "3c80be077756eb1ca5e1b82f19d01e23905bacc483c24edb2050367c9c41977e", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/segment/yolo26m-seg/rk3588/fp16/metadata.yaml", + "metadata_sha256": "ec4d3fdd9b1cebba371d91d38856ae70b69fd1832c02259d3a3638d8d2a77f4c", + "classes": 80, + "end2end": true, + "coordinates": "normalized", + "preprocess": "letterbox", + "dense_resize": "none", + "nms": false, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/segment/m/rk3588/int8/model.json b/src/cpp/models/yolo26/segment/m/rk3588/int8/model.json new file mode 100644 index 0000000..7c4c8ae --- /dev/null +++ b/src/cpp/models/yolo26/segment/m/rk3588/int8/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26m-seg-rk3588-int8", + "name": "yolo26m-seg", + "family": "yolo26", + "task": "segment", + "backend": "yolo", + "chip": "rk3588", + "size": "m", + "precision": "int8", + "tag": "rk3588-yolo26m-seg-int8", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/segment/yolo26m-seg/rk3588/int8/model.rknn", + "sha256": "425ea785e40f603319b5d1bd9be30ba930d76bccfbf9e747f23f5faa55792ede", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/segment/yolo26m-seg/rk3588/int8/metadata.yaml", + "metadata_sha256": "031d633f923aed83654de8f3f6cb17c462e74cb02abe6e5022d1b8a37fd5baf6", + "classes": 80, + "end2end": true, + "coordinates": "normalized", + "preprocess": "letterbox", + "dense_resize": "none", + "nms": false, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/segment/n/rk3576/fp16/model.json b/src/cpp/models/yolo26/segment/n/rk3576/fp16/model.json new file mode 100644 index 0000000..8cdff5b --- /dev/null +++ b/src/cpp/models/yolo26/segment/n/rk3576/fp16/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26n-seg-rk3576-fp16", + "name": "yolo26n-seg", + "family": "yolo26", + "task": "segment", + "backend": "yolo", + "chip": "rk3576", + "size": "n", + "precision": "fp16", + "tag": "rk3576-yolo26n-seg-fp16", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/segment/yolo26n-seg/rk3576/fp16/model.rknn", + "sha256": "0eb75559435a013e34144681bcf100a4b805c41de783a95c2a0a613875339342", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/segment/yolo26n-seg/rk3576/fp16/metadata.yaml", + "metadata_sha256": "29d38b3dc4066e8d0f6f4347c76cbac390b29e79c89b28973f374d0808f5a59b", + "classes": 80, + "end2end": true, + "coordinates": "normalized", + "preprocess": "letterbox", + "dense_resize": "none", + "nms": false, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/segment/n/rk3576/int8/model.json b/src/cpp/models/yolo26/segment/n/rk3576/int8/model.json new file mode 100644 index 0000000..30a8096 --- /dev/null +++ b/src/cpp/models/yolo26/segment/n/rk3576/int8/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26n-seg-rk3576-int8", + "name": "yolo26n-seg", + "family": "yolo26", + "task": "segment", + "backend": "yolo", + "chip": "rk3576", + "size": "n", + "precision": "int8", + "tag": "rk3576-yolo26n-seg-int8", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/segment/yolo26n-seg/rk3576/int8/model.rknn", + "sha256": "ee6a50ea2682203da03efce85b40d2ba7f2aa7c87ff918ac3c75614813f249f3", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/segment/yolo26n-seg/rk3576/int8/metadata.yaml", + "metadata_sha256": "d9c4728ffcd2121fe36dec65e624693b8107d08a796a1f54b8244b9a66ab5854", + "classes": 80, + "end2end": true, + "coordinates": "normalized", + "preprocess": "letterbox", + "dense_resize": "none", + "nms": false, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/segment/n/rk3588/fp16/model.json b/src/cpp/models/yolo26/segment/n/rk3588/fp16/model.json new file mode 100644 index 0000000..afc4b39 --- /dev/null +++ b/src/cpp/models/yolo26/segment/n/rk3588/fp16/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26n-seg-rk3588-fp16", + "name": "yolo26n-seg", + "family": "yolo26", + "task": "segment", + "backend": "yolo", + "chip": "rk3588", + "size": "n", + "precision": "fp16", + "tag": "rk3588-yolo26n-seg-fp16", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/segment/yolo26n-seg/rk3588/fp16/model.rknn", + "sha256": "bcf012e6ea06be4635787bf7ddc01b5b00b0cdc9b80e99fe67abe87b3af14559", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/segment/yolo26n-seg/rk3588/fp16/metadata.yaml", + "metadata_sha256": "a28549d2798048fec9ffb65040e093d82f4cf03de8c779ed832db1c998cbd2e4", + "classes": 80, + "end2end": true, + "coordinates": "normalized", + "preprocess": "letterbox", + "dense_resize": "none", + "nms": false, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/segment/n/rk3588/int8/model.json b/src/cpp/models/yolo26/segment/n/rk3588/int8/model.json new file mode 100644 index 0000000..af2c2fb --- /dev/null +++ b/src/cpp/models/yolo26/segment/n/rk3588/int8/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26n-seg-rk3588-int8", + "name": "yolo26n-seg", + "family": "yolo26", + "task": "segment", + "backend": "yolo", + "chip": "rk3588", + "size": "n", + "precision": "int8", + "tag": "rk3588-yolo26n-seg-int8", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/segment/yolo26n-seg/rk3588/int8/model.rknn", + "sha256": "85e8fe409076be5c72d6617215ec10e127fb1be56a91fe0ad93ad2f664bc0c28", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/segment/yolo26n-seg/rk3588/int8/metadata.yaml", + "metadata_sha256": "d2e29c1951cf29dafe3f3b0109a533668281d127f8da8d900bff56e5e1428eb7", + "classes": 80, + "end2end": true, + "coordinates": "normalized", + "preprocess": "letterbox", + "dense_resize": "none", + "nms": false, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/segment/s/rk3576/fp16/model.json b/src/cpp/models/yolo26/segment/s/rk3576/fp16/model.json new file mode 100644 index 0000000..e343be5 --- /dev/null +++ b/src/cpp/models/yolo26/segment/s/rk3576/fp16/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26s-seg-rk3576-fp16", + "name": "yolo26s-seg", + "family": "yolo26", + "task": "segment", + "backend": "yolo", + "chip": "rk3576", + "size": "s", + "precision": "fp16", + "tag": "rk3576-yolo26s-seg-fp16", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/segment/yolo26s-seg/rk3576/fp16/model.rknn", + "sha256": "7dd651e2f68a5481cdea9d365f70eb22d680f53eb45a2bcd9fb87dd1623c87ad", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/segment/yolo26s-seg/rk3576/fp16/metadata.yaml", + "metadata_sha256": "a34eb9c7611e151cb76a63666b52ceda3fe4643cf44f7a45c7e97f1bf9d2916a", + "classes": 80, + "end2end": true, + "coordinates": "normalized", + "preprocess": "letterbox", + "dense_resize": "none", + "nms": false, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/segment/s/rk3576/int8/model.json b/src/cpp/models/yolo26/segment/s/rk3576/int8/model.json new file mode 100644 index 0000000..9d11a2d --- /dev/null +++ b/src/cpp/models/yolo26/segment/s/rk3576/int8/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26s-seg-rk3576-int8", + "name": "yolo26s-seg", + "family": "yolo26", + "task": "segment", + "backend": "yolo", + "chip": "rk3576", + "size": "s", + "precision": "int8", + "tag": "rk3576-yolo26s-seg-int8", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/segment/yolo26s-seg/rk3576/int8/model.rknn", + "sha256": "03c5b12d94baed88d35514259c8256efbf17ade7f0c6e983b4911fe73e6ae1b7", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/segment/yolo26s-seg/rk3576/int8/metadata.yaml", + "metadata_sha256": "affb082a967dca89882d8c97fc8a0276bed5c7b506840a571c072758e75ca76f", + "classes": 80, + "end2end": true, + "coordinates": "normalized", + "preprocess": "letterbox", + "dense_resize": "none", + "nms": false, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/segment/s/rk3588/fp16/model.json b/src/cpp/models/yolo26/segment/s/rk3588/fp16/model.json new file mode 100644 index 0000000..d8abdff --- /dev/null +++ b/src/cpp/models/yolo26/segment/s/rk3588/fp16/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26s-seg-rk3588-fp16", + "name": "yolo26s-seg", + "family": "yolo26", + "task": "segment", + "backend": "yolo", + "chip": "rk3588", + "size": "s", + "precision": "fp16", + "tag": "rk3588-yolo26s-seg-fp16", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/segment/yolo26s-seg/rk3588/fp16/model.rknn", + "sha256": "087d57488839e7dc40db8f87fb1b0448cfa73bcb2bb92865f9a8a9f9d6c4662d", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/segment/yolo26s-seg/rk3588/fp16/metadata.yaml", + "metadata_sha256": "cbe9492500651f4ae1b55100867fee43cfdfa7525f08c940bdad8a6f4a6c3042", + "classes": 80, + "end2end": true, + "coordinates": "normalized", + "preprocess": "letterbox", + "dense_resize": "none", + "nms": false, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/segment/s/rk3588/int8/model.json b/src/cpp/models/yolo26/segment/s/rk3588/int8/model.json new file mode 100644 index 0000000..79c567e --- /dev/null +++ b/src/cpp/models/yolo26/segment/s/rk3588/int8/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26s-seg-rk3588-int8", + "name": "yolo26s-seg", + "family": "yolo26", + "task": "segment", + "backend": "yolo", + "chip": "rk3588", + "size": "s", + "precision": "int8", + "tag": "rk3588-yolo26s-seg-int8", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/segment/yolo26s-seg/rk3588/int8/model.rknn", + "sha256": "499abe28eb3c466bf4df34bcb95f8e5035feeb2c07ed7fe30bf5867342f3b662", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/segment/yolo26s-seg/rk3588/int8/metadata.yaml", + "metadata_sha256": "d17cf5f3d5557c5bf2edc56c586d1add1f1d5df7ac34c8b8360fe38790fad3eb", + "classes": 80, + "end2end": true, + "coordinates": "normalized", + "preprocess": "letterbox", + "dense_resize": "none", + "nms": false, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/semantic/m/rk3576/fp16/model.json b/src/cpp/models/yolo26/semantic/m/rk3576/fp16/model.json new file mode 100644 index 0000000..c52a9aa --- /dev/null +++ b/src/cpp/models/yolo26/semantic/m/rk3576/fp16/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26m-sem-rk3576-fp16", + "name": "yolo26m-sem", + "family": "yolo26", + "task": "semantic", + "backend": "yolo", + "chip": "rk3576", + "size": "m", + "precision": "fp16", + "tag": "rk3576-yolo26m-sem-fp16", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/semantic/yolo26m-sem/rk3576/fp16/model.rknn", + "sha256": "051e0be9a744b5ef8be720b88158a81b6415511f3529bcec438249693c83e01f", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/semantic/yolo26m-sem/rk3576/fp16/metadata.yaml", + "metadata_sha256": "1ce5d28d881d61febff958af9674662773bd6cae8f1246f082f773364d8ec347", + "classes": 19, + "end2end": false, + "coordinates": "pixels", + "preprocess": "letterbox", + "dense_resize": "half_pixel", + "nms": true, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/semantic/m/rk3576/int8/model.json b/src/cpp/models/yolo26/semantic/m/rk3576/int8/model.json new file mode 100644 index 0000000..1c98ad6 --- /dev/null +++ b/src/cpp/models/yolo26/semantic/m/rk3576/int8/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26m-sem-rk3576-int8", + "name": "yolo26m-sem", + "family": "yolo26", + "task": "semantic", + "backend": "yolo", + "chip": "rk3576", + "size": "m", + "precision": "int8", + "tag": "rk3576-yolo26m-sem-int8", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/semantic/yolo26m-sem/rk3576/int8/model.rknn", + "sha256": "fb6cd2bad417a4f56ef0a8c0fa5aea0eb4087b9fbf105fdac3be991141ec79bf", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/semantic/yolo26m-sem/rk3576/int8/metadata.yaml", + "metadata_sha256": "e8d3c20797b41f2a9d23ec4df16c08f6ec1cc083fc69eb59331c05c94f0e78e4", + "classes": 19, + "end2end": false, + "coordinates": "pixels", + "preprocess": "letterbox", + "dense_resize": "half_pixel", + "nms": true, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/semantic/m/rk3588/fp16/model.json b/src/cpp/models/yolo26/semantic/m/rk3588/fp16/model.json new file mode 100644 index 0000000..6bbdb5a --- /dev/null +++ b/src/cpp/models/yolo26/semantic/m/rk3588/fp16/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26m-sem-rk3588-fp16", + "name": "yolo26m-sem", + "family": "yolo26", + "task": "semantic", + "backend": "yolo", + "chip": "rk3588", + "size": "m", + "precision": "fp16", + "tag": "rk3588-yolo26m-sem-fp16", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/semantic/yolo26m-sem/rk3588/fp16/model.rknn", + "sha256": "5ed2e60c89d2e2a62be25c4c32578c06e71248fca92f903971405c1123b7b776", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/semantic/yolo26m-sem/rk3588/fp16/metadata.yaml", + "metadata_sha256": "c503e82e85f52eb96bbd2f28d0a1858cc97db5cd37fc1a878344d1fb672415f2", + "classes": 19, + "end2end": false, + "coordinates": "pixels", + "preprocess": "letterbox", + "dense_resize": "half_pixel", + "nms": true, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/semantic/m/rk3588/int8/model.json b/src/cpp/models/yolo26/semantic/m/rk3588/int8/model.json new file mode 100644 index 0000000..38c34ba --- /dev/null +++ b/src/cpp/models/yolo26/semantic/m/rk3588/int8/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26m-sem-rk3588-int8", + "name": "yolo26m-sem", + "family": "yolo26", + "task": "semantic", + "backend": "yolo", + "chip": "rk3588", + "size": "m", + "precision": "int8", + "tag": "rk3588-yolo26m-sem-int8", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/semantic/yolo26m-sem/rk3588/int8/model.rknn", + "sha256": "463708324984e91ba9db67afde8363a990f25deb1d90edd9282a73cf5a4a099c", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/semantic/yolo26m-sem/rk3588/int8/metadata.yaml", + "metadata_sha256": "e589b2e0a5fd5080d81666d1e47f4a4c8351a28b8a70e3ad7134848d9b5bfa5b", + "classes": 19, + "end2end": false, + "coordinates": "pixels", + "preprocess": "letterbox", + "dense_resize": "half_pixel", + "nms": true, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/semantic/n/rk3576/fp16/model.json b/src/cpp/models/yolo26/semantic/n/rk3576/fp16/model.json new file mode 100644 index 0000000..b2d8eb1 --- /dev/null +++ b/src/cpp/models/yolo26/semantic/n/rk3576/fp16/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26n-sem-rk3576-fp16", + "name": "yolo26n-sem", + "family": "yolo26", + "task": "semantic", + "backend": "yolo", + "chip": "rk3576", + "size": "n", + "precision": "fp16", + "tag": "rk3576-yolo26n-sem-fp16", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/semantic/yolo26n-sem/rk3576/fp16/model.rknn", + "sha256": "ca0092c1db59d1fdd781aae9f05a00e82877b0a574d6b19a981c847ee5165450", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/semantic/yolo26n-sem/rk3576/fp16/metadata.yaml", + "metadata_sha256": "356cf38ea07dd89ebc8cc40e2421db1d03f8a635af9a053505fb8d72c1ef705b", + "classes": 19, + "end2end": false, + "coordinates": "pixels", + "preprocess": "letterbox", + "dense_resize": "half_pixel", + "nms": true, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/semantic/n/rk3576/int8/model.json b/src/cpp/models/yolo26/semantic/n/rk3576/int8/model.json new file mode 100644 index 0000000..62d73ce --- /dev/null +++ b/src/cpp/models/yolo26/semantic/n/rk3576/int8/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26n-sem-rk3576-int8", + "name": "yolo26n-sem", + "family": "yolo26", + "task": "semantic", + "backend": "yolo", + "chip": "rk3576", + "size": "n", + "precision": "int8", + "tag": "rk3576-yolo26n-sem-int8", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/semantic/yolo26n-sem/rk3576/int8/model.rknn", + "sha256": "67bb511cdc549efb85f20b19eea17feb3456941719feb785539abb94324eeb1e", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/semantic/yolo26n-sem/rk3576/int8/metadata.yaml", + "metadata_sha256": "0c648e807c58e5651e2f5eec2c7e4bb7c42101a5f56c9f8e0e437b8f384402a4", + "classes": 19, + "end2end": false, + "coordinates": "pixels", + "preprocess": "letterbox", + "dense_resize": "half_pixel", + "nms": true, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/semantic/n/rk3588/fp16/model.json b/src/cpp/models/yolo26/semantic/n/rk3588/fp16/model.json new file mode 100644 index 0000000..6091095 --- /dev/null +++ b/src/cpp/models/yolo26/semantic/n/rk3588/fp16/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26n-sem-rk3588-fp16", + "name": "yolo26n-sem", + "family": "yolo26", + "task": "semantic", + "backend": "yolo", + "chip": "rk3588", + "size": "n", + "precision": "fp16", + "tag": "rk3588-yolo26n-sem-fp16", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/semantic/yolo26n-sem/rk3588/fp16/model.rknn", + "sha256": "cf84416cfd974a2006695ca9bd87514e27a2c68005e6e33452de84baf24080b5", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/semantic/yolo26n-sem/rk3588/fp16/metadata.yaml", + "metadata_sha256": "57b648b82067421bd2eae71632bd43b1a64186a12eb6884f32158fd39a5cafcd", + "classes": 19, + "end2end": false, + "coordinates": "pixels", + "preprocess": "letterbox", + "dense_resize": "half_pixel", + "nms": true, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/semantic/n/rk3588/int8/model.json b/src/cpp/models/yolo26/semantic/n/rk3588/int8/model.json new file mode 100644 index 0000000..4274f02 --- /dev/null +++ b/src/cpp/models/yolo26/semantic/n/rk3588/int8/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26n-sem-rk3588-int8", + "name": "yolo26n-sem", + "family": "yolo26", + "task": "semantic", + "backend": "yolo", + "chip": "rk3588", + "size": "n", + "precision": "int8", + "tag": "rk3588-yolo26n-sem-int8", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/semantic/yolo26n-sem/rk3588/int8/model.rknn", + "sha256": "b7933d81f546b5dab89e3e73c5db301f471f81a5e7781d5543163378a7699474", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/semantic/yolo26n-sem/rk3588/int8/metadata.yaml", + "metadata_sha256": "83f6341cc61ec1909d369cc30b9e631088b297256a5fa0479a148a5eed1bba6e", + "classes": 19, + "end2end": false, + "coordinates": "pixels", + "preprocess": "letterbox", + "dense_resize": "half_pixel", + "nms": true, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/semantic/s/rk3576/fp16/model.json b/src/cpp/models/yolo26/semantic/s/rk3576/fp16/model.json new file mode 100644 index 0000000..191cf73 --- /dev/null +++ b/src/cpp/models/yolo26/semantic/s/rk3576/fp16/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26s-sem-rk3576-fp16", + "name": "yolo26s-sem", + "family": "yolo26", + "task": "semantic", + "backend": "yolo", + "chip": "rk3576", + "size": "s", + "precision": "fp16", + "tag": "rk3576-yolo26s-sem-fp16", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/semantic/yolo26s-sem/rk3576/fp16/model.rknn", + "sha256": "f7a06b3f1996d1d1f5b41dadcd58254dd4880f98a07bc6824153dbe1d61c2bd5", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/semantic/yolo26s-sem/rk3576/fp16/metadata.yaml", + "metadata_sha256": "70740f065fbe144494d707381c28119acb2259eca64ab3faabc13481e5299759", + "classes": 19, + "end2end": false, + "coordinates": "pixels", + "preprocess": "letterbox", + "dense_resize": "half_pixel", + "nms": true, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/semantic/s/rk3576/int8/model.json b/src/cpp/models/yolo26/semantic/s/rk3576/int8/model.json new file mode 100644 index 0000000..7b949c0 --- /dev/null +++ b/src/cpp/models/yolo26/semantic/s/rk3576/int8/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26s-sem-rk3576-int8", + "name": "yolo26s-sem", + "family": "yolo26", + "task": "semantic", + "backend": "yolo", + "chip": "rk3576", + "size": "s", + "precision": "int8", + "tag": "rk3576-yolo26s-sem-int8", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/semantic/yolo26s-sem/rk3576/int8/model.rknn", + "sha256": "fe5f1375b80f0e216e61aa5d399b522aa41e020645c64d898c66a22ee3f5752f", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/semantic/yolo26s-sem/rk3576/int8/metadata.yaml", + "metadata_sha256": "337cc00b409ba7cb934fe080a87ff1d7186fdd97254551aa7bf13c86eca663d4", + "classes": 19, + "end2end": false, + "coordinates": "pixels", + "preprocess": "letterbox", + "dense_resize": "half_pixel", + "nms": true, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/semantic/s/rk3588/fp16/model.json b/src/cpp/models/yolo26/semantic/s/rk3588/fp16/model.json new file mode 100644 index 0000000..82aac45 --- /dev/null +++ b/src/cpp/models/yolo26/semantic/s/rk3588/fp16/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26s-sem-rk3588-fp16", + "name": "yolo26s-sem", + "family": "yolo26", + "task": "semantic", + "backend": "yolo", + "chip": "rk3588", + "size": "s", + "precision": "fp16", + "tag": "rk3588-yolo26s-sem-fp16", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/semantic/yolo26s-sem/rk3588/fp16/model.rknn", + "sha256": "267940e081249cc363afc8e4d20fa512f57fc15cbd80cd37744f6b127aa85b7b", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/semantic/yolo26s-sem/rk3588/fp16/metadata.yaml", + "metadata_sha256": "01d3852a3d3cad1a86f43e95665737c1bb29bc5d40b2d7c5088966e886b78dd7", + "classes": 19, + "end2end": false, + "coordinates": "pixels", + "preprocess": "letterbox", + "dense_resize": "half_pixel", + "nms": true, + "conversion": "custom" +} diff --git a/src/cpp/models/yolo26/semantic/s/rk3588/int8/model.json b/src/cpp/models/yolo26/semantic/s/rk3588/int8/model.json new file mode 100644 index 0000000..3989920 --- /dev/null +++ b/src/cpp/models/yolo26/semantic/s/rk3588/int8/model.json @@ -0,0 +1,22 @@ +{ + "id": "yolo26s-sem-rk3588-int8", + "name": "yolo26s-sem", + "family": "yolo26", + "task": "semantic", + "backend": "yolo", + "chip": "rk3588", + "size": "s", + "precision": "int8", + "tag": "rk3588-yolo26s-sem-int8", + "url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/semantic/yolo26s-sem/rk3588/int8/model.rknn", + "sha256": "d65cd662b775a7177bcbab66331c14977517a7a688d34a7a29dd8b8c7e32c546", + "metadata_url": "https://huggingface.co/HanzoHuang/YOLO26-RKNN2/resolve/9b33ad58d4bbc3407de5caa8099681ad5550933a/models/v2.3.2/semantic/yolo26s-sem/rk3588/int8/metadata.yaml", + "metadata_sha256": "300cd5df77c92b4fc3c6bbe46c47caaf02f823903193e899559ebc76b91cb2f7", + "classes": 19, + "end2end": false, + "coordinates": "pixels", + "preprocess": "letterbox", + "dense_resize": "half_pixel", + "nms": true, + "conversion": "custom" +} diff --git a/src/cpp/native/bindings.cpp b/src/cpp/native/bindings.cpp new file mode 100644 index 0000000..5da8473 --- /dev/null +++ b/src/cpp/native/bindings.cpp @@ -0,0 +1,340 @@ +#include "engine/affinity.hpp" +#include "engine/task_json.hpp" +#include "engine/decoder.hpp" +#include "engine/detector_pool.hpp" +#include "engine/jpeg_preview.hpp" + +#include +#include +#include +#include +#include + +extern "C" +{ +#include +} + +namespace py = pybind11; +using namespace nvr; + +// ARM64 Linux uses little-endian IEEE floats. Append fields individually to avoid padding. +static std::string candidate_packet(const Frame &frame, const Json &result, + const std::vector &candidates) +{ + const uint32_t size = candidates.size() * 6 * sizeof(float); + std::string packet; + packet.reserve(32 + size); + auto append = [&](const auto &value) { + packet.append(reinterpret_cast(&value), sizeof(value)); + }; + append(size); + append(result.at("source_width").get()); + append(result.at("source_height").get()); + append(uint32_t(frame.cycle)); + append(result.at("media_time").get()); + append(frame.sequence); + for (const auto &candidate : candidates) + { + packet.append(reinterpret_cast(candidate.box.data()), 4 * sizeof(float)); + append(candidate.confidence); + append(float(candidate.class_id)); + } + return packet; +} + +// A request context is serialized; live workers have their own independent contexts. +class Predictor +{ + std::mutex mutex_; + std::unique_ptr image_; + Detector detector_; + +public: + Predictor(const std::string &model, const std::string &options) + : detector_(model, .25f, .45f, RKNN_NPU_CORE_ALL, model_options(Json::parse(options))) {} + + Json frame(const Frame &frame, float confidence, float iou) + { + prefer_performance_cpus(); + std::lock_guard lock(mutex_); + // Request buffers and live decoder pools can reuse file descriptors. + detector_.clear_sources(); + detector_.configure(confidence, iou); + try + { + auto result = detector_.infer(frame); + detector_.clear_sources(); + return result; + } + catch (...) + { + detector_.clear_sources(); + throw; + } + } + + std::string image(const std::string &pixels, int width, int height, float confidence, float iou) + { + prefer_performance_cpus(); + auto entered = Clock::now(); + std::lock_guard lock(mutex_); + double wait_ms = milliseconds(entered); + auto upload_started = Clock::now(); + require(width > 0 && height > 0 && width <= 8192 && height <= 8192 && + width % 2 == 0 && height % 2 == 0 && + pixels.size() == size_t(width) * height * 3 / 2, "invalid NV12 image"); + detector_.clear_sources(); + int hs = (width + 15) & ~15, vs = (height + 15) & ~15; + size_t size = size_t(hs) * vs * 3 / 2; + if (!image_ || image_->size < size) + { + image_ = std::make_unique(size); + } + dma_buf_sync sync{}; + sync.flags = DMA_BUF_SYNC_START | DMA_BUF_SYNC_WRITE; + require(ioctl(image_->fd, DMA_BUF_IOCTL_SYNC, &sync) == 0, "sync uploaded image"); + auto *base = static_cast(image_->data); + for (int row = 0; row < height; ++row) + std::memcpy(base + size_t(row) * hs, pixels.data() + size_t(row) * width, width); + for (int row = 0; row < height / 2; ++row) + std::memcpy(base + size_t(hs) * vs + size_t(row) * hs, + pixels.data() + size_t(width) * height + size_t(row) * width, width); + sync.flags = DMA_BUF_SYNC_END | DMA_BUF_SYNC_WRITE; + require(ioctl(image_->fd, DMA_BUF_IOCTL_SYNC, &sync) == 0, "finish uploaded image sync"); + MppBufferInfo info{}; + info.type = MPP_BUFFER_TYPE_EXT_DMA; + info.fd = image_->fd; + info.size = image_->size; + MppBuffer buffer = nullptr; + require(mpp_buffer_import(&buffer, &info) == MPP_OK, "import uploaded image"); + MppFrame raw = nullptr; + auto rc = mpp_frame_init(&raw); + if (rc != MPP_OK) { mpp_buffer_put(buffer); throw std::runtime_error("create image frame"); } + Frame frame(raw); + mpp_frame_set_width(raw, width); + mpp_frame_set_height(raw, height); + mpp_frame_set_hor_stride(raw, hs); + mpp_frame_set_ver_stride(raw, vs); + mpp_frame_set_fmt(raw, MPP_FMT_YUV420SP); + mpp_frame_set_buffer(raw, buffer); + mpp_buffer_put(buffer); + detector_.configure(confidence, iou); + double upload_ms = milliseconds(upload_started); + auto result = detector_.infer(frame); + result["context_wait_ms"] = wait_ms; + result["dma_upload_ms"] = upload_ms; + return result.dump(); + } + + std::string description() const { return detector_.description.dump(); } +}; + +class Live +{ + std::atomic stop_{false}; + FrameQueue queue_{6}; + // Destruction order: retained frames and consumers before the decoder. + std::unique_ptr decoder_; + std::unique_ptr pool_; + std::unique_ptr preview_; + std::mutex mutex_; + std::mutex preview_mutex_; + FramePtr latest_; + Json result_; + uint64_t candidate_sequence_ = 0; + std::string candidates_; + FramePtr preview_frame_; + Json preview_result_; + uint64_t jpeg_frame_ = 0; + std::vector jpeg_; + std::string error_; + bool ended_ = false; + std::thread thread_; + + Json status() const + { + return {{"error", error_}, {"ended", ended_}, {"dropped", queue_.dropped.load()}}; + } + +public: + Live(const std::string &source, const std::string &model, int workers, + const std::string &options, bool loop_file) + { + avdevice_register_all(); + decoder_ = std::make_unique(source, stop_, source.rfind("/dev/video", 0) == 0); + pool_ = std::make_unique(model, .25f, .45f, workers, model_options(Json::parse(options))); + const bool publish_candidates = loop_file && pool_->supports_candidates(); + pool_->description["candidate_stream"] = publish_candidates; + preview_ = std::make_unique(); + thread_ = std::thread([this, source, loop_file, publish_candidates] + { + std::thread producer; + try + { + producer = std::thread([this, source, loop_file] + { + try + { + const bool file = source.find("://") == std::string::npos && + source.rfind("/dev/", 0) != 0; + for (uint64_t cycle = 0; !stop_; ++cycle) + { + auto start = Clock::now(); + int64_t origin = loop_file ? decoder_->start_us() : AV_NOPTS_VALUE; + auto pace = [&](int64_t due_us) { + while (!stop_ && milliseconds(start) * 1000 < due_us) + std::this_thread::sleep_for(std::chrono::milliseconds(2)); + }; + bool received = false; + decoder_->run([&](FramePtr frame) + { + received = true; + if (file && frame->pts_us != AV_NOPTS_VALUE) + { + if (origin == AV_NOPTS_VALUE) origin = frame->pts_us; + pace(frame->pts_us - origin); + } + frame->cycle = cycle; + queue_.push(std::move(frame), stop_); + }); + if (!loop_file || stop_) break; + require(received, "video contains no decoded frames"); + pace(decoder_->duration_us()); + if (!stop_) decoder_->rewind(); + } + } + catch (const std::exception &e) + { + std::lock_guard lock(mutex_); + error_ = e.what(); + } + queue_.close(); + }); + auto last_preview = Clock::now() - std::chrono::seconds(1); + pool_->run(queue_, [this](const Frame &) { return !stop_; }, publish_candidates, + [&](const FramePtr &frame, Json result, const std::vector &unfiltered) + { + std::string packet; + if (loop_file && frame->pts_us != AV_NOPTS_VALUE) + { + result["media_time"] = (frame->pts_us - decoder_->start_us()) / 1000000.0; + result["cycle"] = frame->cycle; + if (publish_candidates) + packet = candidate_packet(*frame, result, unfiltered); + } + std::lock_guard lock(mutex_); + latest_ = frame; + result_ = std::move(result); + if (!packet.empty()) + { + candidate_sequence_ = frame->sequence; + candidates_ = std::move(packet); + } + // Keep the legacy preview at 10 FPS and encode only when it is read. + if (milliseconds(last_preview) >= 100) + { + preview_frame_ = frame; + preview_result_ = result_; + last_preview = Clock::now(); + } + }); + } + catch (const std::exception &e) + { + std::lock_guard lock(mutex_); + error_ = e.what(); + stop_ = true; + queue_.close(); + } + if (producer.joinable()) producer.join(); + std::lock_guard lock(mutex_); + ended_ = true; + }); + } + + ~Live() { close(); } + void close() + { + stop_ = true; + queue_.close(); + if (thread_.joinable()) thread_.join(); + queue_.clear(); + { + std::lock_guard preview_lock(preview_mutex_); + std::lock_guard lock(mutex_); + latest_.reset(); + preview_frame_.reset(); + preview_.reset(); + } + pool_.reset(); + decoder_.reset(); + } + void configure(float confidence, float iou) { pool_->configure(confidence, iou); } + std::string description() const { return pool_->description.dump(); } + std::string predict(Predictor &predictor, float confidence, float iou) + { + FramePtr frame; + { + std::lock_guard lock(mutex_); + frame = latest_; + } + require(bool(frame), "live source has no frame yet"); + return predictor.frame(*frame, confidence, iou).dump(); + } + py::tuple snapshot() + { + std::lock_guard preview_lock(preview_mutex_); + FramePtr frame; + Json state; + { + std::lock_guard lock(mutex_); + frame = preview_frame_; + state = status(); + state["result"] = preview_result_; + } + if (frame && frame->sequence != jpeg_frame_) + { + jpeg_ = preview_->encode(*frame); + jpeg_frame_ = frame->sequence; + } + return py::make_tuple(state.dump(), py::bytes(reinterpret_cast(jpeg_.data()), jpeg_.size())); + } + + std::string state(bool include_result, uint64_t after) + { + std::lock_guard lock(mutex_); + auto current = status(); + if (include_result) + current["result"] = latest_ && latest_->sequence > after ? result_ : Json(); + return current.dump(); + } + + py::object candidates(uint64_t after) + { + std::lock_guard lock(mutex_); + require(error_.empty(), error_); + if (candidate_sequence_ <= after) return py::none(); + return py::make_tuple(candidate_sequence_, py::bytes(candidates_)); + } +}; + +PYBIND11_MODULE(_native, m) +{ + py::class_(m, "Predictor") + .def(py::init(), py::arg("model"), py::arg("options") = "{}") + .def("predict", &Predictor::image, py::call_guard()) + .def("description", &Predictor::description); + py::class_(m, "Live") + .def(py::init(), + py::arg("source"), py::arg("model"), py::arg("workers") = 0, py::arg("options") = "{}", + py::arg("loop_file") = false) + .def("configure", &Live::configure) + .def("description", &Live::description) + .def("predict", &Live::predict, py::call_guard()) + .def("snapshot", &Live::snapshot) + .def("state", &Live::state, py::arg("include_result") = false, py::arg("after") = 0, + py::call_guard()) + .def("candidates", &Live::candidates) + .def("close", &Live::close, py::call_guard()); +} diff --git a/src/cpp/native/engine/affinity.hpp b/src/cpp/native/engine/affinity.hpp new file mode 100644 index 0000000..8e1eb41 --- /dev/null +++ b/src/cpp/native/engine/affinity.hpp @@ -0,0 +1,30 @@ +#pragma once + +#include + +namespace nvr +{ +inline void prefer_performance_cpus() +{ + // RK3576 and RK3588 place their performance CPUs at IDs 4..7. Keep + // inference on those CPUs without overriding a caller's restricted CPU set. + cpu_set_t allowed, performance; + CPU_ZERO(&performance); + if (sched_getaffinity(0, sizeof(allowed), &allowed) != 0) + { + return; + } + for (int cpu = 4; cpu < 8; ++cpu) + { + if (CPU_ISSET(cpu, &allowed)) + { + CPU_SET(cpu, &performance); + } + } + if (CPU_COUNT(&performance) > 0) + { + // Affinity is a best-effort optimization, never a prerequisite to inference. + (void)sched_setaffinity(0, sizeof(performance), &performance); + } +} +} // namespace nvr diff --git a/src/cpp/native/engine/common.hpp b/src/cpp/native/engine/common.hpp new file mode 100644 index 0000000..fe807f5 --- /dev/null +++ b/src/cpp/native/engine/common.hpp @@ -0,0 +1,23 @@ +#pragma once + +#include +#include +#include + +namespace nvr +{ +using Clock = std::chrono::steady_clock; + +inline double milliseconds(Clock::time_point start) +{ + return std::chrono::duration(Clock::now() - start).count(); +} + +inline void require(bool ok, const std::string &message) +{ + if (!ok) + { + throw std::runtime_error(message); + } +} +} // namespace nvr diff --git a/src/cpp/native/engine/decoder.cpp b/src/cpp/native/engine/decoder.cpp new file mode 100644 index 0000000..5baf681 --- /dev/null +++ b/src/cpp/native/engine/decoder.cpp @@ -0,0 +1,305 @@ +#include "engine/decoder.hpp" + +#include +#include + + +namespace nvr +{ +int Decoder::interrupt(void *opaque) +{ + auto *self = static_cast(opaque); + const auto now = + std::chrono::duration_cast(Clock::now().time_since_epoch()) + .count(); + return self->stop_ || now > self->deadline_ns_; +} + +void Decoder::deadline() +{ + deadline_ns_ = std::chrono::duration_cast( + (Clock::now() + std::chrono::seconds(8)).time_since_epoch()) + .count(); +} + +// Decoder lifetime. +Decoder::Decoder(const std::string &uri, std::atomic &stop, bool camera) : stop_(stop) +{ + try + { + format_ = avformat_alloc_context(); + require(format_, "allocate demux context"); + format_->interrupt_callback = {interrupt, this}; + deadline(); + AVDictionary *options = nullptr; + av_dict_set(&options, "rtsp_transport", "tcp", 0); + av_dict_set(&options, "rw_timeout", "5000000", 0); + if (camera) + { + av_dict_set(&options, "input_format", "h264", 0); + } + int rc = + avformat_open_input(&format_, uri.c_str(), + camera ? av_find_input_format("video4linux2") : nullptr, &options); + av_dict_free(&options); + require(rc >= 0, "open input: " + av_error(rc)); + deadline(); + rc = avformat_find_stream_info(format_, nullptr); + require(rc >= 0, "read stream info: " + av_error(rc)); + stream_ = av_find_best_stream(format_, AVMEDIA_TYPE_VIDEO, -1, -1, nullptr, 0); + require(stream_ >= 0, "no video stream"); + auto codec = parameters()->codec_id; + require(codec == AV_CODEC_ID_H264 || codec == AV_CODEC_ID_HEVC, + "demo supports H.264/H.265 video"); + const auto *filter = + av_bsf_get_by_name(codec == AV_CODEC_ID_H264 ? "h264_mp4toannexb" : "hevc_mp4toannexb"); + require(filter && av_bsf_alloc(filter, &bsf_) >= 0, "create Annex-B filter"); + require(avcodec_parameters_copy(bsf_->par_in, parameters()) >= 0, + "copy decoder parameters"); + bsf_->time_base_in = time_base(); + require(av_bsf_init(bsf_) >= 0, "init Annex-B filter"); + require(mpp_create(&context_, &api_) == MPP_OK, "mpp_create"); + RK_U32 split = 1; + require(api_->control(context_, MPP_DEC_SET_PARSER_SPLIT_MODE, &split) == MPP_OK, + "MPP split parser"); + const auto init_result = + mpp_init(context_, MPP_CTX_DEC, + codec == AV_CODEC_ID_H264 ? MPP_VIDEO_CodingAVC : MPP_VIDEO_CodingHEVC); + require(init_result == MPP_OK, + "MPP decoder initialization failed (" + std::to_string(init_result) + + "): check the board MPP driver and read/write permissions on /dev/mpp_service"); + MppFrameFormat fmt = MPP_FMT_YUV420SP; + require(api_->control(context_, MPP_DEC_SET_OUTPUT_FORMAT, &fmt) == MPP_OK, + "set linear NV12 output"); + RK_S64 timeout = MPP_POLL_NON_BLOCK; + require(api_->control(context_, MPP_SET_OUTPUT_TIMEOUT, &timeout) == MPP_OK, + "set nonblocking decode output"); + } + catch (...) + { + if (context_) + { + mpp_destroy(context_); + } + context_ = nullptr; + if (bsf_) + { + av_bsf_free(&bsf_); + } + if (format_) + { + avformat_close_input(&format_); + } + throw; + } +} + +Decoder::~Decoder() +{ + // A seek/stop may leave reordered pictures inside MPP. Flush those slots + // after consumers release their frames, before destroying the decoder. + if (context_) + { + api_->reset(context_); + mpp_destroy(context_); + } + if (group_) + { + mpp_buffer_group_put(group_); + } + if (bsf_) + { + av_bsf_free(&bsf_); + } + if (format_) + { + avformat_close_input(&format_); + } +} + +// MPP packet submission and frame delivery. +bool Decoder::drain(const std::function &consume, bool &eos) +{ + bool progress = false; + for (;;) + { + MppFrame raw = nullptr; + MPP_RET rc = api_->decode_get_frame(context_, &raw); + require(rc == MPP_OK || rc == MPP_ERR_TIMEOUT, "MPP get frame error " + std::to_string(rc)); + if (!raw) + { + break; + } + progress = true; + progress_ = Clock::now(); + auto frame = std::make_shared(raw); + if (mpp_frame_get_info_change(raw)) + { + // Retain old frame ownership in consumers until their RGA work completes. + if (!group_) + { + require(mpp_buffer_group_get_internal(&group_, MPP_BUFFER_TYPE_DRM) == MPP_OK, + "create MPP frame pool"); + } + require(mpp_buffer_group_limit_config(group_, mpp_frame_get_buf_size(raw), 24) == + MPP_OK, + "size MPP frame pool"); + require(api_->control(context_, MPP_DEC_SET_EXT_BUF_GROUP, group_) == MPP_OK, + "bind MPP frame pool"); + require(api_->control(context_, MPP_DEC_SET_INFO_CHANGE_READY, nullptr) == MPP_OK, + "acknowledge MPP format change"); + continue; + } + eos = mpp_frame_get_eos(raw) != 0; + if (mpp_frame_get_errinfo(raw) || mpp_frame_get_discard(raw)) + { + continue; + } + if (!mpp_frame_get_buffer(raw)) + { + continue; + } + frame->sequence = ++sequence_; + frame->pts_us = mpp_frame_get_pts(raw); + consume(std::move(frame)); + } + return progress; +} + +void Decoder::submit(AVPacket *avpacket, bool end, const std::function &consume, + bool &eos) +{ + MppPacket packet = nullptr; + require(mpp_packet_init(&packet, avpacket ? avpacket->data : nullptr, + avpacket ? static_cast(avpacket->size) : 0) == MPP_OK, + "create MPP packet"); + if (end) + { + mpp_packet_set_eos(packet); + } + if (avpacket && avpacket->pts != AV_NOPTS_VALUE) + { + mpp_packet_set_pts( + packet, av_rescale_q(avpacket->pts, bsf_->time_base_out, AVRational{1, 1000000})); + } + try + { + while (!stop_) + { + MPP_RET rc = api_->decode_put_packet(context_, packet); + if (rc == MPP_OK) + { + progress_ = Clock::now(); + break; + } + require(rc == MPP_ERR_BUFFER_FULL || rc == MPP_NOK, + "MPP submit error " + std::to_string(rc)); + drain(consume, eos); + require(milliseconds(progress_) < 8000, "MPP submit stalled"); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + mpp_packet_deinit(&packet); + } + catch (...) + { + mpp_packet_deinit(&packet); + throw; + } + drain(consume, eos); +} + +int64_t Decoder::start_us() const +{ + return format_->start_time == AV_NOPTS_VALUE ? 0 : format_->start_time; +} + +int64_t Decoder::duration_us() const +{ + return format_->duration == AV_NOPTS_VALUE ? 0 : format_->duration; +} + +void Decoder::rewind() +{ + deadline(); + const auto start = av_rescale_q(start_us(), AV_TIME_BASE_Q, time_base()); + const int result = av_seek_frame(format_, stream_, start, AVSEEK_FLAG_BACKWARD); + if (stop_) return; + require(result >= 0, "rewind video: " + av_error(result)); + av_bsf_flush(bsf_); + require(api_->reset(context_) == MPP_OK, "reset MPP decoder"); + progress_ = Clock::now(); +} + +void Decoder::run(const std::function &consume) +{ + auto free_packet = [](AVPacket *p) + { + av_packet_free(&p); + }; + std::unique_ptr packet(av_packet_alloc(), free_packet), + filtered(av_packet_alloc(), free_packet); + require(packet && filtered, "allocate demux packets"); + bool eos = false, awaiting_key = true; + auto drain_filter = [&] + { + for (;;) + { + int rc = av_bsf_receive_packet(bsf_, filtered.get()); + if (rc == AVERROR(EAGAIN) || rc == AVERROR_EOF) + { + break; + } + require(rc >= 0, "Annex-B receive: " + av_error(rc)); + submit(filtered.get(), false, consume, eos); + av_packet_unref(filtered.get()); + } + }; + while (!stop_) + { + deadline(); + int rc = av_read_frame(format_, packet.get()); + if (rc == AVERROR_EOF) + { + break; + } + if (stop_) + { + break; + } + require(rc >= 0, "read input: " + av_error(rc)); + if (packet->stream_index == stream_) + { + if (awaiting_key) + { + if (!(packet->flags & AV_PKT_FLAG_KEY) || + (packet->pts == AV_NOPTS_VALUE && packet->dts == AV_NOPTS_VALUE)) + { + av_packet_unref(packet.get()); + continue; + } + awaiting_key = false; + } + rc = av_bsf_send_packet(bsf_, packet.get()); + require(rc >= 0, "Annex-B submit: " + av_error(rc)); + drain_filter(); + } + av_packet_unref(packet.get()); + } + if (!stop_) + { + require(av_bsf_send_packet(bsf_, nullptr) >= 0, "flush Annex-B"); + drain_filter(); + submit(nullptr, true, consume, eos); + while (!eos && !stop_) + { + drain(consume, eos); + require(milliseconds(progress_) < 8000, "MPP EOS stalled"); + if (!eos) + { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + } + } +} + +} // namespace nvr diff --git a/src/cpp/native/engine/decoder.hpp b/src/cpp/native/engine/decoder.hpp new file mode 100644 index 0000000..233b2c7 --- /dev/null +++ b/src/cpp/native/engine/decoder.hpp @@ -0,0 +1,61 @@ +#pragma once + +#include "engine/frame.hpp" +#include "engine/media.hpp" + +#include +#include + +extern "C" +{ +#include +} + +namespace nvr +{ +// FFmpeg input and MPP hardware decoding. +class Decoder +{ + AVFormatContext *format_ = nullptr; + AVBSFContext *bsf_ = nullptr; + MppCtx context_ = nullptr; + MppApi *api_ = nullptr; + MppBufferGroup group_ = nullptr; + int stream_ = -1; + std::atomic &stop_; + std::atomic deadline_ns_{0}; + uint64_t sequence_ = 0; + Clock::time_point progress_ = Clock::now(); + + static int interrupt(void *opaque); + void deadline(); + + bool drain(const std::function &consume, bool &eos); + void submit(AVPacket *packet, bool end_of_stream, const std::function &consume, + bool &received_eos); + +public: + Decoder(const std::string &uri, std::atomic &stop, bool camera = false); + ~Decoder(); + + Decoder(const Decoder &) = delete; + Decoder &operator=(const Decoder &) = delete; + + const AVCodecParameters *parameters() const + { + return format_->streams[stream_]->codecpar; + } + + AVRational time_base() const + { + return format_->streams[stream_]->time_base; + } + + int64_t start_us() const; + int64_t duration_us() const; + void rewind(); + + // Decoded frames transfer shared ownership to the consumer. + void run(const std::function &consume); +}; +} // namespace nvr diff --git a/src/cpp/native/engine/detector.cpp b/src/cpp/native/engine/detector.cpp new file mode 100644 index 0000000..60d193b --- /dev/null +++ b/src/cpp/native/engine/detector.cpp @@ -0,0 +1,420 @@ +#include "engine/detector.hpp" +#include "engine/task_json.hpp" + +#include + +#include + +namespace nvr +{ +Detector::Detector(const std::string &model, float threshold, float iou, + rknn_core_mask core_mask, const ModelOptions &options) + : options_(options), confidence_(threshold), nms_iou_(iou) +{ + try + { + std::ifstream file(model, std::ios::binary); + require(bool(file), "open RKNN model"); + std::vector bytes((std::istreambuf_iterator(file)), {}); + require(!bytes.empty(), "empty RKNN model"); + require(rknn_init(&context_, bytes.data(), static_cast(bytes.size()), 0, + nullptr) == RKNN_SUCC, + "initialize RKNN model"); + require(rknn_set_core_mask(context_, core_mask) == 0, "enable available NPU cores"); + + // Validate the model input contract before allocating hardware buffers. + rknn_input_output_num io{}; + require(rknn_query(context_, RKNN_QUERY_IN_OUT_NUM, &io, sizeof(io)) == 0, + "query RKNN I/O"); + require(io.n_input == 1 && io.n_output > 0 && io.n_output <= 16, + "Model requires one image input and 1..16 output tensors"); + require(rknn_query(context_, RKNN_QUERY_INPUT_ATTR, &input_attr_, sizeof(input_attr_)) == 0, + "query RKNN input"); + require(rknn_query(context_, RKNN_QUERY_OUTPUT_ATTR, &output_attr_, sizeof(output_attr_)) == + 0, + "query RKNN output"); + require(input_attr_.n_dims == 4 && input_attr_.dims[0] == 1, + "model input must be batch-1 image"); + bool nhwc = input_attr_.fmt == RKNN_TENSOR_NHWC; + height_ = static_cast(input_attr_.dims[nhwc ? 1 : 2]); + width_ = static_cast(input_attr_.dims[nhwc ? 2 : 3]); + require(input_attr_.dims[nhwc ? 3 : 1] == 3 && width_ % 16 == 0 && height_ % 2 == 0, + "input needs RGB, width aligned to 16 and even height"); + + std::vector specs; + output_attrs_.resize(io.n_output); + for (uint32_t i = 0; i < io.n_output; ++i) + { + auto &attr = output_attrs_[i]; + attr.index = i; + require(rknn_query(context_, RKNN_QUERY_OUTPUT_ATTR, &attr, sizeof(attr)) == 0, + "query model output"); + specs.push_back({attr.name, std::vector(attr.dims, attr.dims + attr.n_dims), + attr.fmt != RKNN_TENSOR_NHWC}); + } + task_postprocessor_ = std::make_unique(options_, specs, width_, height_); + const auto layout_kind = task_postprocessor_->layout(); + // Preserve the quantized detection fast path; task heads use reusable float buffers. + const bool fast_dfl = + layout_kind == TaskPostprocessor::Layout::Dfl && !options_.end_to_end && + io.n_output == 9 && + std::all_of(output_attrs_.begin(), output_attrs_.end(), + [](const rknn_tensor_attr &attr) + { + return attr.type == RKNN_TENSOR_INT8 && + attr.qnt_type == RKNN_TENSOR_QNT_AFFINE_ASYMMETRIC && + attr.scale > 0; + }) && + output_attrs_[0].dims[1] == 64 && output_attrs_[3].dims[1] == 64 && + output_attrs_[6].dims[1] == 64; + task_output_ = options_.task != ModelTask::Detect || + layout_kind == TaskPostprocessor::Layout::EndToEnd || + task_postprocessor_->classes() > 1020 || + (layout_kind == TaskPostprocessor::Layout::Split && options_.normalized) || + (layout_kind == TaskPostprocessor::Layout::Dfl && !fast_dfl) || + (layout_kind == TaskPostprocessor::Layout::Decoded && + (options_.normalized || + task_postprocessor_->proposals() <= task_postprocessor_->classes() + 4)); + if (task_output_) + { + if (options_.task == ModelTask::Semantic || + (options_.task == ModelTask::Depth && options_.dense_resize != DenseResize::None)) + { + dense_output_ = std::make_unique(context_, output_attr_); + if (!dense_output_->bound()) + { + dense_output_.reset(); + } + } + if (!dense_output_) + { + task_outputs_.resize(specs.size()); + for (size_t i = 0; i < specs.size(); ++i) + { + task_outputs_[i].resize(specs[i].elements()); + } + } + features_ = task_postprocessor_->classes() + 4; + } + else + { + // Select the output decoder and allocate its reusable result storage. + dfl_ = io.n_output == 9; + split_ = io.n_output == 2; + if (dfl_) + { + output_attrs_.resize(9); + quant_outputs_.resize(9); + proposals_ = 0; + for (uint32_t i = 0; i < 9; ++i) + { + auto &a = output_attrs_[i]; + a.index = i; + require(rknn_query(context_, RKNN_QUERY_OUTPUT_ATTR, &a, sizeof(a)) == 0, + "query DFL output"); + require(a.n_dims == 4 && a.dims[0] == 1 && a.fmt == RKNN_TENSOR_NCHW && + a.type == RKNN_TENSOR_INT8 && + a.qnt_type == RKNN_TENSOR_QNT_AFFINE_ASYMMETRIC && a.scale > 0, + "DFL requires affine INT8 NCHW outputs"); + quant_outputs_[i].resize(a.n_elems); + if (i % 3 == 0) + { + require(a.dims[1] == 64 && a.dims[2] > 0 && a.dims[3] > 0, + "DFL needs 16 bins per side"); + require(width_ % int(a.dims[3]) == 0 && height_ % int(a.dims[2]) == 0 && + width_ / int(a.dims[3]) == height_ / int(a.dims[2]), + "DFL stride mismatch"); + proposals_ += int(a.dims[2] * a.dims[3]); + } + else + { + const auto &b = output_attrs_[i - i % 3]; + require(a.dims[2] == b.dims[2] && a.dims[3] == b.dims[3], + "DFL grid mismatch"); + if (i % 3 == 1) + { + if (i == 1) + { + features_ = int(a.dims[1]) + 4; + } + require(a.dims[1] == uint32_t(features_ - 4) && features_ > 4 && + features_ <= 1024, + "DFL class mismatch"); + } + else + { + require(a.dims[1] == 1, "DFL confidence sum must have one channel"); + } + } + } + } + else if (split_) + { + output_attrs_.resize(2); + for (uint32_t i = 0; i < 2; ++i) + { + auto &a = output_attrs_[i]; + a.index = i; + require(rknn_query(context_, RKNN_QUERY_OUTPUT_ATTR, &a, sizeof(a)) == 0, + "query YOLO26 output"); + require(a.n_dims == 3 && a.dims[0] == 1 && a.dims[1] > 0 && a.dims[2] > 0, + "YOLO26 outputs need batch-1 rank-3 tensors"); + if (a.dims[1] == 4 || a.dims[2] == 4) + { + if (boxes_index_ < 0 || + std::string(a.name).find("Mul_2") != std::string::npos) + { + boxes_index_ = int(i); + } + } + } + require(boxes_index_ >= 0, "YOLO26 requires a four-coordinate boxes output"); + scores_index_ = 1 - boxes_index_; + const auto &b = output_attrs_[size_t(boxes_index_)]; + const auto &s = output_attrs_[size_t(scores_index_)]; + boxes_first_ = b.dims[1] == 4; + proposals_ = int(b.dims[boxes_first_ ? 2 : 1]); + scores_first_ = s.dims[2] == uint32_t(proposals_); + require(s.dims[scores_first_ ? 2 : 1] == uint32_t(proposals_), + "YOLO26 score/box proposal count mismatch"); + features_ = int(s.dims[scores_first_ ? 1 : 2]) + 4; + require(features_ > 4 && features_ <= 1024 && proposals_ > 0, + "unsupported YOLO26 class count"); + require(s.n_elems == uint32_t(features_ - 4) * uint32_t(proposals_) && + b.n_elems == 4 * uint32_t(proposals_), + "YOLO26 tensor size mismatch"); + output_.resize(size_t(features_) * size_t(proposals_)); + } + else + { + require(output_attr_.n_dims == 3 && output_attr_.dims[0] == 1, + "unsupported YOLO output rank/batch"); + features_first_ = output_attr_.dims[1] < output_attr_.dims[2]; + features_ = static_cast(output_attr_.dims[features_first_ ? 1 : 2]); + proposals_ = static_cast(output_attr_.dims[features_first_ ? 2 : 1]); + require(features_ > 4 && features_ <= 1024 && proposals_ > features_, + "unsupported YOLO output layout"); + } + + YoloLayout layout; + layout.kind = dfl_ ? YoloLayout::Kind::Dfl + : split_ ? YoloLayout::Kind::ScoresAndBoxes + : YoloLayout::Kind::Decoded; + layout.input_width = width_; + layout.input_height = height_; + layout.classes = features_ - 4; + layout.apply_nms = options_.apply_nms && !options_.end_to_end; + layout.proposals = proposals_; + layout.features_first = features_first_; + layout.scores_first = scores_first_; + layout.boxes_first = boxes_first_; + if (dfl_) + { + for (size_t branch = 0; branch < layout.grids.size(); ++branch) + { + const auto &boxes = output_attrs_[branch * 3]; + const auto &scores = output_attrs_[branch * 3 + 1]; + const auto &sums = output_attrs_[branch * 3 + 2]; + layout.grids[branch] = { + int(boxes.dims[3]), int(boxes.dims[2]), boxes.zp, scores.zp, sums.zp, + boxes.scale, scores.scale, sums.scale}; + } + } + postprocessor_ = std::make_unique(layout, threshold, iou); + } + + // Share one RGB DMA buffer between RGA preprocessing and RKNN input. + stride_ = input_attr_.w_stride ? static_cast(input_attr_.w_stride) : width_; + require(stride_ >= width_ && stride_ % 16 == 0, "unsupported RKNN width stride"); + input_ = std::make_unique( + std::max(input_attr_.size_with_stride, size_t(stride_) * height_ * 3)); + tensor_ = rknn_create_mem_from_fd(context_, input_->fd, input_->data, + static_cast(input_->size), 0); + require(tensor_, "import RGA destination DMA-BUF into RKNN"); + input_attr_.type = RKNN_TENSOR_UINT8; + input_attr_.fmt = RKNN_TENSOR_NHWC; + input_attr_.pass_through = 0; + input_attr_.h_stride = static_cast(height_); + require(rknn_set_io_mem(context_, tensor_, &input_attr_) == 0, + "bind external RGB input to RKNN"); + if (!task_output_ && !dfl_ && !split_) + { + output_.resize(output_attr_.n_elems); + } + preprocessor_ = std::make_unique(*input_, width_, height_, stride_, + options_.center_crop); + rknn_sdk_version version{}; + require(rknn_query(context_, RKNN_QUERY_SDK_VERSION, &version, sizeof(version)) == 0, + "query RKNN version"); + description = { + {"runtime", version.api_version}, + {"driver", version.drv_version}, + {"input_width", width_}, + {"input_height", height_}, + {"input_stride", stride_}, + {"classes", task_postprocessor_->classes()}, + {"task", task_name(options_.task)}, + {"nms", options_.apply_nms && !options_.end_to_end}, + {"proposals", task_postprocessor_->proposals()}, + {"preprocess", "rga"}, + {"output_type", get_type_string(output_attr_.type)}, + {"postprocessor", task_output_ ? task_postprocessor_->description() + : dfl_ ? "INT8 DFL (9 outputs)" + : split_ ? "YOLO26 scores/xyxy (2 outputs)" + : "decoded YOLO (1 output)"}, + {"core_mask", core_mask == RKNN_NPU_CORE_ALL ? Json("all") : Json(int(core_mask))}, + {"input_api", "rknn_create_mem_from_fd + rknn_set_io_mem"}}; + } + catch (...) + { + dense_output_.reset(); + preprocessor_.reset(); + if (tensor_) + { + rknn_destroy_mem(context_, tensor_); + } + tensor_ = nullptr; + input_.reset(); + if (context_) + { + rknn_destroy(context_); + } + context_ = 0; + throw; + } +} + +Detector::~Detector() +{ + dense_output_.reset(); + preprocessor_.reset(); + if (tensor_) + { + rknn_destroy_mem(context_, tensor_); + } + input_.reset(); + if (context_) + { + rknn_destroy(context_); + } +} + +void Detector::run_output() +{ + auto started = Clock::now(); + require(rknn_run(context_, nullptr) == 0, "RKNN inference"); + last_run_ms_ = milliseconds(started); + started = Clock::now(); + if (dense_output_) + { + dense_output_->sync(); + } + else if (task_output_) + { + std::vector outputs(task_outputs_.size()); + for (size_t i = 0; i < outputs.size(); ++i) + { + outputs[i].index = uint32_t(i); + outputs[i].want_float = 1; + outputs[i].is_prealloc = 1; + outputs[i].buf = task_outputs_[i].data(); + outputs[i].size = uint32_t(task_outputs_[i].size() * sizeof(float)); + } + require(rknn_outputs_get(context_, uint32_t(outputs.size()), outputs.data(), nullptr) == 0, + "get task outputs"); + require(rknn_outputs_release(context_, uint32_t(outputs.size()), outputs.data()) == 0, + "release task outputs"); + } + else if (dfl_) + { + std::vector outputs(9); + for (uint32_t i = 0; i < 9; ++i) + { + auto &out = outputs[i]; + out.index = i; + out.want_float = 0; + out.is_prealloc = 1; + out.buf = quant_outputs_[i].data(); + out.size = uint32_t(quant_outputs_[i].size()); + } + require(rknn_outputs_get(context_, 9, outputs.data(), nullptr) == 0, + "get quantized RKNN outputs"); + require(rknn_outputs_release(context_, 9, outputs.data()) == 0, + "release quantized RKNN outputs"); + } + else if (split_) + { + rknn_output outputs[2]{}; + for (uint32_t i = 0; i < 2; ++i) + { + auto &out = outputs[i]; + out.index = i; + out.want_float = 1; + out.is_prealloc = 1; + out.buf = output_.data() + + (int(i) == scores_index_ ? 0 : size_t(features_ - 4) * size_t(proposals_)); + out.size = static_cast(output_attrs_[i].n_elems * sizeof(float)); + } + require(rknn_outputs_get(context_, 2, outputs, nullptr) == 0, + "get YOLO26 scores and boxes"); + require(rknn_outputs_release(context_, 2, outputs) == 0, "release YOLO26 outputs"); + } + else + { + rknn_output out{}; + out.want_float = 1; + out.is_prealloc = 1; + out.buf = output_.data(); + out.size = static_cast(output_.size() * sizeof(float)); + require(rknn_outputs_get(context_, 1, &out, nullptr) == 0, "get RKNN output"); + require(rknn_outputs_release(context_, 1, &out) == 0, "release RKNN output"); + } + last_output_ms_ = milliseconds(started); +} + +Json Detector::infer(const Frame &f, std::vector *unfiltered) +{ + auto pre = Clock::now(); + auto geometry = preprocessor_->process(f.image()); + double pre_ms = milliseconds(pre); + auto start = Clock::now(); + run_output(); + double infer_ms = milliseconds(start); + double run_ms = last_run_ms_, output_ms = last_output_ms_; + auto post = Clock::now(); + Json result = {{"task", task_name(options_.task)}, {"detections", Json::array()}}; + double nms_ms = 0; + if (dense_output_) + { + result = task_result_json( + task_postprocessor_->decode_dense(geometry, dense_output_->view()), options_.task); + } + else if (task_output_) + { + result = task_result_json( + task_postprocessor_->decode(geometry, task_outputs_, confidence_, nms_iou_), + options_.task); + } + else + { + for (const auto &detection : postprocessor_->decode(geometry, output_, quant_outputs_, &nms_ms, unfiltered)) + { + result["detections"].push_back({{"class_id", detection.class_id}, + {"confidence", detection.confidence}, + {"box", detection.box}}); + } + } + if (!task_output_) result["nms_ms"] = nms_ms; + double post_ms = milliseconds(post); + result.update(Json{{"frame", f.sequence}, + {"source_width", geometry.width}, + {"source_height", geometry.height}, + {"preprocess_ms", pre_ms}, + {"inference_ms", infer_ms}, + {"rknn_run_ms", run_ms}, + {"output_get_ms", output_ms}, + {"postprocess_ms", post_ms}}); + return result; +} + +} // namespace nvr diff --git a/src/cpp/native/engine/detector.hpp b/src/cpp/native/engine/detector.hpp new file mode 100644 index 0000000..eda8209 --- /dev/null +++ b/src/cpp/native/engine/detector.hpp @@ -0,0 +1,66 @@ +#pragma once + +#include "engine/frame.hpp" +#include "engine/json.hpp" +#include "engine/preprocess.hpp" +#include "engine/postprocess.hpp" +#include "engine/task_model.hpp" +#include "engine/native_output.hpp" + +#include +#include +#include +#include + +namespace nvr +{ +// RKNN inference and task postprocessing. +class Detector +{ + rknn_context context_ = 0; + rknn_tensor_attr input_attr_{}, output_attr_{}; + rknn_tensor_mem *tensor_ = nullptr; + std::unique_ptr input_; + std::unique_ptr preprocessor_; + std::unique_ptr postprocessor_; + std::unique_ptr task_postprocessor_; + std::vector> task_outputs_; + std::unique_ptr dense_output_; + ModelOptions options_; + bool task_output_ = false; + float confidence_ = 0.35f, nms_iou_ = 0.45f; + std::vector output_; + int width_ = 0, height_ = 0, stride_ = 0; + std::vector output_attrs_; + std::vector> quant_outputs_; + bool dfl_ = false; + bool split_ = false, scores_first_ = true, boxes_first_ = true; + int scores_index_ = -1, boxes_index_ = -1; + int features_ = 0, proposals_ = 0; + bool features_first_ = true; + double last_run_ms_ = 0, last_output_ms_ = 0; + + void run_output(); + +public: + Json description; + + Detector(const std::string &model, float confidence, float iou, rknn_core_mask core_mask = RKNN_NPU_CORE_ALL, + const ModelOptions &options = {}); + ~Detector(); + + Detector(const Detector &) = delete; + Detector &operator=(const Detector &) = delete; + + bool supports_candidates() const { return split_ && !task_output_; } + Json infer(const Frame &frame, std::vector *unfiltered = nullptr); + void configure(float confidence, float iou) + { + confidence_ = confidence; + nms_iou_ = iou; + if (postprocessor_) + postprocessor_->configure(confidence, iou); + } + void clear_sources() { preprocessor_->clear_sources(); } +}; +} // namespace nvr diff --git a/src/cpp/native/engine/detector_pool.cpp b/src/cpp/native/engine/detector_pool.cpp new file mode 100644 index 0000000..f09cacf --- /dev/null +++ b/src/cpp/native/engine/detector_pool.cpp @@ -0,0 +1,138 @@ +#include "engine/detector_pool.hpp" +#include "engine/affinity.hpp" + +#include +#include + +namespace nvr +{ +DetectorPool::DetectorPool(const std::string &model, + float confidence, float iou, int workers, const ModelOptions &options) +{ + configure(confidence, iou); + std::ifstream compatible("/proc/device-tree/compatible", std::ios::binary); + std::string chip((std::istreambuf_iterator(compatible)), {}); + const int cores = chip.find("rockchip,rk3588") != std::string::npos ? 3 + : chip.find("rockchip,rk3576") != std::string::npos ? 2 + : 1; + require(workers == 0 || workers == 1, "inference_workers must be 0 (auto) or 1 (single context)"); + if (workers == 0) + { + workers = cores; + } + Json masks = Json::array(); + for (int i = 0; i < workers; ++i) + { + auto mask = workers == 1 ? RKNN_NPU_CORE_ALL : static_cast(1 << i); + detectors_.push_back( + std::make_unique(model, confidence, iou, mask, options)); + masks.push_back(detectors_.back()->description.at("core_mask")); + } + description = detectors_.front()->description; + description["inference_workers"] = workers; + description["core_masks"] = masks; + description["prefer_performance_cpus"] = cores > 1; + description["core_mask"] = workers == 1 ? "all" : "per-worker"; +} + +void DetectorPool::run(FrameQueue &queue, const std::function &active, + bool publish_candidates, + const std::function &)> &consume) +{ + std::mutex mutex, dequeue_mutex; + std::condition_variable ready; + uint64_t issued = 0, published = 0; + std::atomic failed{false}; + std::exception_ptr error; + std::vector threads; + auto fail = [&] + { + std::lock_guard lock(mutex); + if (!error) + { + error = std::current_exception(); + } + failed = true; + queue.close(); + ready.notify_all(); + }; + try + { + for (auto &detector : detectors_) + { + threads.emplace_back( + [&, worker = detector.get()] + { + try + { + if (description.at("prefer_performance_cpus").get()) + { + prefer_performance_cpus(); + } + std::vector unfiltered; + while (true) + { + FramePtr frame; + uint64_t ticket; + { + // Assign tickets in source order, including across idle workers. + std::lock_guard lock(dequeue_mutex); + if (failed) + { + break; + } + frame = queue.pop(); + if (!frame) + { + break; + } + ticket = issued++; + } + Json result; + if (active(*frame)) + { + { + std::lock_guard lock(config_mutex_); + worker->configure(confidence_, iou_); + } + result = worker->infer(*frame, publish_candidates ? &unfiltered : nullptr); + } + std::unique_lock lock(mutex); + ready.wait(lock, + [&] + { + return failed || ticket == published; + }); + if (failed) + { + break; + } + if (!result.is_null() && active(*frame)) + { + consume(frame, std::move(result), unfiltered); + } + ++published; + ready.notify_all(); + } + } + catch (...) + { + fail(); + } + }); + } + } + catch (...) + { + fail(); + } + for (auto &thread : threads) + { + thread.join(); + } + if (error) + { + std::rethrow_exception(error); + } +} +} // namespace nvr diff --git a/src/cpp/native/engine/detector_pool.hpp b/src/cpp/native/engine/detector_pool.hpp new file mode 100644 index 0000000..9a31a86 --- /dev/null +++ b/src/cpp/native/engine/detector_pool.hpp @@ -0,0 +1,34 @@ +#pragma once + +#include "engine/detector.hpp" +#include "engine/frame_queue.hpp" + +#include +#include +#include + +namespace nvr +{ +// Each worker owns its RKNN context, DMA input, preprocessing and output storage. +// Results are published in dequeue order even when cores finish out of order. +class DetectorPool +{ + std::vector> detectors_; + std::mutex config_mutex_; + float confidence_ = 0.25f, iou_ = 0.45f; + +public: + Json description; + DetectorPool(const std::string &model, + float confidence, float iou, int workers = 0, const ModelOptions &options = {}); + void configure(float confidence, float iou) + { + std::lock_guard lock(config_mutex_); + confidence_ = confidence; + iou_ = iou; + } + bool supports_candidates() const { return detectors_.front()->supports_candidates(); } + void run(FrameQueue &queue, const std::function &active, bool publish_candidates, + const std::function &)> &consume); +}; +} // namespace nvr diff --git a/src/cpp/native/engine/dma_buffer.cpp b/src/cpp/native/engine/dma_buffer.cpp new file mode 100644 index 0000000..8190923 --- /dev/null +++ b/src/cpp/native/engine/dma_buffer.cpp @@ -0,0 +1,45 @@ +#include "engine/dma_buffer.hpp" +#include "engine/common.hpp" + +#include +#include +#include +#include +#include + +namespace nvr +{ +// Shared DMA buffer ownership. +DmaBuffer::DmaBuffer(size_t requested) : size((requested + 4095) & ~size_t(4095)) +{ + int heap = open("/dev/dma_heap/system-uncached", O_RDWR | O_CLOEXEC); + require(heap >= 0, "open system-uncached DMA heap (check render group permissions)"); + dma_heap_allocation_data request{}; + request.len = size; + request.fd_flags = O_RDWR | O_CLOEXEC; + int rc = ioctl(heap, DMA_HEAP_IOCTL_ALLOC, &request); + close(heap); + require(rc == 0, "allocate uncached DMA-BUF"); + fd = static_cast(request.fd); + data = mmap(nullptr, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (data == MAP_FAILED) + { + close(fd); + fd = -1; + throw std::runtime_error("map DMA-BUF"); + } +} + +DmaBuffer::~DmaBuffer() +{ + if (data && data != MAP_FAILED) + { + munmap(data, size); + } + if (fd >= 0) + { + close(fd); + } +} + +} // namespace nvr diff --git a/src/cpp/native/engine/dma_buffer.hpp b/src/cpp/native/engine/dma_buffer.hpp new file mode 100644 index 0000000..29b8026 --- /dev/null +++ b/src/cpp/native/engine/dma_buffer.hpp @@ -0,0 +1,20 @@ +#pragma once + +#include + +namespace nvr +{ +// Owns one uncached DMA-BUF allocation and its CPU mapping. +class DmaBuffer +{ +public: + int fd = -1; + void *data = nullptr; + size_t size = 0; + + explicit DmaBuffer(size_t bytes); + ~DmaBuffer(); + DmaBuffer(const DmaBuffer &) = delete; + DmaBuffer &operator=(const DmaBuffer &) = delete; +}; +} // namespace nvr diff --git a/src/cpp/native/engine/frame.cpp b/src/cpp/native/engine/frame.cpp new file mode 100644 index 0000000..28ec16f --- /dev/null +++ b/src/cpp/native/engine/frame.cpp @@ -0,0 +1,29 @@ +#include "engine/frame.hpp" + +namespace nvr +{ +Frame::Frame(MppFrame frame) : value(frame) +{ +} + +Frame::~Frame() +{ + if (value) + { + mpp_frame_deinit(&value); + } +} + +Nv12ImageView Frame::image() const +{ + require(value && mpp_frame_get_fmt(value) == MPP_FMT_YUV420SP, "expected linear NV12 from MPP"); + MppBuffer buffer = mpp_frame_get_buffer(value); + require(buffer, "missing MPP image buffer"); + return {int(mpp_frame_get_width(value)), + int(mpp_frame_get_height(value)), + int(mpp_frame_get_hor_stride(value)), + int(mpp_frame_get_ver_stride(value)), + mpp_buffer_get_fd(buffer), + mpp_buffer_get_size(buffer)}; +} +} // namespace nvr diff --git a/src/cpp/native/engine/frame.hpp b/src/cpp/native/engine/frame.hpp new file mode 100644 index 0000000..8402e3b --- /dev/null +++ b/src/cpp/native/engine/frame.hpp @@ -0,0 +1,35 @@ +#pragma once + +#include "engine/common.hpp" +#include "engine/image.hpp" + +#include +#include +#include + +extern "C" +{ +#include +} + +namespace nvr +{ +// Decoded frame ownership. +struct Frame +{ + MppFrame value = nullptr; + uint64_t sequence = 0; + int64_t pts_us = AV_NOPTS_VALUE; + uint64_t cycle = 0; + + explicit Frame(MppFrame frame); + ~Frame(); + + Nv12ImageView image() const; + + Frame(const Frame &) = delete; + Frame &operator=(const Frame &) = delete; +}; + +using FramePtr = std::shared_ptr; +} // namespace nvr diff --git a/src/cpp/native/engine/frame_queue.cpp b/src/cpp/native/engine/frame_queue.cpp new file mode 100644 index 0000000..e37a167 --- /dev/null +++ b/src/cpp/native/engine/frame_queue.cpp @@ -0,0 +1,55 @@ +#include "engine/frame_queue.hpp" + +namespace nvr +{ +FrameQueue::FrameQueue(size_t n) : capacity_(n) +{ + require(n > 0 && n <= 8, "queue_capacity must be 1..8"); +} + +void FrameQueue::push(FramePtr f, const std::atomic &stop) +{ + std::unique_lock lock(mutex_); + if (closed_ || stop) + { + return; + } + if (queue_.size() >= capacity_) + { + queue_.pop_front(); + ++dropped; + } + queue_.push_back(std::move(f)); + ready_.notify_one(); +} + +FramePtr FrameQueue::pop() +{ + std::unique_lock lock(mutex_); + ready_.wait(lock, + [&] + { + return closed_ || !queue_.empty(); + }); + if (queue_.empty()) + { + return {}; + } + auto result = std::move(queue_.front()); + queue_.pop_front(); + return result; +} + +void FrameQueue::close() +{ + std::lock_guard lock(mutex_); + closed_ = true; + ready_.notify_all(); +} + +void FrameQueue::clear() +{ + std::lock_guard lock(mutex_); + queue_.clear(); +} +} // namespace nvr diff --git a/src/cpp/native/engine/frame_queue.hpp b/src/cpp/native/engine/frame_queue.hpp new file mode 100644 index 0000000..ec49d60 --- /dev/null +++ b/src/cpp/native/engine/frame_queue.hpp @@ -0,0 +1,32 @@ +#pragma once + +#include "engine/frame.hpp" + +#include +#include +#include +#include + +namespace nvr +{ +// One producer and multiple consumers. A full queue drops the oldest frame. +// Closing wakes waiting consumers. +class FrameQueue +{ + std::mutex mutex_; + std::condition_variable ready_; + std::deque queue_; + size_t capacity_; + bool closed_ = false; + +public: + std::atomic dropped{0}; + + explicit FrameQueue(size_t capacity); + + void push(FramePtr frame, const std::atomic &stop); + FramePtr pop(); + void close(); + void clear(); +}; +} // namespace nvr diff --git a/src/cpp/native/engine/geometry.cpp b/src/cpp/native/engine/geometry.cpp new file mode 100644 index 0000000..2d13383 --- /dev/null +++ b/src/cpp/native/engine/geometry.cpp @@ -0,0 +1,25 @@ +#include "engine/geometry.hpp" +#include "engine/common.hpp" + +#include +#include + +namespace nvr +{ +Geometry letterbox_geometry(int source_width, int source_height, int width, int height) +{ + require(source_width > 0 && source_height > 0 && width >= 2 && height >= 2, + "invalid letterbox dimensions"); + Geometry g; + g.width = source_width; + g.height = source_height; + g.scale = std::min(float(width) / float(g.width), float(height) / float(g.height)); + // Even letterbox rectangles also satisfy NV12 chroma alignment requirements. + g.rw = std::max(2, int(std::round(float(g.width) * g.scale)) & ~1); + g.rh = std::max(2, int(std::round(float(g.height) * g.scale)) & ~1); + g.px = ((width - g.rw) / 2) & ~1; + g.py = ((height - g.rh) / 2) & ~1; + + return g; +} +} // namespace nvr diff --git a/src/cpp/native/engine/geometry.hpp b/src/cpp/native/engine/geometry.hpp new file mode 100644 index 0000000..140f86b --- /dev/null +++ b/src/cpp/native/engine/geometry.hpp @@ -0,0 +1,13 @@ +#pragma once + +namespace nvr +{ +struct Geometry +{ + int width = 0, height = 0, rw = 0, rh = 0, px = 0, py = 0; + float scale = 1; +}; + +// Even resize and padding coordinates suitable for NV12 chroma alignment. +Geometry letterbox_geometry(int source_width, int source_height, int width, int height); +} // namespace nvr diff --git a/src/cpp/native/engine/image.hpp b/src/cpp/native/engine/image.hpp new file mode 100644 index 0000000..f8b200e --- /dev/null +++ b/src/cpp/native/engine/image.hpp @@ -0,0 +1,16 @@ +#pragma once + +#include + +namespace nvr +{ +// Borrowed NV12 storage. The caller keeps the DMA-BUF alive through +// processing. Strides describe the allocated luma plane, including padding. +struct Nv12ImageView +{ + int width = 0, height = 0; + int horizontal_stride = 0, vertical_stride = 0; + int fd = -1; + size_t size = 0; +}; +} // namespace nvr diff --git a/src/cpp/native/engine/jpeg_preview.cpp b/src/cpp/native/engine/jpeg_preview.cpp new file mode 100644 index 0000000..cc12608 --- /dev/null +++ b/src/cpp/native/engine/jpeg_preview.cpp @@ -0,0 +1,118 @@ +#include "engine/jpeg_preview.hpp" + +#include +#include +#include + +namespace nvr +{ +void JpegPreview::initialize(int width, int height) +{ + width_ = std::min(640, width) & ~1; + height_ = std::max(2, (height * width_ / width) & ~1); + hs_ = (width_ + 15) & ~15; + vs_ = (height_ + 15) & ~15; + require(mpp_check_support_format(MPP_CTX_ENC, MPP_VIDEO_CodingMJPEG) == MPP_OK, + "MPP JPEG encoding is unavailable"); + image_ = std::make_unique(size_t(hs_) * size_t(vs_) * 3 / 2); + destination_ = importbuffer_fd(image_->fd, int(image_->size)); + require(destination_, "import preview destination"); + MppBufferInfo info{}; + info.type = MPP_BUFFER_TYPE_EXT_DMA; + info.fd = image_->fd; + info.size = image_->size; + require(mpp_buffer_import(&input_buffer_, &info) == MPP_OK, + "import preview DMA into MPP encoder"); + require(mpp_create(&context_, &api_) == MPP_OK, "create preview encoder"); + require(mpp_init(context_, MPP_CTX_ENC, MPP_VIDEO_CodingMJPEG) == MPP_OK, + "initialize MPP JPEG encoder"); + require(mpp_enc_cfg_init(&cfg_) == MPP_OK, "allocate JPEG encoder configuration"); + require(api_->control(context_, MPP_ENC_GET_CFG, cfg_) == MPP_OK, "get JPEG encoder config"); + mpp_enc_cfg_set_s32(cfg_, "prep:width", width_); + mpp_enc_cfg_set_s32(cfg_, "prep:height", height_); + mpp_enc_cfg_set_s32(cfg_, "prep:hor_stride", hs_); + mpp_enc_cfg_set_s32(cfg_, "prep:ver_stride", vs_); + mpp_enc_cfg_set_s32(cfg_, "prep:format", MPP_FMT_YUV420SP); + mpp_enc_cfg_set_s32(cfg_, "codec:type", MPP_VIDEO_CodingMJPEG); + mpp_enc_cfg_set_s32(cfg_, "jpeg:q_factor", 70); + mpp_enc_cfg_set_s32(cfg_, "jpeg:qf_min", 40); + mpp_enc_cfg_set_s32(cfg_, "jpeg:qf_max", 80); + require(api_->control(context_, MPP_ENC_SET_CFG, cfg_) == MPP_OK, "configure MPP JPEG encoder"); + RK_S64 timeout = 1000; + require(api_->control(context_, MPP_SET_OUTPUT_TIMEOUT, &timeout) == MPP_OK, + "set preview encoding timeout"); +} + +JpegPreview::~JpegPreview() +{ + if (context_) + { + mpp_destroy(context_); + } + if (cfg_) + { + mpp_enc_cfg_deinit(cfg_); + } + if (input_buffer_) + { + mpp_buffer_put(input_buffer_); + } + for (auto &entry : sources_) + { + releasebuffer_handle(entry.second); + } + if (destination_) + { + releasebuffer_handle(destination_); + } +} + +std::vector JpegPreview::encode(const Frame &frame) +{ + auto source = frame.value; + int w = int(mpp_frame_get_width(source)), h = int(mpp_frame_get_height(source)); + if (!context_) + { + initialize(w, h); + } + require(mpp_frame_get_fmt(source) == MPP_FMT_YUV420SP, "preview needs linear NV12"); + auto buffer = mpp_frame_get_buffer(source); + int fd = mpp_buffer_get_fd(buffer); + auto found = sources_.find(fd); + if (found == sources_.end()) + { + require(sources_.size() < 96, "preview import cache limit"); + auto handle = importbuffer_fd(fd, int(mpp_buffer_get_size(buffer))); + require(handle, "import preview source"); + found = sources_.emplace(fd, handle).first; + } + auto src = wrapbuffer_handle_t(found->second, w, h, int(mpp_frame_get_hor_stride(source)), + int(mpp_frame_get_ver_stride(source)), RK_FORMAT_YCbCr_420_SP); + auto dst = wrapbuffer_handle_t(destination_, width_, height_, hs_, vs_, RK_FORMAT_YCbCr_420_SP); + auto status = imresize(src, dst, 0, 0, INTER_LINEAR, 1); + require(status == IM_STATUS_SUCCESS, + std::string("RGA preview resize: ") + imStrError_t(status)); + MppFrame input = nullptr; + require(mpp_frame_init(&input) == MPP_OK, "allocate preview frame"); + mpp_frame_set_width(input, uint32_t(width_)); + mpp_frame_set_height(input, uint32_t(height_)); + mpp_frame_set_hor_stride(input, uint32_t(hs_)); + mpp_frame_set_ver_stride(input, uint32_t(vs_)); + mpp_frame_set_fmt(input, MPP_FMT_YUV420SP); + mpp_frame_set_buffer(input, input_buffer_); + mpp_frame_set_pts(input, frame.pts_us); + auto result = api_->encode_put_frame(context_, input); + mpp_frame_deinit(&input); + require(result == MPP_OK, "submit preview frame"); + MppPacket packet = nullptr; + result = api_->encode_get_packet(context_, &packet); + require(result == MPP_OK && packet, "MPP JPEG encode timeout"); + auto *bytes = static_cast(mpp_packet_get_pos(packet)); + size_t size = mpp_packet_get_length(packet); + std::vector encoded(bytes, bytes + size); + mpp_packet_deinit(&packet); + require(encoded.size() > 2 && encoded[0] == 0xff && encoded[1] == 0xd8, + "invalid hardware JPEG"); + return encoded; +} +} // namespace nvr diff --git a/src/cpp/native/engine/jpeg_preview.hpp b/src/cpp/native/engine/jpeg_preview.hpp new file mode 100644 index 0000000..5db92ef --- /dev/null +++ b/src/cpp/native/engine/jpeg_preview.hpp @@ -0,0 +1,36 @@ +#pragma once + +#include "engine/dma_buffer.hpp" +#include "engine/frame.hpp" + +#include +#include +#include +#include + +namespace nvr +{ +// Owned by one inference thread. Only compressed JPEG bytes leave the worker. +class JpegPreview +{ + MppCtx context_ = nullptr; + MppApi *api_ = nullptr; + MppEncCfg cfg_ = nullptr; + MppBuffer input_buffer_ = nullptr; + std::unique_ptr image_; + rga_buffer_handle_t destination_ = 0; + std::map sources_; + int width_ = 0, height_ = 0, hs_ = 0, vs_ = 0; + + void initialize(int width, int height); + +public: + JpegPreview() = default; + ~JpegPreview(); + + JpegPreview(const JpegPreview &) = delete; + JpegPreview &operator=(const JpegPreview &) = delete; + + std::vector encode(const Frame &frame); +}; +} // namespace nvr diff --git a/src/cpp/native/engine/json.hpp b/src/cpp/native/engine/json.hpp new file mode 100644 index 0000000..f2de637 --- /dev/null +++ b/src/cpp/native/engine/json.hpp @@ -0,0 +1,8 @@ +#pragma once + +#include + +namespace nvr +{ +using Json = nlohmann::json; +} // namespace nvr diff --git a/src/cpp/native/engine/media.hpp b/src/cpp/native/engine/media.hpp new file mode 100644 index 0000000..670625b --- /dev/null +++ b/src/cpp/native/engine/media.hpp @@ -0,0 +1,17 @@ +#pragma once + +#include +extern "C" +{ +#include +} + +namespace nvr +{ +inline std::string av_error(int code) +{ + char text[AV_ERROR_MAX_STRING_SIZE]{}; + av_strerror(code, text, sizeof(text)); + return text; +} +} // namespace nvr diff --git a/src/cpp/native/engine/native_output.cpp b/src/cpp/native/engine/native_output.cpp new file mode 100644 index 0000000..07d2a09 --- /dev/null +++ b/src/cpp/native/engine/native_output.cpp @@ -0,0 +1,130 @@ +#include "engine/native_output.hpp" +#include "engine/common.hpp" + +namespace nvr +{ +NativeOutput::NativeOutput(rknn_context context, const rknn_tensor_attr &logical) + : context_(context) +{ + if (logical.n_dims != 4 || logical.dims[0] != 1 || logical.fmt != RKNN_TENSOR_NCHW) + { + return; + } + + rknn_tensor_attr native{}; + native.index = logical.index; + if (rknn_query(context_, RKNN_QUERY_NATIVE_OUTPUT_ATTR, &native, sizeof(native)) != 0) + { + return; + } + size_t scalar_bytes = 0; + switch (native.type) + { + case RKNN_TENSOR_FLOAT16: + view_.type = TensorView::Type::Float16; + scalar_bytes = 2; + break; + case RKNN_TENSOR_FLOAT32: + view_.type = TensorView::Type::Float32; + scalar_bytes = 4; + break; + case RKNN_TENSOR_INT8: + view_.type = TensorView::Type::Int8; + scalar_bytes = 1; + break; + case RKNN_TENSOR_UINT8: + view_.type = TensorView::Type::Uint8; + scalar_bytes = 1; + break; + case RKNN_TENSOR_INT32: + view_.type = TensorView::Type::Int32; + scalar_bytes = 4; + break; + default: + return; + } + const size_t channels = logical.dims[1]; + view_.height = logical.dims[2]; + view_.width = logical.dims[3]; + view_.count = logical.n_elems; + if (!channels || !view_.height || !view_.width || native.dims[0] != 1) + { + return; + } + size_t native_width = 0; + if (native.fmt == RKNN_TENSOR_NC1HWC2 && native.n_dims == 5 && native.dims[2] == view_.height && + native.dims[1] * native.dims[4] >= channels) + { + view_.channel_block = native.dims[4]; + native_width = native.dims[3]; + } + else if (native.fmt == RKNN_TENSOR_NHWC && native.n_dims == 4 && + native.dims[1] == view_.height && native.dims[3] >= channels) + { + view_.channel_block = native.dims[3]; + native_width = native.dims[2]; + } + else if (native.fmt == RKNN_TENSOR_NCHW && native.n_dims == 4 && + native.dims[2] == view_.height && native.dims[1] >= channels) + { + view_.channel_block = 1; + native_width = native.dims[3]; + } + else + { + return; + } + view_.row_stride = native.w_stride ? native.w_stride : native_width; + if (!view_.channel_block || view_.row_stride < view_.width) + { + return; + } + const size_t last_channel = channels - 1; + const size_t last = + ((last_channel / view_.channel_block * view_.height + view_.height - 1) * view_.row_stride + + view_.width - 1) * + view_.channel_block + + last_channel % view_.channel_block; + if (native.size_with_stride / scalar_bytes <= last) + { + return; + } + if (native.qnt_type == RKNN_TENSOR_QNT_AFFINE_ASYMMETRIC) + { + if (!(native.scale > 0)) + { + return; + } + view_.scale = native.scale; + view_.zero_point = native.zp; + } + memory_ = rknn_create_mem(context_, native.size_with_stride); + require(memory_, "Allocate native output memory"); + if (rknn_set_io_mem(context_, memory_, &native) != 0) + { + rknn_destroy_mem(context_, memory_); + memory_ = nullptr; + } +} + +NativeOutput::~NativeOutput() +{ + if (memory_) + { + rknn_destroy_mem(context_, memory_); + } +} + +TensorView NativeOutput::view() const +{ + require(bound(), "Native output is not bound"); + auto result = view_; + result.data = memory_->virt_addr; + return result; +} + +void NativeOutput::sync() const +{ + require(bound() && rknn_mem_sync(context_, memory_, RKNN_MEMORY_SYNC_FROM_DEVICE) == 0, "Sync native output memory"); +} +} // namespace nvr diff --git a/src/cpp/native/engine/native_output.hpp b/src/cpp/native/engine/native_output.hpp new file mode 100644 index 0000000..a133ad0 --- /dev/null +++ b/src/cpp/native/engine/native_output.hpp @@ -0,0 +1,30 @@ +#pragma once + +#include "engine/tensor.hpp" +#include + +namespace nvr +{ +// Owns a bound native output buffer. Its RKNN context must outlive this object. +// Unsupported layouts leave it unbound so callers can use rknn_outputs_get instead. +class NativeOutput +{ + rknn_context context_; + rknn_tensor_mem *memory_ = nullptr; + TensorView view_; + +public: + NativeOutput(rknn_context context, const rknn_tensor_attr &logical); + ~NativeOutput(); + NativeOutput(const NativeOutput &) = delete; + NativeOutput &operator=(const NativeOutput &) = delete; + + bool bound() const + { + return memory_ != nullptr; + } + + TensorView view() const; + void sync() const; +}; +} // namespace nvr diff --git a/src/cpp/native/engine/postprocess.cpp b/src/cpp/native/engine/postprocess.cpp new file mode 100644 index 0000000..37df621 --- /dev/null +++ b/src/cpp/native/engine/postprocess.cpp @@ -0,0 +1,327 @@ +#include "engine/postprocess.hpp" +#include "engine/common.hpp" + +#include +#include +#include +#include + +namespace nvr +{ +namespace +{ +struct Box +{ + float x, y, w, h, score; + int cls; +}; + +// Scan contiguous proposal blocks while preserving class order and threshold ties. +template +void select_classes(int proposals, int classes, bool feature_major, float threshold, Score score_at, + Consume consume) +{ + constexpr int block_size = 128; + const int step = feature_major ? block_size : 1; + for (int base = 0; base < proposals; base += step) + { + const int count = std::min(step, proposals - base); + float best_scores[block_size]; + int best_classes[block_size]; + std::fill_n(best_scores, count, threshold); + std::fill_n(best_classes, count, -1); + for (int cls = 0; cls < classes; ++cls) + { + for (int j = 0; j < count; ++j) + { + const float score = score_at(base + j, cls); + if (std::isfinite(score) && score > best_scores[j]) + { + best_scores[j] = score; + best_classes[j] = cls; + } + } + } + for (int j = 0; j < count; ++j) + { + if (best_classes[j] >= 0) + { + consume(base + j, best_classes[j], best_scores[j]); + } + } + } +} + +// Affine INT8 DFL heads provide boxes, class scores, and confidence sums per grid. +void decode_dfl(const YoloLayout &layout, const std::vector> &quantized, + float threshold, std::vector &candidates) +{ + for (int branch = 0; branch < 3; ++branch) + { + const auto &grid = layout.grids[size_t(branch)]; + const auto &r = quantized[size_t(branch) * 3]; + const auto &s = quantized[size_t(branch) * 3 + 1]; + const auto &sums_data = quantized[size_t(branch) * 3 + 2]; + int gh = grid.height, gw = grid.width, cells = gh * gw; + int stride = layout.input_width / gw; + for (int p = 0; p < cells; ++p) + { + // The model-zoo sum head bounds the maximum class probability. Leave + // one quantization step of margin to avoid filtering borderline hits. + if (float(int(sums_data[size_t(p)]) - grid.sum_zero_point) * grid.sum_scale + + grid.sum_scale < + threshold) + { + continue; + } + int best = -128, cls = -1; + for (int c = 0; c < layout.classes; ++c) + { + int score = s[size_t(c) * cells + p]; + if (score > best) + { + best = score; + cls = c; + } + } + float probability = float(best - grid.score_zero_point) * grid.score_scale; + if (cls < 0 || probability < threshold) + { + continue; + } + float distances[4]{}; + for (int side = 0; side < 4; ++side) + { + float logits[16], max_logit = -1e30f; + for (int bin = 0; bin < 16; ++bin) + { + float x = + float(int(r[size_t(side * 16 + bin) * cells + p]) - grid.box_zero_point) * + grid.box_scale; + logits[bin] = x; + max_logit = std::max(max_logit, x); + } + float total = 0, weighted = 0; + for (int bin = 0; bin < 16; ++bin) + { + float weight = std::exp(logits[bin] - max_logit); + total += weight; + weighted += float(bin) * weight; + } + distances[side] = weighted / total * float(stride); + } + float cx = (float(p % gw) + 0.5f) * float(stride), + cy = (float(p / gw) + 0.5f) * float(stride); + candidates.push_back({cx - distances[0], cy - distances[1], distances[0] + distances[2], + distances[1] + distances[3], probability, cls}); + } + } +} + +// The split one-to-one head supplies probabilities and pixel-space xyxy boxes. +void decode_split(const YoloLayout &layout, const std::vector &output, float threshold, + std::vector &candidates) +{ + const auto *boxes = output.data() + size_t(layout.classes) * size_t(layout.proposals); + select_classes( + layout.proposals, layout.classes, layout.scores_first, threshold, + [&](int proposal, int cls) + { + return output[layout.scores_first ? size_t(cls) * layout.proposals + proposal + : size_t(proposal) * layout.classes + cls]; + }, + [&](int proposal, int cls, float score) + { + auto corner = [&](int coordinate) + { + return boxes[layout.boxes_first ? size_t(coordinate) * layout.proposals + proposal + : size_t(proposal) * 4 + coordinate]; + }; + float x = corner(0), y = corner(1), w = corner(2) - x, h = corner(3) - y; + if (std::isfinite(x) && std::isfinite(y) && std::isfinite(w) && std::isfinite(h) && + w > 0 && h > 0) + { + candidates.push_back({x, y, w, h, score, cls}); + } + }); +} + +// Decoded heads supply xywh boxes followed by class probabilities. +void decode_decoded(const YoloLayout &layout, const std::vector &output, float threshold, + std::vector &candidates) +{ + auto value = [&](int proposal, int feature) + { + return output[layout.features_first ? size_t(feature) * layout.proposals + proposal + : size_t(proposal) * (layout.classes + 4) + feature]; + }; + select_classes( + layout.proposals, layout.classes, layout.features_first, threshold, + [&](int proposal, int cls) + { + return value(proposal, cls + 4); + }, + [&](int proposal, int cls, float score) + { + float w = value(proposal, 2), h = value(proposal, 3); + float x = value(proposal, 0) - w / 2, y = value(proposal, 1) - h / 2; + if (std::isfinite(x) && std::isfinite(y) && std::isfinite(w) && std::isfinite(h) && + w > 0 && h > 0) + { + candidates.push_back({x, y, w, h, score, cls}); + } + }); +} +} // namespace + +YoloPostprocessor::YoloPostprocessor(const YoloLayout &layout, float confidence, float nms_iou) + : layout_(layout), threshold_(confidence), iou_(nms_iou) +{ + require(std::isfinite(confidence) && confidence > 0 && confidence < 1 && + std::isfinite(nms_iou) && nms_iou > 0 && nms_iou < 1, + "thresholds must be between 0 and 1"); + require(layout.classes > 0 && layout.classes <= 1020 && layout.proposals > 0, + "invalid YOLO class or proposal count"); + if (layout.kind == YoloLayout::Kind::Dfl) + { + require(layout.input_width > 0 && layout.input_height > 0, "invalid DFL input size"); + size_t proposals = 0; + for (const auto &grid : layout.grids) + { + require(grid.width > 0 && grid.height > 0 && layout.input_width % grid.width == 0 && + layout.input_height % grid.height == 0 && + layout.input_width / grid.width == layout.input_height / grid.height, + "invalid DFL grid stride"); + require(std::isfinite(grid.box_scale) && grid.box_scale > 0 && + std::isfinite(grid.score_scale) && grid.score_scale > 0 && + std::isfinite(grid.sum_scale) && grid.sum_scale > 0, + "invalid DFL quantization scales"); + proposals += size_t(grid.width) * size_t(grid.height); + } + require(proposals == size_t(layout.proposals), "DFL proposal count mismatch"); + } +} + +std::vector +YoloPostprocessor::decode(const Geometry &g, const std::vector &output, + const std::vector> &quantized, double *nms_ms, + std::vector *unfiltered) const +{ + require(g.width > 0 && g.height > 0 && g.rw > 0 && g.rh > 0, "invalid detection geometry"); + if (layout_.kind == YoloLayout::Kind::Dfl) + { + require(quantized.size() == 9, "DFL needs nine output buffers"); + for (size_t branch = 0; branch < layout_.grids.size(); ++branch) + { + const auto &grid = layout_.grids[branch]; + const size_t cells = size_t(grid.width) * size_t(grid.height); + require(quantized[branch * 3].size() == cells * 64 && + quantized[branch * 3 + 1].size() == cells * size_t(layout_.classes) && + quantized[branch * 3 + 2].size() == cells, + "DFL output buffer size mismatch"); + } + } + else + { + require(output.size() == size_t(layout_.classes + 4) * size_t(layout_.proposals), + "YOLO output buffer size mismatch"); + } + + std::vector candidates; + // Use the actual even resized dimensions to undo the RGA letterbox. + const float sx = float(g.width) / float(g.rw), sy = float(g.height) / float(g.rh); + auto source_detection = [&](const Box &box) -> std::optional + { + float x1 = std::clamp((box.x - float(g.px)) * sx, 0.f, float(g.width)); + float y1 = std::clamp((box.y - float(g.py)) * sy, 0.f, float(g.height)); + float x2 = std::clamp((box.x + box.w - float(g.px)) * sx, 0.f, float(g.width)); + float y2 = std::clamp((box.y + box.h - float(g.py)) * sy, 0.f, float(g.height)); + if (x2 <= x1 || y2 <= y1) return std::nullopt; + return Detection{box.cls, box.score, {x1, y1, x2, y2}}; + }; + switch (layout_.kind) + { + case YoloLayout::Kind::Dfl: + decode_dfl(layout_, quantized, threshold_, candidates); + break; + case YoloLayout::Kind::ScoresAndBoxes: + if (unfiltered) + { + candidates.reserve(layout_.proposals); + decode_split(layout_, output, -std::numeric_limits::infinity(), candidates); + unfiltered->clear(); + unfiltered->reserve(candidates.size()); + for (const auto &box : candidates) + { + if (auto detection = source_detection(box)) + unfiltered->push_back(*detection); + } + candidates.erase(std::remove_if(candidates.begin(), candidates.end(), + [&](const Box &box) { return box.score <= threshold_; }), + candidates.end()); + } + else + decode_split(layout_, output, threshold_, candidates); + break; + case YoloLayout::Kind::Decoded: + decode_decoded(layout_, output, threshold_, candidates); + break; + } + + // NMS policy is explicit: output count alone does not identify an NMS-free head. + auto suppression_started = Clock::now(); + std::sort(candidates.begin(), candidates.end(), + [](const Box &a, const Box &b) + { + return a.score > b.score; + }); + if (candidates.size() > 1000) + { + candidates.resize(1000); + } + std::vector kept; + for (const auto &a : candidates) + { + bool suppressed = false; + if (layout_.apply_nms) + { + for (const auto &b : kept) + { + if (a.cls != b.cls) + { + continue; + } + float iw = std::max(0.f, std::min(a.x + a.w, b.x + b.w) - std::max(a.x, b.x)); + float ih = std::max(0.f, std::min(a.y + a.h, b.y + b.h) - std::max(a.y, b.y)); + float inter = iw * ih, uni = a.w * a.h + b.w * b.h - inter; + if (uni > 0 && inter / uni > iou_) + { + suppressed = true; + break; + } + } + } + if (suppressed) + { + continue; + } + kept.push_back(a); + if (kept.size() >= 300) + { + break; + } + } + if (nms_ms) + { + *nms_ms = layout_.apply_nms ? milliseconds(suppression_started) : 0; + } + std::vector result; + result.reserve(kept.size()); + for (const auto &a : kept) + { + if (auto detection = source_detection(a)) + result.push_back(*detection); + } + return result; +} +} // namespace nvr diff --git a/src/cpp/native/engine/postprocess.hpp b/src/cpp/native/engine/postprocess.hpp new file mode 100644 index 0000000..9d29640 --- /dev/null +++ b/src/cpp/native/engine/postprocess.hpp @@ -0,0 +1,65 @@ +#pragma once + +#include "engine/geometry.hpp" + +#include +#include +#include + +namespace nvr +{ +struct Detection +{ + int class_id = -1; + float confidence = 0; + std::array box{}; // Source-image coordinates: x1, y1, x2, y2. +}; + +// Describes one affine INT8 DFL branch without depending on an inference SDK. +struct DflGrid +{ + int width = 0, height = 0; + int box_zero_point = 0, score_zero_point = 0, sum_zero_point = 0; + float box_scale = 0, score_scale = 0, sum_scale = 0; +}; + +struct YoloLayout +{ + enum class Kind + { + Decoded, + ScoresAndBoxes, + Dfl + }; + + Kind kind = Kind::Decoded; + int input_width = 0, input_height = 0; + int classes = 0, proposals = 0; + bool features_first = true, scores_first = true, boxes_first = true; + std::array grids{}; + bool apply_nms = true; +}; + +// Stateless per-frame decoding. Outputs are borrowed only during decode(). +// Float storage is decoded YOLO, or contiguous [scores][boxes] for split heads. +// DFL storage contains [boxes, scores, confidence sums] for each of three grids. +class YoloPostprocessor +{ + YoloLayout layout_; + float threshold_, iou_; + +public: + YoloPostprocessor(const YoloLayout &layout, float confidence, float nms_iou); + + void configure(float confidence, float iou) + { + threshold_ = confidence; + iou_ = iou; + } + + std::vector decode(const Geometry &geometry, const std::vector &output, + const std::vector> &quantized = {}, + double *nms_ms = nullptr, + std::vector *unfiltered = nullptr) const; +}; +} // namespace nvr diff --git a/src/cpp/native/engine/preprocess.cpp b/src/cpp/native/engine/preprocess.cpp new file mode 100644 index 0000000..ca56674 --- /dev/null +++ b/src/cpp/native/engine/preprocess.cpp @@ -0,0 +1,154 @@ +#include "engine/preprocess.hpp" +#include "engine/common.hpp" + +#include +#include +#include +#include + +namespace nvr +{ +// Model input resizing and color conversion. +Preprocessor::Preprocessor(DmaBuffer &input, int w, int h, int stride, + bool center_crop) + : width_(w), height_(h), stride_(stride), input_(input), center_crop_(center_crop) +{ + require(w > 0 && h > 0 && w % 16 == 0 && h % 2 == 0 && stride >= w && stride % 16 == 0, + "invalid RGB destination dimensions or stride"); + require(input.fd >= 0 && input.data && input.size >= size_t(stride) * size_t(h) * 3, + "invalid RGB destination DMA buffer"); + destination_ = importbuffer_fd(input_.fd, static_cast(input_.size)); + require(destination_ != 0, "RGA import RKNN input DMA-BUF"); +} + +Preprocessor::~Preprocessor() +{ + if (destination_) + { + releasebuffer_handle(destination_); + } + clear_sources(); + for (auto handle : scaled_handles_) + { + if (handle) + { + releasebuffer_handle(handle); + } + } +} + +void Preprocessor::clear_sources() +{ + for (auto &item : sources_) + { + releasebuffer_handle(item.second); + } + sources_.clear(); +} + +rga_buffer_t Preprocessor::resize_buffer(size_t slot, int width, int height) +{ + const int stride = (width + 15) & ~15; + const size_t bytes = size_t(stride) * size_t(height) * 3 / 2; + if (!scaled_[slot] || scaled_[slot]->size < bytes) + { + if (scaled_handles_[slot]) + { + releasebuffer_handle(scaled_handles_[slot]); + scaled_handles_[slot] = 0; + } + scaled_[slot] = std::make_unique(bytes); + } + if (!scaled_handles_[slot]) + { + scaled_handles_[slot] = importbuffer_fd(scaled_[slot]->fd, int(scaled_[slot]->size)); + require(scaled_handles_[slot] != 0, "RGA import resize buffer"); + } + return wrapbuffer_handle_t(scaled_handles_[slot], width, height, stride, height, + RK_FORMAT_YCbCr_420_SP); +} + +Geometry Preprocessor::process(const Nv12ImageView &image) +{ + const int hs = image.horizontal_stride, vs = image.vertical_stride; + require(image.width > 0 && image.height > 0 && hs >= image.width && vs >= image.height, + "invalid NV12 image strides"); + require(image.size >= size_t(hs) * size_t(vs) * 3 / 2, "invalid NV12 image buffer size"); + auto g = letterbox_geometry(image.width, image.height, width_, height_); + int crop_width = image.width, crop_height = image.height, crop_x = 0, crop_y = 0; + if (center_crop_) + { + float ratio = float(width_) / float(height_); + crop_width = std::min(image.width, int(float(image.height) * ratio)) & ~1; + crop_height = std::min(image.height, int(float(image.width) / ratio)) & ~1; + require(crop_width >= 2 && crop_height >= 2, "Invalid classification crop"); + crop_x = ((image.width - crop_width) / 2) & ~1; + crop_y = ((image.height - crop_height) / 2) & ~1; + g.rw = width_; + g.rh = height_; + g.px = 0; + g.py = 0; + g.scale = float(width_) / float(crop_width); + } + + int fd = image.fd; + require(fd >= 0, "RGA preprocessing needs a DMA-BUF file descriptor"); + auto found = sources_.find(fd); + if (found == sources_.end()) + { + require(sources_.size() < 96, + "source buffer cache limit exceeded (reconnect required)"); + auto handle = importbuffer_fd(fd, static_cast(image.size)); + // The import owns its DMA-BUF attachment. The caller retains the source + // during processing; no decoder-specific buffer reference is stored. + require(handle != 0, "RGA import NV12 DMA-BUF"); + found = sources_.emplace(fd, handle).first; + } + auto dst = + wrapbuffer_handle_t(destination_, width_, height_, stride_, height_, RK_FORMAT_RGB_888); + if (!painted_ || previous_.rw != g.rw || previous_.rh != g.rh || previous_.px != g.px || + previous_.py != g.py) + { + // RK3588 color fill needs RGA2 and below-4-GiB memory. Initialize the + // constant background once instead, so ordinary DMA heaps also work. + // Every image is still resized/converted by RGA directly into this buffer. + dma_buf_sync sync{}; + sync.flags = DMA_BUF_SYNC_START | DMA_BUF_SYNC_WRITE; + require(ioctl(input_.fd, DMA_BUF_IOCTL_SYNC, &sync) == 0, + "sync model input for letterbox initialization"); + std::memset(input_.data, 114, input_.size); + sync.flags = DMA_BUF_SYNC_END | DMA_BUF_SYNC_WRITE; + require(ioctl(input_.fd, DMA_BUF_IOCTL_SYNC, &sync) == 0, + "finish letterbox initialization"); + painted_ = true; + previous_ = g; + } + auto src = + wrapbuffer_handle_t(found->second, g.width, g.height, hs, vs, RK_FORMAT_YCbCr_420_SP); + im_rect source_rect{crop_x, crop_y, crop_width, crop_height}; + size_t slot = 0; + // Large reductions can exceed an available core's scale range. Keep each + // pass within 8x and let librga select a compatible engine and DMA mapping. + // Two reusable NV12 buffers also allow reductions requiring several passes. + while (source_rect.width > g.rw * 8 || source_rect.height > g.rh * 8) + { + const int next_width = std::max(g.rw, ((source_rect.width + 15) / 16) * 2); + const int next_height = std::max(g.rh, ((source_rect.height + 15) / 16) * 2); + auto intermediate = resize_buffer(slot, next_width, next_height); + const im_rect intermediate_rect{0, 0, next_width, next_height}; + auto status = improcess(src, intermediate, {}, source_rect, intermediate_rect, {}, -1, + nullptr, nullptr, IM_SYNC); + require(status == IM_STATUS_SUCCESS, + std::string("RGA staged resize: ") + imStrError_t(status)); + src = intermediate; + source_rect = intermediate_rect; + slot ^= 1; + } + // The final job converts NV12 to RGB directly in RKNN-bound DMA memory. + auto status = improcess(src, dst, {}, source_rect, im_rect{g.px, g.py, g.rw, g.rh}, {}, -1, + nullptr, nullptr, IM_SYNC); + require(status == IM_STATUS_SUCCESS, + std::string("RGA preprocess: ") + imStrError_t(status)); + return g; +} +} // namespace nvr diff --git a/src/cpp/native/engine/preprocess.hpp b/src/cpp/native/engine/preprocess.hpp new file mode 100644 index 0000000..91f6938 --- /dev/null +++ b/src/cpp/native/engine/preprocess.hpp @@ -0,0 +1,40 @@ +#pragma once + +#include "engine/dma_buffer.hpp" +#include "engine/image.hpp" +#include "engine/geometry.hpp" + +#include +#include +#include +#include + +namespace nvr +{ +// One instance per processing thread. The destination must outlive this object. +// Clear source imports before replacing a source buffer pool whose FDs may be reused. +class Preprocessor +{ + std::map sources_; + rga_buffer_handle_t destination_ = 0; + std::array, 2> scaled_; + std::array scaled_handles_{}; + int width_, height_, stride_; + DmaBuffer &input_; + bool painted_ = false, center_crop_ = false; + Geometry previous_; + + rga_buffer_t resize_buffer(size_t slot, int width, int height); + +public: + Preprocessor(DmaBuffer &input, int width, int height, int stride, + bool center_crop = false); + ~Preprocessor(); + + Preprocessor(const Preprocessor &) = delete; + Preprocessor &operator=(const Preprocessor &) = delete; + + Geometry process(const Nv12ImageView &image); + void clear_sources(); +}; +} // namespace nvr diff --git a/src/cpp/native/engine/task_decode.cpp b/src/cpp/native/engine/task_decode.cpp new file mode 100644 index 0000000..15c7192 --- /dev/null +++ b/src/cpp/native/engine/task_decode.cpp @@ -0,0 +1,507 @@ +#include "engine/task_model.hpp" +#include "engine/common.hpp" + +#include +#include +#include +#include + +namespace nvr +{ +namespace +{ +struct Candidate +{ + int class_id = -1; + float score = 0, x = 0, y = 0, width = 0, height = 0, angle = 0; + std::vector extra; +}; + +float overlap(const Candidate &a, const Candidate &b, bool rotated) +{ + if (rotated) + { + // Probabilistic IoU uses the covariance of each rotated rectangle. + auto covariance = [](const Candidate &box) + { + float w = box.width * box.width / 12, h = box.height * box.height / 12; + float c = std::cos(box.angle), s = std::sin(box.angle); + return std::array{w * c * c + h * s * s, w * s * s + h * c * c, + (w - h) * c * s}; + }; + auto x = covariance(a), y = covariance(b); + float ax = x[0] + y[0], ay = x[1] + y[1], ac = x[2] + y[2]; + float dx = a.x - b.x, dy = a.y - b.y, denominator = ax * ay - ac * ac + 1e-7f; + float distance = (ax * dy * dy + ay * dx * dx - 2 * ac * dx * dy) / (4 * denominator); + float determinant = + std::max(0.f, x[0] * x[1] - x[2] * x[2]) * std::max(0.f, y[0] * y[1] - y[2] * y[2]); + distance += .5f * std::log(denominator / (4 * std::sqrt(determinant) + 1e-7f) + 1e-7f); + return 1 - std::sqrt(std::max(0.f, 1 - std::exp(-std::clamp(distance, 1e-7f, 100.f)))); + } + float intersection = std::max(0.f, std::min(a.x + a.width / 2, b.x + b.width / 2) - + std::max(a.x - a.width / 2, b.x - b.width / 2)) * + std::max(0.f, std::min(a.y + a.height / 2, b.y + b.height / 2) - + std::max(a.y - a.height / 2, b.y - b.height / 2)); + float total = a.width * a.height + b.width * b.height - intersection; + return total > 0 ? intersection / total : 0; +} + +float matrix_value(const TensorSpec &spec, const std::vector &data, int channels, + int proposals, int feature, int proposal) +{ + const auto &d = spec.dimensions; + bool major = d.size() == 4 ? spec.channels_first : d[1] == channels; + return data[major ? size_t(feature) * proposals + proposal + : size_t(proposal) * channels + feature]; +} +} // namespace + +TaskResult TaskPostprocessor::decode(const Geometry &g, + const std::vector> &outputs, + float confidence, float nms_iou) const +{ + require(outputs.size() == specs_.size(), "Model output count changed"); + for (size_t i = 0; i < outputs.size(); ++i) + { + require(outputs[i].size() == specs_[i].elements(), "Model output size mismatch"); + } + require(g.width > 0 && g.height > 0 && g.rw > 0 && g.rh > 0, "Invalid result geometry"); + require(confidence > 0 && confidence < 1 && nms_iou > 0 && nms_iou < 1, + "Thresholds must be between zero and one"); + const float source_width = float(g.width), source_height = float(g.height); + const float resize_width = float(g.rw), resize_height = float(g.rh); + const float padding_x = float(g.px), padding_y = float(g.py); + const float model_width = float(input_width_), model_height = float(input_height_); + TaskResult result; + auto source_point = [&](float x, float y) + { + return std::array{(x - padding_x) * source_width / resize_width, + (y - padding_y) * source_height / resize_height}; + }; + if (layout_ == Layout::Classification) + { + auto probabilities = outputs[0]; + require(std::all_of(probabilities.begin(), probabilities.end(), + [](float x) + { + return std::isfinite(x); + }), + "Non-finite classification output"); + if (std::any_of(probabilities.begin(), probabilities.end(), + [](float x) + { + return x < 0 || x > 1; + })) + { + float maximum = *std::max_element(probabilities.begin(), probabilities.end()), sum = 0; + for (float &value : probabilities) + { + value = std::exp(value - maximum); + sum += value; + } + for (float &value : probabilities) + { + value /= sum; + } + } + std::vector order(probabilities.size()); + std::iota(order.begin(), order.end(), 0); + size_t count = std::min(size_t(5), order.size()); + std::partial_sort(order.begin(), order.begin() + count, order.end(), + [&](int a, int b) + { + return probabilities[size_t(a)] > probabilities[size_t(b)]; + }); + for (size_t i = 0; i < count; ++i) + { + result.classification.push_back({order[i], probabilities[size_t(order[i])]}); + } + return result; + } + if (layout_ == Layout::Dense) + { + return decode_dense(g, {outputs[0].data(), outputs[0].size()}); + } + + // Generate candidates in model coordinates; task attachments retain proposal identity. + const int extra_count = options_.task == ModelTask::Segment ? mask_channels_ + : options_.task == ModelTask::Pose + ? options_.keypoints * options_.keypoint_dimensions + : options_.task == ModelTask::Obb ? 1 + : 0; + std::vector candidates; + auto accept = [&](Candidate candidate, bool xyxy) + { + if (!std::isfinite(candidate.score) || !(candidate.score >= confidence) || + candidate.class_id < 0 || candidate.class_id >= options_.classes) + { + return; + } + if (options_.normalized && layout_ != Layout::Dfl) + { + candidate.x *= model_width; + candidate.width *= model_width; + candidate.y *= model_height; + candidate.height *= model_height; + } + if (xyxy) + { + candidate.width -= candidate.x; + candidate.height -= candidate.y; + candidate.x += candidate.width / 2; + candidate.y += candidate.height / 2; + } + if (!std::isfinite(candidate.x) || !std::isfinite(candidate.y) || + !std::isfinite(candidate.width) || !std::isfinite(candidate.height) || + candidate.width <= 0 || candidate.height <= 0) + { + return; + } + if (options_.task == ModelTask::Obb) + { + candidate.angle = candidate.extra[0]; + if (!std::isfinite(candidate.angle)) + { + return; + } + } + candidates.push_back(std::move(candidate)); + }; + if (layout_ == Layout::Decoded || layout_ == Layout::EndToEnd) + { + const auto &data = outputs[size_t(prediction_)]; + auto value = [&](int p, int f) + { + return data[feature_major_ ? size_t(f) * proposals_ + p : size_t(p) * features_ + f]; + }; + for (int p = 0; p < proposals_; ++p) + { + Candidate c; + if (layout_ == Layout::EndToEnd) + { + float label = value(p, 5); + if (!std::isfinite(label) || label != std::round(label) || label < 0 || + label >= float(options_.classes)) + { + continue; + } + c.class_id = int(label); + c.score = value(p, 4); + } + else + { + for (int cls = 0; cls < options_.classes; ++cls) + { + float score = value(p, 4 + cls); + if (std::isfinite(score) && score > c.score) + { + c.score = score; + c.class_id = cls; + } + } + } + if (c.score < confidence) + { + continue; + } + c.x = value(p, 0); + c.y = value(p, 1); + c.width = value(p, 2); + c.height = value(p, 3); + int base = layout_ == Layout::EndToEnd ? 6 : 4 + options_.classes; + for (int j = 0; j < extra_count; ++j) + { + c.extra.push_back(value(p, base + j)); + } + accept(std::move(c), layout_ == Layout::EndToEnd && options_.task != ModelTask::Obb); + } + } + else if (layout_ == Layout::Split) + { + for (int p = 0; p < proposals_; ++p) + { + Candidate c; + for (int cls = 0; cls < options_.classes; ++cls) + { + float score = matrix_value(specs_[size_t(scores_)], outputs[size_t(scores_)], + options_.classes, proposals_, cls, p); + if (std::isfinite(score) && score > c.score) + { + c.score = score; + c.class_id = cls; + } + } + if (c.score < confidence) + { + continue; + } + auto value = [&](int f) + { + return matrix_value(specs_[size_t(boxes_)], outputs[size_t(boxes_)], 4, proposals_, + f, p); + }; + c.x = value(0); + c.y = value(1); + c.width = value(2); + c.height = value(3); + for (int j = 0; j < extra_count; ++j) + { + c.extra.push_back(matrix_value(specs_[size_t(auxiliary_)], + outputs[size_t(auxiliary_)], extra_count, proposals_, + j, p)); + } + accept(std::move(c), options_.task != ModelTask::Obb); + } + } + else + { + int offset = 0; + for (size_t branch = 0; branch < regressions_.size(); ++branch) + { + int reg = regressions_[branch], cls = class_scores_[branch]; + const auto &d = specs_[size_t(reg)].dimensions; + int width = d[3], height = d[2], cells = width * height; + // Rockchip pose and OBB pack raw class logits after the DFL bins. + const bool packed_logits = reg == cls; + const float score_floor = + packed_logits ? std::log(confidence / (1 - confidence)) : confidence; + int channels = d[1] - (packed_logits ? options_.classes : 0), bins = channels / 4, + stride = input_width_ / width; + for (int p = 0; p < cells; ++p) + { + Candidate c; + c.score = -std::numeric_limits::infinity(); + for (int category = 0; category < options_.classes; ++category) + { + float score = + outputs[size_t(cls)] + [size_t(category + (packed_logits ? channels : 0)) * cells + p]; + if (std::isfinite(score) && score > c.score) + { + c.score = score; + c.class_id = category; + } + } + if (c.score < score_floor) + { + continue; + } + if (packed_logits) + { + c.score = 1 / (1 + std::exp(-c.score)); + } + float distances[4]{}; + for (int side = 0; side < 4; ++side) + { + if (bins == 1) + { + distances[side] = + outputs[size_t(reg)][size_t(side) * cells + p] * float(stride); + } + else + { + float maximum = -std::numeric_limits::infinity(), sum = 0, + weighted = 0; + for (int bin = 0; bin < bins; ++bin) + { + maximum = std::max( + maximum, + outputs[size_t(reg)][size_t(side * bins + bin) * cells + p]); + } + for (int bin = 0; bin < bins; ++bin) + { + float weight = std::exp( + outputs[size_t(reg)][size_t(side * bins + bin) * cells + p] - + maximum); + sum += weight; + weighted += weight * float(bin); + } + distances[side] = weighted / sum * float(stride); + } + } + for (int j = 0; j < extra_count; ++j) + { + int index = extras_.empty() ? auxiliary_ : extras_[branch]; + c.extra.push_back(matrix_value( + specs_[size_t(index)], outputs[size_t(index)], extra_count, + extras_.empty() ? proposals_ : cells, j, extras_.empty() ? offset + p : p)); + } + float x = (distances[2] - distances[0]) / 2, y = (distances[3] - distances[1]) / 2; + if (options_.task == ModelTask::Obb) + { + // Model-zoo OBB exposes the angle sigmoid before its affine transform. + c.extra[0] = (c.extra[0] - .25f) * 3.1415927410125732f; + float cosine = std::cos(c.extra[0]), sine = std::sin(c.extra[0]); + float rotated_x = x * cosine - y * sine; + y = x * sine + y * cosine; + x = rotated_x; + } + c.x = (float(p % width) + .5f) * float(stride) + x; + c.y = (float(p / width) + .5f) * float(stride) + y; + c.width = distances[0] + distances[2]; + c.height = distances[1] + distances[3]; + // Raw spatial heads are already in pixels, independent of decoded INT8 + // normalization. + accept(std::move(c), false); + } + offset += cells; + } + } + std::stable_sort(candidates.begin(), candidates.end(), + [](const Candidate &a, const Candidate &b) + { + return a.score > b.score; + }); + if (candidates.size() > 1000) + { + candidates.resize(1000); + } + std::vector kept; + const bool nms = options_.apply_nms && !options_.end_to_end && layout_ != Layout::EndToEnd; + for (size_t index = 0; index < candidates.size(); ++index) + { + const auto &candidate = candidates[index]; + bool suppressed = false; + if (nms && options_.task == ModelTask::Obb) + { + // Ultralytics rotated Fast-NMS compares every higher-scoring proposal, + // including proposals that were themselves suppressed. + for (size_t previous = 0; previous < index; ++previous) + { + if (candidate.class_id == candidates[previous].class_id && + overlap(candidate, candidates[previous], true) > nms_iou) + { + suppressed = true; + break; + } + } + } + else if (nms) + { + for (const auto &previous : kept) + { + if (candidate.class_id == previous.class_id && + overlap(candidate, previous, false) > nms_iou) + { + suppressed = true; + break; + } + } + } + if (!suppressed) + { + kept.push_back(candidate); + } + if (kept.size() == 300) + { + break; + } + } + for (const auto &candidate : kept) + { + TaskDetection item; + item.detection.class_id = candidate.class_id; + item.detection.confidence = candidate.score; + auto a = + source_point(candidate.x - candidate.width / 2, candidate.y - candidate.height / 2); + auto b = + source_point(candidate.x + candidate.width / 2, candidate.y + candidate.height / 2); + if (options_.task == ModelTask::Obb) + { + float cosine = std::cos(candidate.angle), sine = std::sin(candidate.angle); + a = {std::numeric_limits::infinity(), std::numeric_limits::infinity()}; + b = {-a[0], -a[1]}; + for (auto corner : + std::array, 4>{{{-1, -1}, {1, -1}, {1, 1}, {-1, 1}}}) + { + float x = corner[0] * candidate.width / 2, y = corner[1] * candidate.height / 2; + auto point = source_point(candidate.x + x * cosine - y * sine, + candidate.y + x * sine + y * cosine); + item.polygon.push_back(point); + a[0] = std::min(a[0], point[0]); + a[1] = std::min(a[1], point[1]); + b[0] = std::max(b[0], point[0]); + b[1] = std::max(b[1], point[1]); + } + } + item.detection.box = {std::clamp(a[0], 0.f, float(source_width)), + std::clamp(a[1], 0.f, float(source_height)), + std::clamp(b[0], 0.f, float(source_width)), + std::clamp(b[1], 0.f, float(source_height))}; + if (item.detection.box[2] <= item.detection.box[0] || + item.detection.box[3] <= item.detection.box[1]) + { + continue; + } + if (options_.task == ModelTask::Pose) + { + for (int k = 0; k < options_.keypoints; ++k) + { + size_t index = size_t(k * options_.keypoint_dimensions); + float x = candidate.extra[index], y = candidate.extra[index + 1]; + if (options_.normalized && layout_ != Layout::Dfl) + { + x *= model_width; + y *= model_height; + } + float visibility = + options_.keypoint_dimensions == 3 ? candidate.extra[index + 2] : 1; + auto point = source_point(x, y); + if (!std::isfinite(point[0]) || !std::isfinite(point[1]) || + !std::isfinite(visibility)) + { + visibility = 0; + } + item.keypoints.push_back({std::isfinite(point[0]) ? point[0] : 0, + std::isfinite(point[1]) ? point[1] : 0, + std::clamp(visibility, 0.f, 1.f)}); + } + } + if (options_.task == ModelTask::Segment) + { + const auto &spec = specs_[size_t(prototypes_)]; + const auto &d = spec.dimensions; + int height = d[spec.channels_first ? 2 : 1], width = d[spec.channels_first ? 3 : 2]; + ResultRaster mask; + mask.box = item.detection.box; + int limit = + std::max(1, int(std::sqrt(32768.0 / double(std::max(size_t(1), kept.size()))))); + limit = std::min(limit, 160); + mask.width = std::max( + 1, std::min(limit, int(std::ceil((mask.box[2] - mask.box[0]) * resize_width / + source_width * float(width) / model_width)))); + mask.height = std::max( + 1, std::min(limit, int(std::ceil((mask.box[3] - mask.box[1]) * resize_height / + source_height * float(height) / model_height)))); + for (int y = 0; y < mask.height; ++y) + { + for (int x = 0; x < mask.width; ++x) + { + float sx = mask.box[0] + + (float(x) + .5f) * (mask.box[2] - mask.box[0]) / float(mask.width); + float sy = mask.box[1] + + (float(y) + .5f) * (mask.box[3] - mask.box[1]) / float(mask.height); + int px = std::clamp(int((sx * resize_width / source_width + padding_x) * + float(width) / model_width), + 0, width - 1); + int py = std::clamp(int((sy * resize_height / source_height + padding_y) * + float(height) / model_height), + 0, height - 1); + float logit = 0; + for (int c = 0; c < mask_channels_; ++c) + { + size_t index = spec.channels_first + ? size_t(c) * width * height + size_t(py) * width + px + : (size_t(py) * width + px) * mask_channels_ + c; + logit += candidate.extra[size_t(c)] * outputs[size_t(prototypes_)][index]; + } + append_run(mask, std::isfinite(logit) && logit > 0 ? 1 : 0); + } + } + item.mask = std::move(mask); + } + result.detections.push_back(std::move(item)); + } + return result; +} +} // namespace nvr diff --git a/src/cpp/native/engine/task_dense.cpp b/src/cpp/native/engine/task_dense.cpp new file mode 100644 index 0000000..4f125e4 --- /dev/null +++ b/src/cpp/native/engine/task_dense.cpp @@ -0,0 +1,174 @@ +#include "engine/task_model.hpp" +#include "engine/common.hpp" + +#include +#include +#include + +namespace nvr +{ +void TaskPostprocessor::append_run(ResultRaster &raster, uint32_t value) +{ + if (!raster.runs.empty() && raster.runs[raster.runs.size() - 2] == value) + { + ++raster.runs.back(); + } + else + { + raster.runs.push_back(value); + raster.runs.push_back(1); + } +} + +TaskResult TaskPostprocessor::decode_dense(const Geometry &g, const TensorView &output) const +{ + require(layout_ == Layout::Dense, "Model does not produce dense predictions"); + require(output.data && output.count == specs_[0].elements(), "Dense output size mismatch"); + require(g.width > 0 && g.height > 0 && g.rw > 0 && g.rh > 0, "Invalid result geometry"); + const float source_width = float(g.width), source_height = float(g.height); + const float resize_width = float(g.rw), resize_height = float(g.rh); + const float padding_x = float(g.px), padding_y = float(g.py); + const float model_width = float(input_width_), model_height = float(input_height_); + TaskResult result; + const auto &spec = specs_[0]; + const auto &d = spec.dimensions; + int channels = d.size() == 4 ? d[spec.channels_first ? 1 : 3] : 1; + int height = d[d.size() == 3 ? 1 : spec.channels_first ? 2 : 1]; + int width = d[d.size() == 3 ? 2 : spec.channels_first ? 3 : 2]; + auto value_at = [&](int channel, int x, int y) + { + if (output.channel_block) + { + return output.spatial(size_t(channel), size_t(x), size_t(y)); + } + return output[spec.channels_first || d.size() == 3 + ? size_t(channel) * width * height + size_t(y) * width + x + : (size_t(y) * width + x) * channels + channel]; + }; + ResultRaster raster; + raster.width = std::min(128, g.width); + raster.height = + std::max(1, int(std::round(float(raster.width) * source_height / source_width))); + if (raster.height > 128) + { + raster.height = 128; + raster.width = + std::max(1, int(std::round(float(raster.height) * source_width / source_height))); + } + raster.box = {0, 0, float(source_width), float(source_height)}; + std::vector samples; + for (int y = 0; y < raster.height; ++y) + { + for (int x = 0; x < raster.width; ++x) + { + int px = + std::clamp(int(((float(x) + .5f) * resize_width / float(raster.width) + padding_x) * + float(width) / model_width), + 0, width - 1); + int py = std::clamp( + int(((float(y) + .5f) * resize_height / float(raster.height) + padding_y) * + float(height) / model_height), + 0, height - 1); + int x0 = px, y0 = py, x1 = px, y1 = py; + float dx = 0, dy = 0; + if (options_.dense_resize != DenseResize::None) + { + // Sample the same model pixels as a full-resolution output, then + // reconstruct only their logits/depth with the export's resize convention. + const int mx = std::clamp( + int((float(x) + .5f) * resize_width / float(raster.width) + padding_x), 0, + input_width_ - 1); + const int my = std::clamp( + int((float(y) + .5f) * resize_height / float(raster.height) + padding_y), 0, + input_height_ - 1); + const bool corners = options_.dense_resize == DenseResize::AlignCorners; + float fx = corners ? float(mx) * float(width - 1) / (model_width - 1) + : (float(mx) + .5f) * float(width) / model_width - .5f; + float fy = corners ? float(my) * float(height - 1) / (model_height - 1) + : (float(my) + .5f) * float(height) / model_height - .5f; + fx = std::clamp(fx, 0.f, float(width - 1)); + fy = std::clamp(fy, 0.f, float(height - 1)); + x0 = int(fx); + y0 = int(fy); + x1 = std::min(x0 + 1, width - 1); + y1 = std::min(y0 + 1, height - 1); + dx = fx - float(x0); + dy = fy - float(y0); + } + auto sample = [&](int channel) + { + if (options_.dense_resize == DenseResize::None) + { + return value_at(channel, px, py); + } + const float a = value_at(channel, x0, y0), b = value_at(channel, x1, y0); + const float c = value_at(channel, x0, y1), d = value_at(channel, x1, y1); + const float top = a + (b - a) * dx, bottom = c + (d - c) * dx; + return top + (bottom - top) * dy; + }; + if (options_.task == ModelTask::Depth) + { + samples.push_back(sample(0)); + } + else + { + int label = 0; + if (d.size() == 3) + { + float value = value_at(0, px, py); + require(std::isfinite(value) && value >= 0 && value < float(options_.classes) && + std::abs(value - std::round(value)) < .01f, + "Semantic output contains an invalid class ID"); + label = int(std::round(value)); + } + else if (channels == 1) + { + label = 0; + } + else + { + float best = -std::numeric_limits::infinity(); + for (int c = 0; c < channels; ++c) + { + float value = sample(c); + if (std::isfinite(value) && value > best) + { + best = value; + label = c; + } + } + require(std::isfinite(best), "Semantic output has no finite class score"); + } + append_run(raster, uint32_t(label)); + } + } + } + if (options_.task == ModelTask::Depth) + { + float minimum = std::numeric_limits::infinity(), maximum = 0; + for (float value : samples) + { + if (std::isfinite(value) && value > 0) + { + minimum = std::min(minimum, value); + maximum = std::max(maximum, value); + } + } + require(std::isfinite(minimum), "Depth output has no valid positive samples"); + raster.offset = minimum; + raster.scale = maximum > minimum ? (maximum - minimum) / 65535 : 0; + for (float value : samples) + { + append_run( + raster, + std::isfinite(value) && value > 0 + ? uint32_t(std::clamp( + std::round(raster.scale > 0 ? (value - minimum) / raster.scale : 0), 0.f, + 65535.f)) + : 65536); + } + } + result.dense = std::move(raster); + return result; +} +} // namespace nvr diff --git a/src/cpp/native/engine/task_json.cpp b/src/cpp/native/engine/task_json.cpp new file mode 100644 index 0000000..f492c1b --- /dev/null +++ b/src/cpp/native/engine/task_json.cpp @@ -0,0 +1,75 @@ +#include "engine/task_json.hpp" +#include "engine/common.hpp" + +namespace nvr +{ +ModelOptions model_options(const Json &info) +{ + ModelOptions options; + options.task = model_task(info.value("task", std::string("detect"))); + options.classes = info.value("classes", info.contains("names") ? int(info["names"].size()) : 0); + if (info.contains("kpt_shape")) + { + const auto &shape = info.at("kpt_shape"); + require(shape.is_array() && shape.size() == 2, "kpt_shape must be [count,dimensions]"); + options.keypoints = shape[0].get(); + options.keypoint_dimensions = shape[1].get(); + } + options.end_to_end = info.value("end2end", false); + options.apply_nms = info.value("nms", true); + options.normalized = info.value("coordinates", std::string("pixels")) == "normalized"; + options.center_crop = info.value("preprocess", options.task == ModelTask::Classify + ? std::string("center_crop") + : std::string("letterbox")) == "center_crop"; + const std::string resize = info.value("dense_resize", std::string("none")); + require(resize == "none" || resize == "half_pixel" || resize == "align_corners", + "Unsupported dense interpolation policy"); + options.dense_resize = resize == "half_pixel" ? DenseResize::HalfPixel + : resize == "align_corners" ? DenseResize::AlignCorners + : DenseResize::None; + return options; +} + +Json task_result_json(const TaskResult &result, ModelTask task) +{ + auto raster_json = [](const ResultRaster &raster) + { + return Json{{"width", raster.width}, {"height", raster.height}, {"box", raster.box}, + {"runs", raster.runs}, {"offset", raster.offset}, {"scale", raster.scale}}; + }; + Json output = {{"task", task_name(task)}, {"detections", Json::array()}}; + for (const auto &item : result.detections) + { + Json detection = {{"class_id", item.detection.class_id}, + {"confidence", item.detection.confidence}, + {"box", item.detection.box}}; + if (!item.keypoints.empty()) + { + detection["keypoints"] = item.keypoints; + } + if (!item.polygon.empty()) + { + detection["polygon"] = item.polygon; + } + if (item.mask) + { + detection["mask"] = raster_json(*item.mask); + } + output["detections"].push_back(std::move(detection)); + } + if (task == ModelTask::Classify) + { + output["classification"] = Json::array(); + for (const auto &prediction : result.classification) + { + output["classification"].push_back( + {{"class_id", prediction.class_id}, {"confidence", prediction.confidence}}); + } + } + if (result.dense) + { + output["dense"] = raster_json(*result.dense); + } + return output; +} +} // namespace nvr diff --git a/src/cpp/native/engine/task_json.hpp b/src/cpp/native/engine/task_json.hpp new file mode 100644 index 0000000..767165d --- /dev/null +++ b/src/cpp/native/engine/task_json.hpp @@ -0,0 +1,10 @@ +#pragma once + +#include "engine/task_model.hpp" +#include "engine/json.hpp" + +namespace nvr +{ +ModelOptions model_options(const Json &info); +Json task_result_json(const TaskResult &result, ModelTask task); +} // namespace nvr diff --git a/src/cpp/native/engine/task_model.cpp b/src/cpp/native/engine/task_model.cpp new file mode 100644 index 0000000..fe9bd09 --- /dev/null +++ b/src/cpp/native/engine/task_model.cpp @@ -0,0 +1,317 @@ +#include "engine/task_model.hpp" +#include "engine/common.hpp" + +#include +#include +#include + +namespace nvr +{ +ModelTask model_task(const std::string &name) +{ + if (name == "detect") + { + return ModelTask::Detect; + } + if (name == "segment") + { + return ModelTask::Segment; + } + if (name == "pose") + { + return ModelTask::Pose; + } + if (name == "obb") + { + return ModelTask::Obb; + } + if (name == "classify") + { + return ModelTask::Classify; + } + if (name == "semantic") + { + return ModelTask::Semantic; + } + if (name == "depth") + { + return ModelTask::Depth; + } + throw std::runtime_error("Unsupported model task: " + name); +} + +const char *task_name(ModelTask task) +{ + switch (task) + { + case ModelTask::Detect: + return "detect"; + case ModelTask::Segment: + return "segment"; + case ModelTask::Pose: + return "pose"; + case ModelTask::Obb: + return "obb"; + case ModelTask::Classify: + return "classify"; + case ModelTask::Semantic: + return "semantic"; + case ModelTask::Depth: + return "depth"; + } + throw std::runtime_error("Invalid model task"); +} + +size_t TensorSpec::elements() const +{ + require(!dimensions.empty() && dimensions.size() <= 4, "Unsupported tensor rank"); + size_t count = 1; + for (int dimension : dimensions) + { + require(dimension > 0 && count <= 268435456 / size_t(dimension), + "Invalid or excessive tensor dimensions"); + count *= size_t(dimension); + } + return count; +} + +TaskPostprocessor::TaskPostprocessor(const ModelOptions &options, + const std::vector &specs, int width, int height) + : options_(options), specs_(specs), input_width_(width), input_height_(height) +{ + require(width > 0 && height > 0 && !specs.empty() && specs.size() <= 16, + "Invalid model input or output count"); + require(options.classes >= 0 && options.classes <= 65535, "Invalid metadata class count"); + require(!options.center_crop || options.task == ModelTask::Classify, + "Center-crop preprocessing is only supported for classification"); + require(options.dense_resize == DenseResize::None || options.task == ModelTask::Semantic || + options.task == ModelTask::Depth, + "Dense interpolation requires a dense task"); + for (const auto &spec : specs) + { + spec.elements(); + require(spec.dimensions.size() < 3 || spec.dimensions[0] == 1, + "Only batch-1 model outputs are supported"); + } + if (options.task == ModelTask::Classify) + { + require(specs.size() == 1 && specs[0].dimensions.size() <= 4, + "Classification requires one probability tensor"); + const auto &d = specs[0].dimensions; + require(d.size() == 1 || d[0] == 1, "Classification requires batch 1"); + int count = int(specs[0].elements()); + require(options.classes == 0 || options.classes == count, + "Classification class count does not match metadata"); + require(count > 0 && count <= 65535, "Invalid classification output size"); + options_.classes = count; + layout_ = Layout::Classification; + return; + } + if (options.task == ModelTask::Semantic || options.task == ModelTask::Depth) + { + require(specs.size() == 1, "Dense prediction requires one output tensor"); + const auto &s = specs[0]; + const auto &d = s.dimensions; + require(d.size() == 3 || d.size() == 4, "Dense output must be [1,H,W] or [1,C,H,W]"); + if (d.size() == 4) + { + int channels = d[s.channels_first ? 1 : 3]; + require(options.task != ModelTask::Depth || channels == 1, + "Depth output requires one channel"); + require(options.task != ModelTask::Semantic || options.classes == 0 || + channels == options.classes, + "Semantic class count differs from metadata"); + options_.classes = channels; + } + else + { + require(options.task != ModelTask::Semantic || options.classes > 0, + "A semantic class map requires metadata class names"); + } + require(options.dense_resize == DenseResize::None || d.size() == 4, + "Class-ID maps cannot be interpolated as logits"); + layout_ = Layout::Dense; + return; + } + if (options.task == ModelTask::Pose) + { + require(options.keypoints > 0 && options.keypoints <= 512 && + (options.keypoint_dimensions == 2 || options.keypoint_dimensions == 3), + "kpt_shape must be [count,2] or [count,3]"); + } + // A prototype image is distinct from the per-proposal prediction tensor. + if (options.task == ModelTask::Segment) + { + for (size_t i = 0; i < specs.size(); ++i) + { + const auto &s = specs[i]; + const auto &d = s.dimensions; + if (d.size() == 4 && d[s.channels_first ? 1 : 3] <= 256) + { + // Rockchip raw heads append the prototype tensor after all branches. + prototypes_ = int(i); + } + } + require(prototypes_ >= 0, "Segmentation requires mask prototypes"); + const auto &proto = specs[size_t(prototypes_)]; + mask_channels_ = proto.dimensions[proto.channels_first ? 1 : 3]; + } + const int extras = options.task == ModelTask::Segment ? mask_channels_ + : options.task == ModelTask::Pose + ? options.keypoints * options.keypoint_dimensions + : options.task == ModelTask::Obb ? 1 + : 0; + // Standard exports: a decoded matrix, optionally followed by mask prototypes. + if (specs.size() == size_t(prototypes_ >= 0 ? 2 : 1)) + { + prediction_ = prototypes_ == 0 ? 1 : 0; + const auto &d = specs[size_t(prediction_)].dimensions; + require(d.size() == 3, "Decoded predictions require a rank-3 tensor"); + const bool end_to_end = + d[2] == 6 + extras && (options.end_to_end || d[1] != 4 + options.classes + extras); + if (end_to_end) + { + require(options.classes > 0, "End-to-end outputs require metadata class names"); + layout_ = Layout::EndToEnd; + feature_major_ = false; + features_ = d[2]; + proposals_ = d[1]; + } + else + { + int expected = options.classes ? 4 + options.classes + extras : 0; + feature_major_ = expected ? d[1] == expected : d[1] < d[2]; + features_ = d[feature_major_ ? 1 : 2]; + proposals_ = d[feature_major_ ? 2 : 1]; + int classes = features_ - 4 - extras; + require(classes > 0 && (options.classes == 0 || classes == options.classes), + "Decoded class count or task channels differ from metadata"); + options_.classes = classes; + } + return; + } + // Rockchip's truncated head exposes scores and xyxy boxes separately. + for (size_t i = 0; i < specs.size(); ++i) + { + if (int(i) == prototypes_) + { + continue; + } + const auto &d = specs[i].dimensions; + if (d.size() == 3 && (d[1] == 4 || d[2] == 4)) + { + if (boxes_ < 0 || specs[i].name == "boxes" || + specs[i].name.find("Mul_2") != std::string::npos) + { + boxes_ = int(i); + } + } + } + if (boxes_ >= 0) + { + const auto &d = specs[size_t(boxes_)].dimensions; + box_major_ = d[1] == 4; + proposals_ = d[box_major_ ? 2 : 1]; + for (size_t i = 0; i < specs.size(); ++i) + { + if (int(i) == boxes_ || int(i) == prototypes_) + { + continue; + } + const auto &shape = specs[i].dimensions; + if (shape.size() != 3) + { + continue; + } + int channels = shape[shape[2] == proposals_ ? 1 : 2]; + if (shape[1] != proposals_ && shape[2] != proposals_) + { + continue; + } + if (extras && channels == extras && auxiliary_ < 0 && + (options.classes != extras || (specs[i].name != "scores" && + specs[i].name.find("Sigmoid") == std::string::npos))) + { + auxiliary_ = int(i); + } + else if (options.classes == 0 || channels == options.classes) + { + scores_ = int(i); + options_.classes = channels; + score_major_ = shape[2] == proposals_; + } + } + require(scores_ >= 0 && (extras == 0 || auxiliary_ >= 0), + "Split head lacks scores or task-specific output"); + layout_ = Layout::Split; + return; + } + // Raw model-zoo exports use three spatial prediction branches. + int count = int(specs.size()) - (prototypes_ >= 0 ? 1 : 0); + int stride = count == 13 - 1 ? 4 : count == 9 || count == 10 ? 3 : count == 6 ? 2 : 0; + const bool packed_task = + (options.task == ModelTask::Pose || options.task == ModelTask::Obb) && count == 4; + if (packed_task) + { + stride = 1; + } + require(stride > 0 && count >= stride * 3, "Unrecognized YOLO output layout"); + require(prototypes_ < 0 || prototypes_ == int(specs.size()) - 1, + "Raw segmentation heads require the prototype tensor last"); + proposals_ = 0; + for (int branch = 0; branch < 3; ++branch) + { + int reg = branch * stride, cls = packed_task ? reg : reg + 1; + const auto &d = specs[size_t(reg)].dimensions; + const auto &c = specs[size_t(cls)].dimensions; + require(d.size() == 4 && c.size() == 4 && specs[size_t(reg)].channels_first && + specs[size_t(cls)].channels_first && d[2] == c[2] && d[3] == c[3] && + width % d[3] == 0 && height % d[2] == 0 && width / d[3] == height / d[2], + "Raw YOLO branches require matching NCHW grids"); + int classes = packed_task ? d[1] - 64 : c[1]; + require(classes > 0 && (options_.classes == 0 || classes == options_.classes), + "Raw head class count differs from metadata"); + options_.classes = classes; + require(packed_task || d[1] == 4 || (d[1] >= 8 && d[1] % 4 == 0 && d[1] <= 128), + "Invalid DFL regression channels"); + regressions_.push_back(reg); + class_scores_.push_back(cls); + if (stride == 4) + { + const auto &extra = specs[size_t(reg + 3)].dimensions; + require(extra.size() == 4 && extra[1] == extras && extra[2] == d[2] && extra[3] == d[3], + "Raw task channel/grid mismatch"); + extras_.push_back(reg + 3); + } + proposals_ += d[2] * d[3]; + } + if (extras && extras_.empty()) + { + auxiliary_ = stride * 3; + require(auxiliary_ < count && + specs[size_t(auxiliary_)].elements() == size_t(extras) * proposals_, + "Raw model is missing keypoints, angles, or mask coefficients"); + } + layout_ = Layout::Dfl; +} + +const char *TaskPostprocessor::description() const +{ + switch (layout_) + { + case Layout::Decoded: + return "decoded YOLO"; + case Layout::EndToEnd: + return "end-to-end YOLO"; + case Layout::Split: + return "split scores/boxes"; + case Layout::Dfl: + return "spatial YOLO heads"; + case Layout::Classification: + return "classification probabilities"; + case Layout::Dense: + return "dense prediction"; + } + return "unknown"; +} +} // namespace nvr diff --git a/src/cpp/native/engine/task_model.hpp b/src/cpp/native/engine/task_model.hpp new file mode 100644 index 0000000..8c49f50 --- /dev/null +++ b/src/cpp/native/engine/task_model.hpp @@ -0,0 +1,141 @@ +#pragma once + +#include "engine/geometry.hpp" +#include "engine/postprocess.hpp" +#include "engine/tensor.hpp" + +#include +#include +#include +#include +#include +#include + +namespace nvr +{ +enum class ModelTask +{ + Detect, + Segment, + Pose, + Obb, + Classify, + Semantic, + Depth +}; + +ModelTask model_task(const std::string &name); +const char *task_name(ModelTask task); + +enum class DenseResize +{ + None, + HalfPixel, + AlignCorners +}; + +struct ModelOptions +{ + ModelTask task = ModelTask::Detect; + int classes = 0; + int keypoints = 17, keypoint_dimensions = 3; + bool normalized = false, end_to_end = false; + bool center_crop = false; + bool apply_nms = true; + DenseResize dense_resize = DenseResize::None; +}; + +struct TensorSpec +{ + std::string name; + std::vector dimensions; + bool channels_first = true; + + size_t elements() const; +}; + +// Row-major runs alternate value and count. box locates the raster in source pixels. +// Depth values use offset + value * scale; semantic values are exact class IDs. +struct ResultRaster +{ + int width = 0, height = 0; + std::array box{}; + std::vector runs; + float offset = 0, scale = 1; +}; + +struct TaskDetection +{ + Detection detection; + std::vector> keypoints; + std::vector> polygon; + std::optional mask; +}; + +struct ClassPrediction +{ + int class_id = 0; + float confidence = 0; +}; + +struct TaskResult +{ + std::vector detections; + std::vector classification; + std::optional dense; +}; + +// Tensor contracts are validated once, and shared by model inspection and inference. +class TaskPostprocessor +{ +public: + enum class Layout + { + Decoded, + EndToEnd, + Split, + Dfl, + Classification, + Dense + }; + +private: + ModelOptions options_; + std::vector specs_; + int input_width_, input_height_; + Layout layout_ = Layout::Decoded; + int prediction_ = 0, boxes_ = -1, scores_ = -1, prototypes_ = -1, auxiliary_ = -1; + int features_ = 0, proposals_ = 0, mask_channels_ = 0; + bool feature_major_ = true, box_major_ = true, score_major_ = true; + std::vector regressions_, class_scores_, extras_; + + static void append_run(ResultRaster &raster, uint32_t value); + +public: + TaskPostprocessor(const ModelOptions &options, const std::vector &specs, + int input_width, int input_height); + + Layout layout() const + { + return layout_; + } + + int classes() const + { + return options_.classes; + } + + int proposals() const + { + return proposals_; + } + + const char *description() const; + + // Dense outputs can be decoded directly from native FP16 or quantized storage. + TaskResult decode_dense(const Geometry &geometry, const TensorView &output) const; + + TaskResult decode(const Geometry &geometry, const std::vector> &outputs, + float confidence, float nms_iou) const; +}; +} // namespace nvr diff --git a/src/cpp/native/engine/tensor.hpp b/src/cpp/native/engine/tensor.hpp new file mode 100644 index 0000000..344f1fc --- /dev/null +++ b/src/cpp/native/engine/tensor.hpp @@ -0,0 +1,101 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace nvr +{ +// Borrowed tensor storage. The caller retains the buffer through decoding. +// Quantized values are converted only when read, rather than expanding a whole image. +struct TensorView +{ + enum class Type + { + Float32, + Float16, + Int8, + Uint8, + Int32 + }; + + const void *data = nullptr; + size_t count = 0; + Type type = Type::Float32; + float scale = 1; + int zero_point = 0; + + // Optional native channel blocking; indices presented to callers remain NCHW. + size_t width = 0, height = 0, row_stride = 0, channel_block = 0; + + float operator[](size_t index) const + { + if (channel_block) + { + const size_t plane = width * height; + const size_t channel = index / plane, pixel = index % plane; + index = + ((channel / channel_block * height + pixel / width) * row_stride + pixel % width) * + channel_block + + channel % channel_block; + } + return scalar(index); + } + + float spatial(size_t channel, size_t x, size_t y) const + { + const size_t index = + ((channel / channel_block * height + y) * row_stride + x) * channel_block + + channel % channel_block; + return scalar(index); + } + +private: + float scalar(size_t index) const + { + switch (type) + { + case Type::Float32: + { + float value; + std::memcpy(&value, static_cast(data) + index * 4, 4); + return value; + } + case Type::Float16: + { +#if defined(__aarch64__) || defined(__ARM_FP16_FORMAT_IEEE) + __fp16 value; + std::memcpy(&value, static_cast(data) + index * 2, 2); + return float(value); +#else + uint16_t bits; + std::memcpy(&bits, static_cast(data) + index * 2, 2); + const float sign = bits & 0x8000 ? -1.f : 1.f; + const int exponent = (bits >> 10) & 31, fraction = bits & 1023; + if (exponent == 31) + { + return fraction ? std::numeric_limits::quiet_NaN() + : sign * std::numeric_limits::infinity(); + } + return sign * std::ldexp(float(exponent ? fraction + 1024 : fraction), + exponent ? exponent - 25 : -24); +#endif + } + case Type::Int8: + return (float(static_cast(data)[index]) - float(zero_point)) * scale; + case Type::Uint8: + return (float(static_cast(data)[index]) - float(zero_point)) * scale; + case Type::Int32: + { + int32_t value; + std::memcpy(&value, static_cast(data) + index * 4, 4); + return (float(value) - float(zero_point)) * scale; + } + } + throw std::runtime_error("Invalid tensor scalar type"); + } +}; +} // namespace nvr diff --git a/src/cpp/requirements.txt b/src/cpp/requirements.txt new file mode 100644 index 0000000..6c9a1ba --- /dev/null +++ b/src/cpp/requirements.txt @@ -0,0 +1,6 @@ +fastapi==0.162.0 +pydantic==2.14.3 +uvicorn==0.54.0 +python-multipart==0.0.33 +numpy==2.3.5 +opencv-python-headless==4.10.0.84 diff --git a/src/cpp/rkapi/__init__.py b/src/cpp/rkapi/__init__.py new file mode 100644 index 0000000..aef59a4 --- /dev/null +++ b/src/cpp/rkapi/__init__.py @@ -0,0 +1 @@ +"""REST API backed by the Rockchip C++ hardware pipeline.""" diff --git a/src/cpp/rkapi/app.py b/src/cpp/rkapi/app.py new file mode 100644 index 0000000..6f018eb --- /dev/null +++ b/src/cpp/rkapi/app.py @@ -0,0 +1,313 @@ +import asyncio +import json +import logging +import os +import tempfile +from contextlib import asynccontextmanager +from pathlib import Path +from time import perf_counter + +import cv2 +import numpy as np +from fastapi import FastAPI, File, Form, HTTPException, Request, UploadFile +from fastapi.responses import FileResponse, JSONResponse, RedirectResponse, StreamingResponse +from pydantic import BaseModel, ConfigDict, Field +from starlette.concurrency import run_in_threadpool +from starlette.datastructures import MutableHeaders + +from .runtime import load_runtime +from .backends.base import SourceUnavailable, UnsupportedOperation + +MAX_UPLOAD = int(os.getenv("RK_MAX_UPLOAD_MB", "128")) * 1024 * 1024 +MAX_PIXELS = 8192 * 8192 +log = logging.getLogger("rkapi") + + +class RequestTiming: + """Measure ingress before FastAPI parses multipart data, without buffering it.""" + + def __init__(self, app): + self.app = app + + async def __call__(self, scope, receive, send): + if scope["type"] != "http": + return await self.app(scope, receive, send) + state = scope["rk_timing"] = {"start": perf_counter(), "receive_wait_ms": 0.0} + + async def timed_receive(): + started = perf_counter() + message = await receive() + state["receive_wait_ms"] += (perf_counter() - started) * 1000 + return message + + async def timed_send(message): + if message["type"] == "http.response.start": + now = perf_counter() + metrics = {"server_total": (now - state["start"]) * 1000, + "receive_wait": state["receive_wait_ms"]} + if "handler_start" in state: + metrics["request_parse"] = (state["handler_start"] - state["start"]) * 1000 + MutableHeaders(scope=message).append( + "Server-Timing", ", ".join(f"{name};dur={value:.3f}" for name, value in metrics.items())) + await send(message) + + await self.app(scope, timed_receive, timed_send) + + +def prediction_response(result): + started = perf_counter() + response = JSONResponse(result) + response.headers["Server-Timing"] = f"response_encode;dur={(perf_counter() - started) * 1000:.3f}" + response.headers["Cache-Control"] = "no-store" + return response + + +class Configuration(BaseModel): + model_config = ConfigDict(extra="forbid", allow_inf_nan=False) + obj_thresh: float | None = Field(default=None, ge=0, le=1) + nms_thresh: float | None = Field(default=None, ge=0, le=1) + + +@asynccontextmanager +async def lifespan(app): + # One process owns the NPU contexts; concurrent requests use its native mutex. + app.state.runtime = await run_in_threadpool(load_runtime) + try: + yield + finally: + runtime = app.state.runtime + await run_in_threadpool(runtime.close) + + +app = FastAPI(title="Rockchip CV API", version="0.3.0", lifespan=lifespan) +app.add_middleware(RequestTiming) + + +@app.get("/health") +def health(): + rt = app.state.runtime + response = {"status": "ok", "model": rt.model_info, "backend": rt.description, + "live": rt.source_status()} + if rt.demo_video: + response["demo_video"] = True + return response + + +@app.get("/api/models") +def models(): + """Describe the model loaded by this image; this is not a remote model registry.""" + return {"models": [app.state.runtime.model_info]} + + +@app.get("/api/metadata") +def metadata(): + """Describe how to interpret predictions from the loaded model.""" + rt = app.state.runtime + return {"schema_version": 1, "model": rt.model_info, **rt.prediction_metadata} + + +@app.get("/api/config") +def get_config(): + rt = app.state.runtime + with rt.config_lock: + return dict(rt.config) + + +@app.post("/api/config") +def set_config(config: Configuration): + rt = app.state.runtime + values = config.model_dump(exclude_none=True) + rt.configure(values) + return {"status": "success"} + + +async def save_upload(upload, target): + size = 0 + while chunk := await upload.read(1024 * 1024): + size += len(chunk) + if size > MAX_UPLOAD: + raise HTTPException(413, "upload exceeds RK_MAX_UPLOAD_MB") + target.write(chunk) + if size == 0: + raise HTTPException(400, "empty upload") + target.flush() + + +def video_frame(path, timestamp): + capture = cv2.VideoCapture(path) + try: + if not capture.isOpened(): + raise HTTPException(400, "cannot open uploaded video") + if timestamp > 0: + if not capture.set(cv2.CAP_PROP_POS_MSEC, timestamp * 1000): + raise HTTPException(400, "video does not support seeking") + ok, image = capture.read() + if not ok: + raise HTTPException(400, "timestamp is outside the video or frame cannot be decoded") + return image + finally: + capture.release() + + +@app.post("/api/models/{model_id}/predict", openapi_extra={"parameters": [ + {"name": "model_id", "in": "path", "required": True, "schema": {"type": "string"}, + "description": "Loaded model ID or family from /api/models"}]}) +@app.post("/api/predict") +async def predict(request: Request, file: UploadFile | None = File(None), video: UploadFile | None = File(None), + timestamp: float = Form(0, ge=0, allow_inf_nan=False), realtime: bool = Form(False), + conf: float | None = Form(None, ge=0, le=1, allow_inf_nan=False), + iou: float | None = Form(None, ge=0, le=1, allow_inf_nan=False)): + request.scope["rk_timing"]["handler_start"] = perf_counter() + rt = app.state.runtime + model_id = request.path_params.get("model_id") + if model_id and model_id not in rt.aliases: + raise HTTPException(404, "model is not loaded; see /api/models") + conf, iou = rt.thresholds(conf, iou) + if file is not None and video is not None: + raise HTTPException(400, "provide either file or video") + if realtime and (file is not None or video is not None): + raise HTTPException(400, "realtime cannot be combined with an upload") + try: + if file is None and video is None: + result = await run_in_threadpool(rt.predict_realtime, conf, iou) + result["model"] = rt.model_info + return prediction_response(result) + upload = file if file is not None else video + # Generated names live outside the source tree and are removed on all paths. + with tempfile.NamedTemporaryFile(suffix=".media") as temporary: + started = perf_counter() + await save_upload(upload, temporary) + upload_store_ms = (perf_counter() - started) * 1000 + decode_started = perf_counter() + if file is not None: + def decode(): + return cv2.imdecode(np.fromfile(temporary.name, dtype=np.uint8), cv2.IMREAD_COLOR) + image = await run_in_threadpool(decode) + source = "image" + else: + image = await run_in_threadpool(video_frame, temporary.name, timestamp) + source = f"video frame at {timestamp:g}s" + decode_ms = (perf_counter() - decode_started) * 1000 + if image is None or image.size == 0: + raise HTTPException(400, "cannot decode image or video frame") + height, width = image.shape[:2] + if width > 8192 or height > 8192 or width * height > MAX_PIXELS: + raise HTTPException(413, "image dimensions exceed 8192 x 8192") + process_started = perf_counter() + result = await run_in_threadpool(rt.image, image, conf, iou, source) + result["model"] = rt.model_info + result.setdefault("timing", {}).update({"upload_store_ms": upload_store_ms, "decode_ms": decode_ms, + "process_ms": (perf_counter() - process_started) * 1000}) + return prediction_response(result) + except SourceUnavailable as error: + raise HTTPException(503, str(error)) from error + except UnsupportedOperation as error: + raise HTTPException(501, str(error)) from error + except HTTPException: + raise + except (RuntimeError, cv2.error, ValueError) as error: + log.exception("prediction failed") + raise HTTPException(400, str(error)) from error + finally: + for upload in (file, video): + if upload is not None: + await upload.close() + + +@app.get("/api/video_feed") +async def video_feed(): + rt = app.state.runtime + try: + await run_in_threadpool(rt.preview) + except SourceUnavailable as error: + raise HTTPException(503, str(error)) from error + except UnsupportedOperation as error: + raise HTTPException(501, str(error)) from error + + async def frames(): + previous = None + while True: + try: + state, jpeg = await run_in_threadpool(rt.preview) + except (SourceUnavailable, UnsupportedOperation): + break + raw = state["result"] + if state["error"]: + break + if raw is not None and raw["frame"] != previous: + previous = raw["frame"] + annotated = await run_in_threadpool(rt.annotate, jpeg, raw) + yield (b"--frame\r\nContent-Type: image/jpeg\r\nContent-Length: " + + str(len(annotated)).encode() + b"\r\n\r\n" + annotated + b"\r\n") + elif state["ended"]: + break + await asyncio.sleep(.05) + + return StreamingResponse(frames(), media_type="multipart/x-mixed-replace; boundary=frame", + headers={"Cache-Control": "no-store"}) + + +def demo_runtime(): + rt = app.state.runtime + if rt.demo_video is None: + raise HTTPException(404, "set RK_LOOP_VIDEO=1 for the local MP4 demo") + return rt + + +@app.get("/api/demo/video", include_in_schema=False) +def demo_video(): + return FileResponse(demo_runtime().demo_video, media_type="video/mp4", + headers={"Cache-Control": "no-cache"}) + + +@app.get("/api/demo/results", include_in_schema=False) +async def demo_results(): + rt = demo_runtime() + + async def events(): + previous = 0 + while True: + try: + result = await run_in_threadpool(rt.demo_result, previous) + except SourceUnavailable: + break + if result is not None: + previous = result["frame"] + yield "data: " + json.dumps(result, separators=(",", ":")) + "\n\n" + await asyncio.sleep(.015) + + return StreamingResponse(events(), media_type="text/event-stream", + headers={"Cache-Control": "no-store", "X-Accel-Buffering": "no"}) + + +@app.get("/api/demo/candidates", include_in_schema=False) +async def demo_candidates(): + rt = demo_runtime() + if not rt.description.get("live", {}).get("candidate_stream"): + raise HTTPException(404, "the loaded model does not publish detection candidates") + + async def frames(): + previous = 0 + while True: + try: + packet = await run_in_threadpool(rt.demo_candidates, previous) + except SourceUnavailable: + break + if packet is None: + await asyncio.sleep(.008) + continue + previous, data = packet + yield data + + return StreamingResponse(frames(), media_type="application/octet-stream", + headers={"Cache-Control": "no-store", "X-Accel-Buffering": "no"}) + + +@app.get("/", include_in_schema=False) +def index(): + return FileResponse(Path(__file__).with_name("demo.html"), media_type="text/html") + + +@app.get("/demo", include_in_schema=False) +def demo(request: Request): + return RedirectResponse(request.url_for("index"), status_code=308) diff --git a/src/cpp/rkapi/backends/__init__.py b/src/cpp/rkapi/backends/__init__.py new file mode 100644 index 0000000..bd0e3bc --- /dev/null +++ b/src/cpp/rkapi/backends/__init__.py @@ -0,0 +1 @@ +"""Task adapters selected by each model package.""" diff --git a/src/cpp/rkapi/backends/base.py b/src/cpp/rkapi/backends/base.py new file mode 100644 index 0000000..a831412 --- /dev/null +++ b/src/cpp/rkapi/backends/base.py @@ -0,0 +1,59 @@ +"""Shared adapter contract; HTTP handling does not depend on detection outputs.""" +import threading +from abc import ABC, abstractmethod + + +class SourceUnavailable(RuntimeError): + pass + + +class UnsupportedOperation(RuntimeError): + pass + + +class BaseBackend(ABC): + def __init__(self): + self.config = {"obj_thresh": 0.25, "nms_thresh": 0.45} + self.config_lock = threading.Lock() + self.description = {} + self.prediction_metadata = {} + self.demo_video = None + + def thresholds(self, conf, iou): + with self.config_lock: + return (self.config["obj_thresh"] if conf is None else conf, + self.config["nms_thresh"] if iou is None else iou) + + def configure(self, values): + with self.config_lock: + self.config.update(values) + + @abstractmethod + def image(self, image, conf, iou, source): + """Return a JSON-compatible task result with a timing dictionary. + + image is decoded BGR. conf and iou are optional detection controls; + adapters for other tasks may ignore them. Task-specific outputs belong + in the returned dictionary, not in the shared HTTP layer. + """ + + def source_status(self): + return None + + def predict_realtime(self, conf, iou): + raise UnsupportedOperation("this backend does not support live inference") + + def preview(self): + raise UnsupportedOperation("this backend does not support video preview") + + def demo_result(self, after): + raise UnsupportedOperation("this backend does not support timed detection results") + + def demo_candidates(self, after): + raise UnsupportedOperation("this backend does not support timed detection candidates") + + def annotate(self, jpeg, raw): + return jpeg + + def close(self): + pass diff --git a/src/cpp/rkapi/backends/yolo.py b/src/cpp/rkapi/backends/yolo.py new file mode 100644 index 0000000..60edd00 --- /dev/null +++ b/src/cpp/rkapi/backends/yolo.py @@ -0,0 +1,217 @@ +"""Rockchip RGA/DMA/RKNN inference and task results.""" +import json +import os +from pathlib import Path +from time import perf_counter + +import cv2 +import numpy as np + +from .. import _native +from .base import BaseBackend, SourceUnavailable + +COCO = ("person,bicycle,car,motorcycle,airplane,bus,train,truck,boat,traffic light," + "fire hydrant,stop sign,parking meter,bench,bird,cat,dog,horse,sheep,cow," + "elephant,bear,zebra,giraffe,backpack,umbrella,handbag,tie,suitcase,frisbee," + "skis,snowboard,sports ball,kite,baseball bat,baseball glove,skateboard," + "surfboard,tennis racket,bottle,wine glass,cup,fork,knife,spoon,bowl,banana," + "apple,sandwich,orange,broccoli,carrot,hot dog,pizza,donut,cake,chair,couch," + "potted plant,bed,dining table,toilet,tv,laptop,mouse,remote,keyboard," + "cell phone,microwave,oven,toaster,sink,refrigerator,book,clock,vase,scissors," + "teddy bear,hair drier,toothbrush").split(",") + + +def prediction_output(task, package): + field = {"classify": "classification", "semantic": "semantic", "depth": "depth"}.get( + task, "predictions") + output = {"field": field} + if task != "classify": + output["coordinate_space"] = "source_image_pixels" + + if task == "pose": + output.update(item_field="keypoints", point_fields=["x", "y", "confidence"], + keypoint_count=package.get("kpt_shape", [17])[0]) + elif task == "obb": + output.update(item_field="polygon", point_fields=["x", "y"], point_count=4) + elif task == "segment": + output["item_field"] = "mask" + + if task in ("segment", "semantic", "depth"): + output["raster"] = { + "encoding": "rle", "order": "row_major", "run_order": ["value", "count"], + "box_order": ["x1", "y1", "x2", "y2"], + "values": {"segment": "binary", "semantic": "class_id", "depth": "quantized_depth"}[task], + } + if task == "depth": + output["raster"]["invalid_value"] = 65536 + return output + + +class Backend(BaseBackend): + def __init__(self, package): + super().__init__() + model = package["file"] + nms_mode = os.getenv("RK_NMS", "on" if package.get("nms", True) else "off").lower() + if nms_mode not in ("on", "off"): + raise ValueError("RK_NMS must be on or off") + options = {key: package[key] for key in ("classes", "end2end", "coordinates", + "preprocess", "dense_resize", "kpt_shape") + if key in package} + options.update(task="detect" if package["task"] == "detection" else package["task"], + nms=nms_mode == "on") + encoded_options = json.dumps(options) + self.predictor = _native.Predictor(model, encoded_options) + self.description = json.loads(self.predictor.description()) + self.description["chip"] = package["chip"] + self.labels = COCO if self.description["classes"] == 80 else [] + if path := os.getenv("RK_CLASSES", package.get("labels")): + with open(path, encoding="utf-8") as f: + self.labels = [line.strip() for line in f if line.strip()] + if len(self.labels) != self.description["classes"]: + raise ValueError("class file must have one label per model class") + task = package["task"] + self.prediction_metadata = { + "model_input": {"width": self.description["input_width"], + "height": self.description["input_height"]}, + "class_count": self.description["classes"] if task != "depth" else 0, + "labels": self.labels if task != "depth" else [], + "output": prediction_output(task, package), + } + self.live = None + source = os.getenv("RK_SOURCE") + if os.getenv("RK_LOOP_VIDEO") == "1": + path = Path(source or "") + if task != "detection" or path.suffix.lower() != ".mp4" or not path.is_file(): + raise ValueError("RK_LOOP_VIDEO requires a detection model and a local MP4 as RK_SOURCE") + self.demo_video = path.resolve() + if source: + self.live = _native.Live(source, model, int(os.getenv("RK_WORKERS", "0")), + encoded_options, loop_file=self.demo_video is not None) + self.description["live"] = json.loads(self.live.description()) + + def response(self, raw, source): + predictions = [] + for detection in raw["detections"]: + idx = detection["class_id"] + label = self.labels[idx] if 0 <= idx < len(self.labels) else str(idx) + prediction = {"class": label, "class_id": idx, "confidence": detection["confidence"], + "box": dict(zip(("x1", "y1", "x2", "y2"), detection["box"]))} + for key in ("keypoints", "polygon", "mask"): + if key in detection: + prediction[key] = detection[key] + predictions.append(prediction) + result = {"success": True, "source": source, "predictions": predictions, + "image": {"width": raw["source_width"], "height": raw["source_height"]}, + "timing": {key: raw[key] for key in ( + "context_wait_ms", "dma_upload_ms", "preprocess_ms", "inference_ms", + "rknn_run_ms", "output_get_ms", "postprocess_ms", "nms_ms") if key in raw}} + if "classification" in raw: + result["classification"] = [ + {**item, "class": self.labels[item["class_id"]] if self.labels else str(item["class_id"])} + for item in raw["classification"]] + if "dense" in raw: + result[raw["task"]] = raw["dense"] + if raw["task"] == "semantic": + result["semantic"]["labels"] = self.labels + return result + + def image(self, image, conf, iou, source): + started = perf_counter() + if image is None or image.size == 0: + raise ValueError("cannot decode image or video frame") + height, width = image.shape[:2] + if width > 8192 or height > 8192 or width * height > 8192 * 8192: + raise ValueError("image dimensions exceed 8192 x 8192") + # NV12 needs even dimensions; preserve original API coordinates below. + w, h = (width + 1) & ~1, (height + 1) & ~1 + if (w, h) != (width, height): + image = cv2.copyMakeBorder(image, 0, h - height, 0, w - width, cv2.BORDER_REPLICATE) + planar = cv2.cvtColor(image, cv2.COLOR_BGR2YUV_I420).reshape(-1) + y_size = w * h + uv_size = y_size // 4 + nv12 = np.empty(y_size + uv_size * 2, dtype=np.uint8) + nv12[:y_size] = planar[:y_size] + nv12[y_size::2] = planar[y_size:y_size + uv_size] + nv12[y_size + 1::2] = planar[y_size + uv_size:] + pixels = nv12.tobytes() + prepare_ms = (perf_counter() - started) * 1000 + started = perf_counter() + encoded = self.predictor.predict(pixels, w, h, conf, iou) + native_ms = (perf_counter() - started) * 1000 + started = perf_counter() + raw = json.loads(encoded) + raw["source_width"], raw["source_height"] = width, height + for item in raw["detections"]: + item["box"] = [max(0, min(value, width if i % 2 == 0 else height)) + for i, value in enumerate(item["box"])] + result = self.response(raw, source) + result["timing"].update({"color_convert_ms": prepare_ms, "native_call_ms": native_ms, + "response_build_ms": (perf_counter() - started) * 1000}) + return result + + def annotate(self, jpeg, raw): + image = cv2.imdecode(np.frombuffer(jpeg, dtype=np.uint8), cv2.IMREAD_COLOR) + if image is None: + return jpeg + sx, sy = image.shape[1] / raw["source_width"], image.shape[0] / raw["source_height"] + for detection in self.response(raw, "realtime")["predictions"]: + box = detection["box"] + x1, y1, x2, y2 = (int(box["x1"] * sx), int(box["y1"] * sy), + int(box["x2"] * sx), int(box["y2"] * sy)) + cv2.rectangle(image, (x1, y1), (x2, y2), (40, 220, 90), 2) + cv2.putText(image, f'{detection["class"]} {detection["confidence"]:.2f}', + (x1, max(14, y1 - 4)), cv2.FONT_HERSHEY_SIMPLEX, .45, (40, 220, 90), 1) + ok, encoded = cv2.imencode(".jpg", image, [cv2.IMWRITE_JPEG_QUALITY, 80]) + return encoded.tobytes() if ok else jpeg + + def preview(self): + if not self.live: + raise SourceUnavailable("configure RK_SOURCE for realtime predictions") + state, jpeg = self.live.snapshot() + state = json.loads(state) + if state["error"]: + raise SourceUnavailable(state["error"]) + return state, jpeg + + def source_status(self): + if not self.live: + return None + return json.loads(self.live.state()) + + def live_result(self, after=0): + if not self.live: + raise SourceUnavailable("configure RK_SOURCE for realtime predictions") + state = json.loads(self.live.state(True, after)) + if state["error"]: + raise SourceUnavailable(state["error"]) + return state["result"] + + def demo_result(self, after): + raw = self.live_result(after) + if raw is None or "media_time" not in raw: + return None + result = self.response(raw, "realtime") + return {key: raw[key] for key in ("frame", "media_time", "cycle")} | { + "predictions": result["predictions"], "image": result["image"]} + + def demo_candidates(self, after): + try: + return self.live.candidates(after) + except RuntimeError as error: + raise SourceUnavailable(str(error)) from error + + def predict_realtime(self, conf, iou): + if self.live_result() is None: + raise SourceUnavailable("live source has no frame yet") + raw = self.live.predict(self.predictor, conf, iou) + return self.response(json.loads(raw), "realtime") + + def configure(self, values): + with self.config_lock: + self.config.update(values) + if self.live: + self.live.configure(self.config["obj_thresh"], self.config["nms_thresh"]) + + def close(self): + if self.live: + self.live.close() diff --git a/src/cpp/rkapi/demo.html b/src/cpp/rkapi/demo.html new file mode 100644 index 0000000..de025b3 --- /dev/null +++ b/src/cpp/rkapi/demo.html @@ -0,0 +1,500 @@ + + + + + + Live detection | Rockchip CV API + + + +
+
+ Rockchip CV API + API docs +
+ +

Live detection

+

Reading model information…

+ +
+
+
+ Video feed + Connecting +
+
+ + + +

Waiting for the board to respond…

+
+
+ +
+

Detection thresholds

+

Changes apply to the stream and API defaults for this container.

+
+
+
+ + — +
+ +
+
+
+ + — +
+ + +
+ +
+

+
+
+ + +
+ + + + diff --git a/src/cpp/rkapi/packages.py b/src/cpp/rkapi/packages.py new file mode 100644 index 0000000..efd9f17 --- /dev/null +++ b/src/cpp/rkapi/packages.py @@ -0,0 +1,96 @@ +"""Model catalog and packaged-model validation; usable without inference libraries.""" + +import hashlib +import json +import re +from pathlib import Path +from urllib.parse import urlsplit + +IDENTIFIER = re.compile(r"^[a-z0-9]+(?:[-_][a-z0-9]+)*$") +HF_REVISION = re.compile(r"/[^/]+/[^/]+/resolve/[0-9a-f]{40}/.+") + + +def checked_path(root, value): + path = (Path(root) / value).resolve() + if not path.is_relative_to(Path(root).resolve()): + raise ValueError(f"Path must stay inside {root}: {value}") + return path + + +def sha256(path): + digest = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def validate_spec(spec): + for key in ("id", "family", "chip", "task", "backend", "tag"): + if not isinstance(spec.get(key), str) or not IDENTIFIER.fullmatch(spec[key]): + raise ValueError(f"Invalid model {key}: {spec.get(key)!r}") + if spec["chip"] not in ("rk3576", "rk3588"): + raise ValueError(f"Unsupported chip: {spec['chip']}") + if not re.fullmatch(r"[0-9a-f]{64}", spec.get("sha256", "")): + raise ValueError(f"{spec['id']}: a SHA-256 checksum is required") + if not isinstance(spec.get("nms", True), bool): + raise ValueError(f"{spec['id']}: nms must be a boolean") + for key in ("aliases", "id_aliases"): + aliases = spec.get(key, []) + if not isinstance(aliases, list) or any(not isinstance(alias, str) or + not IDENTIFIER.fullmatch(alias) for alias in aliases): + raise ValueError(f"{spec['id']}: {key} must be a list of identifiers") + + +def load_catalog(directory, root): + directory = Path(directory) + models = [] + for manifest in sorted(directory.glob("*/*/*/*/*/model.json")): + spec = json.loads(manifest.read_text(encoding="utf-8")) + family, task, size, chip, precision = manifest.relative_to(directory).parts[:-1] + expected = {"family": family, "task": "detection" if task == "detect" else task, + "size": size, "chip": chip, "precision": precision} + if any(spec.get(key) != value for key, value in expected.items()): + raise ValueError(f"Model manifest fields do not match its path: {manifest}") + models.append(spec) + if not models: + raise ValueError("The model catalog is empty") + ids, tags = set(), set() + for spec in models: + validate_spec(spec) + model_ids = [spec["id"], *spec.get("id_aliases", [])] + model_tags = [spec["tag"], *spec.get("aliases", [])] + if (len(set(model_ids)) != len(model_ids) or ids.intersection(model_ids) or + len(set(model_tags)) != len(model_tags) or tags.intersection(model_tags)): + raise ValueError("Model IDs and image tags must be unique") + ids.update(model_ids) + tags.update(model_tags) + for key in ("url", "metadata_url"): + value = spec.get(key) + url = urlsplit(value if isinstance(value, str) else "") + if (url.scheme != "https" or url.netloc != "huggingface.co" or + not HF_REVISION.fullmatch(url.path) or url.query or url.fragment): + raise ValueError(f"{spec['id']}: {key} must pin a Hugging Face commit") + if not re.fullmatch(r"[0-9a-f]{64}", spec.get("metadata_sha256", "")): + raise ValueError(f"{spec['id']}: metadata SHA-256 is required") + backend = checked_path(root, f"rkapi/backends/{spec['backend']}.py") + if not backend.is_file(): + raise ValueError(f"{spec['id']}: backend module missing: {spec['backend']}") + return models + + +def load_package(path): + path = Path(path) + spec = json.loads(path.read_text(encoding="utf-8")) + if spec.get("schema_version") != 1: + raise ValueError("Unsupported model package schema") + validate_spec(spec) + for key in ("file", "labels", "metadata"): + if spec.get(key): + resolved = checked_path(path.parent, spec[key]) + if not resolved.is_file(): + raise ValueError(f"Packaged {key} is missing: {resolved}") + spec[key] = str(resolved) + if not spec.get("file") or sha256(spec["file"]) != spec["sha256"]: + raise ValueError("Packaged model checksum mismatch") + return spec diff --git a/src/cpp/rkapi/runtime.py b/src/cpp/rkapi/runtime.py new file mode 100644 index 0000000..f5a11e3 --- /dev/null +++ b/src/cpp/rkapi/runtime.py @@ -0,0 +1,26 @@ +"""Load one validated model package and its task adapter per service process.""" +import importlib +import os +from pathlib import Path + +from .backends.base import BaseBackend +from .packages import load_package + + +def load_runtime(): + package = load_package(os.getenv("RK_PACKAGE", "/opt/model/package.json")) + compatible = Path("/proc/device-tree/compatible") + if compatible.exists() and f"rockchip,{package['chip']}" not in compatible.read_text(errors="replace"): + raise RuntimeError(f"This image requires {package['chip']}; use the image matching your board") + module = importlib.import_module(f"rkapi.backends.{package['backend']}") + if not issubclass(module.Backend, BaseBackend): + raise TypeError("Model backend must implement BaseBackend") + backend = module.Backend(package) + backend.model_info = {key: package[key] for key in + ("id", "name", "family", "task", "chip", "backend", "tag", "size", "precision") + if key in package} + backend.aliases = {package["id"], package["family"], *package.get("id_aliases", [])} + # Retain the original Seeed-compatible URL for existing YOLO26 clients. + if package["family"] == "yolo26" and package["task"] == "detection": + backend.aliases.add("yolo11") + return backend diff --git a/src/cpp/tools/build_images.py b/src/cpp/tools/build_images.py new file mode 100644 index 0000000..c3fd3af --- /dev/null +++ b/src/cpp/tools/build_images.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +"""Build YOLO26 packages from one loaded ARM64 runtime image.""" + +import argparse +from concurrent.futures import ThreadPoolExecutor, as_completed +import json +from pathlib import Path +import subprocess +import tempfile + +from models import ROOT, image_references, load_catalog, prepare + + +def docker(*args, capture=False): + return subprocess.run(["docker", *args], check=True, text=True, capture_output=capture) + + +def layers(image): + return json.loads(docker("image", "inspect", "--format", "{{json .RootFS.Layers}}", + image, capture=True).stdout) + + +def build(spec, reference, runtime, runtime_layers, revision, publish): + image = f"{reference[0]}:{reference[1]}" + versioned = f"{image}-{revision}" if revision else None + with tempfile.TemporaryDirectory(prefix="rk-model-") as directory: + context = Path(directory) + prepare(spec, context / "package") + docker("build", "--builder", "default", "--platform", "linux/arm64", + "--quiet", "--pull=false", "--build-arg", f"RUNTIME_IMAGE={runtime}", + "--file", str(ROOT / "Dockerfile.model"), "--tag", image, str(context)) + + image_layers = layers(image) + if image_layers[:-1] != runtime_layers: + raise RuntimeError(f"{image} must append exactly one model layer to {runtime}") + + docker("run", "--rm", "--entrypoint", "python", image, "-c", + "import sys; from rkapi import _native; from rkapi.app import app; " + "from rkapi.backends.yolo import Backend; " + "from rkapi.packages import load_package; " + "assert load_package('/opt/model/package.json')['id'] == sys.argv[1]; " + "app.openapi()", spec["id"]) + + if versioned: + docker("image", "tag", image, versioned) + if publish: + if versioned: + docker("push", versioned) + docker("push", image) + print(f"Verified shared runtime layers for {image}", flush=True) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--runtime", required=True, help="Local runtime image tag") + parser.add_argument("--repository", required=True, help="Image namespace, such as ghcr.io/owner/repo") + parser.add_argument("--model", help="Catalog model ID (default: all models)") + parser.add_argument("--revision", help="Add a versioned tag for this source revision") + parser.add_argument("--publish", action="store_true") + args = parser.parse_args() + + specs = load_catalog(ROOT / "models", ROOT) + references = image_references(specs) + selected = [spec for spec in specs if not args.model or spec["id"] == args.model] + if not selected: + parser.error(f"Unknown model ID: {args.model}") + if not args.repository or args.repository != args.repository.lower(): + parser.error("The image namespace must be lowercase") + if args.revision and (len(args.revision) != 40 or any(ch not in "0123456789abcdef" for ch in args.revision)): + parser.error("The revision must be a full lowercase Git SHA") + runtime_layers = layers(args.runtime) + if not runtime_layers: + parser.error("The runtime image has no filesystem layers") + + failures = [] + with ThreadPoolExecutor(max_workers=4) as workers: + builds = {} + for spec in selected: + package, tag = references[spec["id"]] + future = workers.submit(build, spec, (f"{args.repository}/{package}", tag), + args.runtime, runtime_layers, args.revision, args.publish) + builds[future] = spec["id"] + for future in as_completed(builds): + try: + future.result() + except (OSError, ValueError, RuntimeError, subprocess.CalledProcessError) as error: + failures.append(builds[future]) + print(f"{builds[future]}: {error}", flush=True) + print(f"Verified {len(selected) - len(failures)} of {len(selected)} model images") + return 1 if failures else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/cpp/tools/models.py b/src/cpp/tools/models.py new file mode 100644 index 0000000..894ddde --- /dev/null +++ b/src/cpp/tools/models.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +"""Validate the catalog, generate CI matrices, and prepare model image assets.""" + +import argparse +from http.client import IncompleteRead +import json +import shutil +import sys +import tempfile +import time +import urllib.error +import urllib.request +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +from rkapi.packages import IDENTIFIER, load_catalog, sha256 + + +def image_references(models): + references = {} + used = set() + for spec in models: + name = spec.get("name") + if not isinstance(name, str) or not IDENTIFIER.fullmatch(name.lower()): + raise ValueError(f"{spec['id']}: invalid image package name: {name!r}") + package = name.lower() + tag = f"{spec['chip']}-{spec['precision']}" + if not IDENTIFIER.fullmatch(tag): + raise ValueError(f"{spec['id']}: invalid image tag: {tag!r}") + reference = f"{package}:{tag}" + if reference in used: + raise ValueError(f"Duplicate image reference: {reference}") + used.add(reference) + references[spec["id"]] = (package, tag) + return references + + +def download(url, target, checksum): + for attempt in range(3): + try: + with urllib.request.urlopen(url, timeout=120) as response: + if not response.url.startswith("https://"): + raise ValueError(f"Download redirected to an insecure URL: {url}") + with target.open("wb") as stream: + shutil.copyfileobj(response, stream) + break + except urllib.error.HTTPError as error: + if attempt == 2 or (error.code not in (408, 429) and error.code < 500): + raise ValueError(f"Could not download {url}: {error}") from error + except (urllib.error.URLError, TimeoutError, ConnectionError, IncompleteRead) as error: + if attempt == 2: + raise ValueError(f"Could not download {url}: {error}") from error + time.sleep(2 ** attempt) + if sha256(target) != checksum: + raise ValueError(f"Checksum mismatch: {url}") + + +def prepare_remote_metadata(spec, staged, package): + import yaml + + metadata = staged / "metadata.yaml" + download(spec["metadata_url"], metadata, spec["metadata_sha256"]) + info = yaml.safe_load(metadata.read_text(encoding="utf-8")) + expected = {"task": "detect" if spec["task"] == "detection" else spec["task"], + "target_platform": spec["chip"]} + if "precision" in info: + expected["precision"] = spec["precision"] + expected.update({key: spec[key] for key in ("end2end", "coordinates", "preprocess", + "dense_resize", "kpt_shape") if key in spec}) + defaults = {"dense_resize": "none", "coordinates": "pixels", "preprocess": "letterbox", + "end2end": False} + for key, value in expected.items(): + if info.get(key, defaults.get(key)) != value: + raise ValueError(f"Model metadata does not match catalog field {key}") + names = info["names"] + if isinstance(names, dict): + names = {int(index): label for index, label in names.items()} + names = [names[index] for index in range(len(names))] + if len(names) != spec["classes"] or any(not isinstance(label, str) or not label.strip() + or "\n" in label or "\r" in label for label in names): + raise ValueError("Metadata labels do not match the class contract") + (staged / "classes.txt").write_text("\n".join(names) + "\n", encoding="utf-8") + package.update(metadata=metadata.name, labels="classes.txt") + + +def prepare(spec, output): + output = output.resolve() + marker = ".rockchip-model-package" + if output == ROOT or ROOT.is_relative_to(output): + raise ValueError("Package output must not replace the repository or its parent") + if output.exists() and any(output.iterdir()) and not (output / marker).is_file(): + raise ValueError("Refusing to replace a directory not created by this tool") + output.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(dir=output.parent) as temporary: + staged = Path(temporary) / "package" + staged.mkdir() + model = staged / "model.rknn" + download(spec["url"], model, spec["sha256"]) + package = {key: value for key, value in spec.items() + if key not in ("url", "metadata_url")} + package.update({"schema_version": 1, "file": "model.rknn"}) + prepare_remote_metadata(spec, staged, package) + (staged / "package.json").write_text(json.dumps(package, indent=2) + "\n", encoding="utf-8") + (staged / marker).touch() + if output.exists(): + output.rmdir() if not any(output.iterdir()) else shutil.rmtree(output) + shutil.move(str(staged), str(output)) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("command", choices=("validate", "list", "matrix", "prepare")) + parser.add_argument("--model", help="Catalog model ID; omit for the complete CI matrix") + parser.add_argument("--output", type=Path, default=ROOT / ".model-package") + args = parser.parse_args() + models = load_catalog(ROOT / "models", ROOT) + images = image_references(models) + selected = [spec for spec in models + if not args.model or args.model in (spec["id"], *spec.get("id_aliases", []))] + if not selected: + raise ValueError(f"Unknown model ID: {args.model}") + if args.command == "matrix": + matrix = [] + for spec in selected: + package, tag = images[spec["id"]] + matrix.append({"id": spec["id"], "package": package, "tag": tag, + "legacy_tag": spec["tag"], "aliases": spec.get("aliases", [])}) + print(json.dumps({"include": matrix})) + elif args.command == "list": + for spec in selected: + package, tag = images[spec["id"]] + print(f"{spec['id']:24} {f'{package}:{tag}':28} {spec['task']:16} {spec['backend']}") + elif args.command == "validate": + print(f"Validated {len(models)} model packages") + else: + if not args.model: + raise ValueError("prepare requires --model") + prepare(selected[0], args.output) + print(f"Prepared {selected[0]['id']} in {args.output}") + + +if __name__ == "__main__": + try: + main() + except (ValueError, OSError) as error: + sys.exit(str(error)) diff --git a/src/cpp/tools/setup-sdk.sh b/src/cpp/tools/setup-sdk.sh new file mode 100644 index 0000000..39c299d --- /dev/null +++ b/src/cpp/tools/setup-sdk.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +set -euo pipefail +# Pinned SDK sources from rockchip-yolo-cpp tools/setup.sh. +root=/opt/rockchip-build +deps=/opt/rockchip +cache="$root/cache" +mkdir -p "$deps" "$cache" +# Official source snapshots pinned by commit, cached for subsequent builds. +fetch_source() +{ + local name="$1" repo="$2" revision="$3" destination="$cache/$1-$3" + if [[ ! -f "$destination/.nvr-ready" ]]; then + echo "Downloading $name ($revision)" >&2 + mkdir -p "$destination" || return 1 + if [[ "$name" == rknn || "$name" == rga ]]; then + # Fetch only ARM64 runtime/headers, not SDK docs, wheels or other CPUs. + [[ -d "$destination/.git" ]] || git init "$destination" >&2 || return 1 + git -C "$destination" config remote.origin.url "https://github.com/$repo.git" || return 1 + git -C "$destination" config remote.origin.promisor true || return 1 + git -C "$destination" config remote.origin.partialclonefilter blob:none || return 1 + git -C "$destination" fetch --depth 1 --filter=blob:none \ + origin "$revision" >&2 || return 1 + if [[ "$name" == rknn ]]; then + git -C "$destination" sparse-checkout set \ + rknpu2/runtime/Linux/librknn_api/include \ + rknpu2/runtime/Linux/librknn_api/aarch64 >&2 || return 1 + else + git -C "$destination" sparse-checkout set include libs/Linux/gcc-aarch64 >&2 || return 1 + fi + git -C "$destination" checkout --detach "$revision" >&2 || return 1 + else + curl -fL --retry 3 --connect-timeout 15 -o "$cache/$name-$revision.tar.gz" \ + "https://codeload.github.com/$repo/tar.gz/$revision" || return 1 + tar -xzf "$cache/$name-$revision.tar.gz" --strip-components=1 -C "$destination" || return 1 + fi + touch "$destination/.nvr-ready" || return 1 + fi + printf '%s\n' "$destination" +} +copy_sdk() +{ + local name="$1" headers="$2" libraries="$3" + [[ -d "$headers" && -d "$libraries" ]] || { + echo "Invalid $name SDK paths: $headers, $libraries" >&2 + exit 1 + } + mkdir -p "$deps/$name/include" "$deps/$name/lib" + cp -a "$headers/." "$deps/$name/include/" + cp -a "$libraries/." "$deps/$name/lib/" +} +sdk="${ROCKCHIP_SDK:-${RK_SDK_PROJECT:+$RK_SDK_PROJECT/third_party}}" +mpp="${MPP_ROOT:-${RK_SDK_PROJECT:+$RK_SDK_PROJECT/build-rk3576/deps/mpp}}" +toolchain=() +[[ -z "${NVR_TOOLCHAIN_FILE:-}" ]] || toolchain=("-DCMAKE_TOOLCHAIN_FILE=$NVR_TOOLCHAIN_FILE") +if [[ ! -f "$deps/rga/include/im2d.hpp" || ! -e "$deps/rga/lib/librga.so" ]]; then + if [[ -n "$sdk" ]]; then + copy_sdk rga "$sdk/rga/include" "$sdk/rga/lib" + else + source=$(fetch_source rga airockchip/librga 2b32edcb97b601b25683e2941d888c8515da6d55) + copy_sdk rga "$source/include" "$source/libs/Linux/gcc-aarch64" + fi +fi +if [[ ! -f "$deps/rknn/include/rknn_api.h" || ! -e "$deps/rknn/lib/librknnrt.so" ]]; then + if [[ -n "$sdk" ]]; then + copy_sdk rknn "$sdk/rknn/include" "$sdk/rknn/lib" + else + source=$(fetch_source rknn airockchip/rknn-toolkit2 42aa1d426c0a9e0869b6374edba009f7208a1926) + runtime="$source/rknpu2/runtime/Linux/librknn_api" + copy_sdk rknn "$runtime/include" "$runtime/aarch64" + fi +fi +if [[ ! -f "$deps/mpp/include/rockchip/rk_mpi.h" && ! -f "$deps/mpp/include/rk_mpi.h" ]] \ + || [[ ! -e "$deps/mpp/lib/librockchip_mpp.so" ]]; then + if [[ -n "$mpp" ]]; then + copy_sdk mpp "$mpp/include" "$mpp/lib" + else + source=$(fetch_source mpp rockchip-linux/mpp 7916c3cc000fefc0bdf96636b7d54fe174724f44) + cmake -S "$source" -B "$cache/mpp-build" "${toolchain[@]}" -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_INSTALL_PREFIX="$deps/mpp" \ + -DCMAKE_INSTALL_LIBDIR=lib -DRKPLATFORM=ON -DHAVE_DRM=ON -DBUILD_TEST=OFF + cmake --build "$cache/mpp-build" --parallel "${JOBS:-4}" + cmake --install "$cache/mpp-build" + fi +fi + +rm -rf "$root"