Update for v1.0alpha1

.gitignore

@@ -0,0 +1,6 @@
+.vs
+.vscode
+
+out
+build*
+CMakeSettings.json

.gitlab-ci.yml

@@ -0,0 +1,319 @@
+# This file is a template, and might need editing before it works on your project.
+# This is a sample GitLab CI/CD configuration file that should run without any modifications.
+# It demonstrates a basic 3 stage CI/CD pipeline. Instead of real tests or scripts,
+# it uses echo commands to simulate the pipeline execution.
+#
+# A pipeline is composed of independent jobs that run scripts, grouped into stages.
+# Stages run in sequential order, but jobs within stages run in parallel.
+#
+# For more information, see: https://docs.gitlab.com/ee/ci/yaml/index.html#stages
+#
+# You can copy and paste this template into a new `.gitlab-ci.yml` file.
+# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword.
+#
+# To contribute improvements to CI/CD templates, please follow the Development guide at:
+# https://docs.gitlab.com/ee/development/cicd/templates.html
+# This specific template is located at:
+# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml
+
+stages:             # List of stages for jobs, and their order of execution
+  - build
+  - test
+  - package
+  - upload
+  - deploy
+
+image: harbor.bianbu.xyz/spacemit-ai/bianbu-ai-support:v1.1.1
+
+.only-defaults: &only-defaults
+  only:
+    - tags
+    - main
+    - cicd
+    - merge_requests
+    - schedules
+
+.rule-upload: &rule-upload
+  rules:
+    - if: '$CI_COMMIT_TAG =~ /^bianbu-23.10\/.*$/'
+      when: on_success
+
+
+variables:
+  SDK: "/opt"
+  DATA: "$SDK/bianbu-ai-support/data"
+  CROSS_TOOL: "$SDK/spacemit-gcc/bin/riscv64-unknown-linux-gnu-"
+  SYSROOT: "$SDK/spacemit-gcc/sysroot"
+  QEMU_CMD: "$SDK/qemu/bin/qemu-riscv64 -L $SYSROOT"
+  ORT_HOME_X86: "/home/workspace/onnxruntime-linux-x64-1.15.1"
+  CI_NEXUS_URL: "https://nexus.bianbu.xyz/service/rest/repository/browse/bianbu-ai/onnxruntime/"
+  CI_INSTALL_OPTION: "--with-demo --with-ort"
+  KUBERNETES_MEMORY_LIMIT: "6Gi"
+  KUBERNETES_MEMORY_REQUEST: "6Gi"
+
+build-job-x86_64:   # This job runs in the build stage, which runs first.
+  stage: build
+  variables:
+    BUILD_DIR: "build.x86_64"
+    ORT_HOME: "${ORT_HOME_X86}"
+    INSTALL_PREFIX: "bianbu-ai-support.x86_64"
+  before_script:
+    - echo "Before scripts ..."
+  script:
+    - echo "Compiling the code ..."
+    - |
+      mkdir ${BUILD_DIR}
+      pushd ${BUILD_DIR}
+      cmake .. -DORT_HOME=${ORT_HOME} -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} -DDEMO=ON
+      make install -j`nproc` VERBOSE=1
+      popd
+    # copy demo and data
+    - if [[ "${CI_INSTALL_OPTION}" =~ "--with-demo" ]]; then cp -rdf demo ${BUILD_DIR}/${INSTALL_PREFIX} && cp -rdf data ${BUILD_DIR}/${INSTALL_PREFIX}/demo; fi
+    # copy onnxruntime
+    - if [[ "${CI_INSTALL_OPTION}" =~ "--with-ort" ]]; then mkdir -p ${BUILD_DIR}/${INSTALL_PREFIX}/lib/3rdparty/onnxruntime && cp -rdf ${ORT_HOME}/* ${BUILD_DIR}/${INSTALL_PREFIX}/lib/3rdparty/onnxruntime; fi
+    - |
+      pushd demo
+      mkdir ${BUILD_DIR}
+      pushd ${BUILD_DIR}
+      cmake .. -DBIANBUAI_HOME=$(pwd)/../../${BUILD_DIR}/${INSTALL_PREFIX} -DORT_HOME=${ORT_HOME}
+      make -j`nproc` VERBOSE=1
+      popd
+    - echo "Compile complete."
+  artifacts:
+    paths:
+      - ${BUILD_DIR}
+      - demo/${BUILD_DIR}
+  <<: *only-defaults
+
+build-job-riscv64:  # This job runs in the build stage, which runs second.
+  stage: build
+  variables:
+    BUILD_DIR: "build.riscv64"
+    ORT_HOME: "${CI_PROJECT_DIR}/spacemit-ort"
+    # TODO: fix opencv.v4.6.0 output precision error
+    OPENCV_SHARED_DIR: "$SDK/bianbu-ai-support/3rdparty/opencv.v4.7.0.shared" # v4.6.0.shared.qt5
+    OPENCV_STATIC_DIR: "$SDK/bianbu-ai-support/3rdparty/opencv.v4.7.0.static"
+    INSTALL_PREFIX: "bianbu-ai-support.riscv64"
+  before_script:
+    - echo "Before scripts ..."
+    #- set -x
+    #- echo "CI_BUILDS_DIR  ${CI_BUILDS_DIR}"
+    #- echo "CI_PROJECT_DIR ${CI_PROJECT_DIR}"
+    - echo "Downloading latest spacemit-ort ..."
+    - wget $(curl -X GET ${CI_NEXUS_URL} | grep -oP 'https:[^>]*.rv64.v[\d\.]*tar.gz' | tail -n 1) -O spacemit-ort.latest.tar.gz --no-check-certificate
+    - |
+      mkdir spacemit-ort
+      tar xzf spacemit-ort.latest.tar.gz -C spacemit-ort --strip-components 1
+      tree -L 3 .
+  script:
+    - echo "Compiling the code ..."
+    # Note: To use opencv static libraries, one'd better set OpenCV_DIR variable instead of OPENCV_INC and OPENCV_LIB variables.
+    - |
+      mkdir ${BUILD_DIR}
+      pushd ${BUILD_DIR}
+      cmake .. -DORT_HOME=${ORT_HOME} -DOpenCV_DIR=${OPENCV_STATIC_DIR}/lib/cmake/opencv4 -DCMAKE_C_COMPILER=${CROSS_TOOL}gcc -DCMAKE_CXX_COMPILER=${CROSS_TOOL}g++ -DCMAKE_SYSROOT=${SYSROOT} -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} -DDEMO=ON
+      make install -j`nproc` VERBOSE=1
+      popd
+    # copy demo and data
+    - if [[ "${CI_INSTALL_OPTION}" =~ "--with-demo" ]]; then cp -rdf demo ${BUILD_DIR}/${INSTALL_PREFIX} && cp -rdf data ${BUILD_DIR}/${INSTALL_PREFIX}/demo; fi
+    # copy onnxruntime
+    - if [[ "${CI_INSTALL_OPTION}" =~ "--with-ort" ]]; then mkdir -p ${BUILD_DIR}/${INSTALL_PREFIX}/lib/3rdparty/onnxruntime && cp -rdf ${ORT_HOME}/* ${BUILD_DIR}/${INSTALL_PREFIX}/lib/3rdparty/onnxruntime; fi
+  #  - mkdir ${BUILD_DIR} && pushd ${BUILD_DIR} && cmake .. -DORT_HOME=${ORT_HOME} -DOPENCV_INC=${OPENCV_SHARED_DIR}/include/opencv4 -DOPENCV_LIB=${OPENCV_SHARED_DIR}/lib -DCMAKE_C_COMPILER=${CROSS_TOOL}gcc -DCMAKE_CXX_COMPILER=${CROSS_TOOL}g++ -DCMAKE_SYSROOT=${SYSROOT} -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} -DDEMO=ON -DUSE_OPENCV_SHARED_LIBS=ON && make install -j`nproc` VERBOSE=1 && popd
+    - |
+      pushd demo
+      mkdir ${BUILD_DIR}
+      pushd ${BUILD_DIR}
+      cmake .. -DBIANBUAI_HOME=$(pwd)/../../${BUILD_DIR}/${INSTALL_PREFIX} -DORT_HOME=${ORT_HOME} -DOPENCV_INC=${OPENCV_SHARED_DIR}/include/opencv4 -DOPENCV_LIB=${OPENCV_SHARED_DIR}/lib -DCMAKE_C_COMPILER=${CROSS_TOOL}gcc -DCMAKE_CXX_COMPILER=${CROSS_TOOL}g++ -DCMAKE_SYSROOT=${SYSROOT}
+      make -j`nproc` VERBOSE=1
+      popd
+  #  - echo "Compile complete."
+  artifacts:
+    paths:
+      - ${BUILD_DIR}
+      - spacemit-ort
+      - demo/${BUILD_DIR}
+  <<: *only-defaults
+
+build-job-riscv64-deb:
+  stage: build
+  image: harbor.bianbu.xyz/spacemit-ai/bianbu-ai-support:debian
+  dependencies: []
+  script:
+    - echo "Building debian package for tag $CI_COMMIT_TAG"
+    - |
+      set -x
+      mount -o bind $CI_PROJECT_DIR/.. /mnt/
+      schroot -r -c persistent-session --directory /mnt/ai/support
+      apt build-dep -y $(pwd)
+      dpkg-buildpackage -us -b -uc --no-pre-clean --no-post-clean
+      exit
+      mkdir bianbu-dev
+      mv ../bianbu-ai-support-* bianbu-dev
+      mv ../bianbu-ai-support_* bianbu-dev
+  artifacts:
+    paths:
+      - bianbu-dev
+  only:
+    - deb
+
+
+x86_64-test-job:    # This job runs in the test stage.
+  stage: test       # It only starts when the job in the build stage completes successfully.
+  dependencies:
+    - build-job-x86_64
+  script:
+    - echo "Running unit tests ... (This will take several seconds.)"
+    - export LD_LIBRARY_PATH=${ORT_HOME_X86}/lib:$LD_LIBRARY_PATH
+    - demo/build.x86_64/classification_demo $DATA/models/squeezenet1.1-7.onnx $DATA/labels/synset.txt $DATA/imgs/dog.jpg
+    - demo/build.x86_64/detection_demo $DATA/models/nanodet-plus-m_320.onnx $DATA/labels/coco.txt $DATA/imgs/person0.jpg result0.jpg
+    - $([[ "3aeb3a152ce55a94c0c7bac303534b8d" == "$(md5sum result0.jpg | awk '{print $1}')" ]])
+    - echo "Running x86_64 tests done!"
+  <<: *only-defaults
+
+riscv64-test-job:   # This job runs in the test stage.
+  stage: test       # It only starts when the job in the build stage completes successfully.
+  dependencies:
+    - build-job-riscv64
+  script:
+    - echo "Running unit tests ... (This will take several seconds.)"
+    - ${QEMU_CMD} demo/build.riscv64/classification_demo $DATA/models/squeezenet1.1-7.onnx $DATA/labels/synset.txt $DATA/imgs/dog.jpg
+    - ${QEMU_CMD} demo/build.riscv64/detection_demo $DATA/models/nanodet-plus-m_320.onnx $DATA/labels/coco.txt $DATA/imgs/person0.jpg result0.jpg
+    - $([[ "3aeb3a152ce55a94c0c7bac303534b8d" == "$(md5sum result0.jpg | awk '{print $1}')" ]])
+    - echo "Running riscv64 tests done!"
+  <<: *only-defaults
+
+package-daily-dpkg:
+  stage: package
+  variables:
+    CI_PKG_OPTION: "--skip-py" # "--skip-ort --skip-py"
+    CI_PKG_VERSION: "1.0.7"
+  dependencies:
+    - build-job-x86_64
+    - build-job-riscv64
+  script:
+    #- echo $(date +"%Y%m%d_%H%M%S")
+    - mkdir -p output
+    # make deb ball
+    - |
+      bash ci/pack.sh build.x86_64/bianbu-ai-support.x86_64 ${CI_PKG_OPTION}
+      mv bianbu-ai-support-*.deb output/bianbu-ai-support_${CI_PKG_VERSION}_amd64.deb
+    - |
+      bash ci/pack.sh build.riscv64/bianbu-ai-support.riscv64 ${CI_PKG_OPTION}
+      mv bianbu-ai-support-*.deb output/bianbu-ai-support_${CI_PKG_VERSION}_riscv64.deb
+    #- rm -rf build.x86_64 build.riscv64 demo
+  artifacts:
+    paths:
+      - output/bianbu-ai-support_*.deb
+  <<: *only-defaults
+
+package-daily-prebuilt:
+  stage: package
+  dependencies:
+    - build-job-x86_64
+    - build-job-riscv64
+  script:
+    - mkdir -p output
+    # make tar ball
+    - |
+      pushd build.x86_64
+      tar czf ../output/bianbu-ai-support.x86_64.$(date +"%Y%m%d_%H%M%S").tar.gz bianbu-ai-support.x86_64
+      popd
+    - |
+      pushd build.riscv64
+      tar czf ../output/bianbu-ai-support.riscv64.$(date +"%Y%m%d_%H%M%S").tar.gz bianbu-ai-support.riscv64
+      popd
+    #- rm -rf build.x86_64 build.riscv64 demo
+  artifacts:
+    paths:
+      - output/bianbu-ai-support.*.tar.gz
+  <<: *only-defaults
+
+
+upload-archive-nexus:
+  stage: upload
+  dependencies:
+    - build-job-x86_64
+    - build-job-riscv64
+  script:
+    - tag=v${CI_COMMIT_REF_NAME#*/}
+    - |
+      pushd build.x86_64
+      tar czf bianbu-ai-support.x86_64.${tag}.tar.gz bianbu-ai-support.x86_64
+      curl -k -u $NEXUS_USERNAME:$NEXUS_PASSWORD --upload-file bianbu-ai-support.x86_64.${tag}.tar.gz https://nexus.bianbu.xyz/repository/bianbu-ai/support-library/bianbu-ai-support.x86_64.${tag}.tar.gz
+      popd
+    - |
+      pushd build.riscv64
+      tar czf bianbu-ai-support.riscv64.${tag}.tar.gz bianbu-ai-support.riscv64
+      curl -k -u $NEXUS_USERNAME:$NEXUS_PASSWORD --upload-file bianbu-ai-support.riscv64.${tag}.tar.gz https://nexus.bianbu.xyz/repository/bianbu-ai/support-library/bianbu-ai-support.riscv64.${tag}.tar.gz
+      popd
+  <<: *rule-upload
+
+upload-archive-bianbu:
+  stage: upload
+  image: harbor.bianbu.xyz/gitlab/ci-pack
+  dependencies: []
+  script:
+    # TODO: replace by shell script
+    - echo "Building for tag $CI_COMMIT_TAG"
+    - eval $(ssh-agent -s)
+    - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
+    - mkdir -p ~/.ssh
+    - chmod 700 ~/.ssh
+    - ssh-keyscan gitlab.dc.com >> ~/.ssh/known_hosts
+    - chmod 644 ~/.ssh/known_hosts
+    - git config --global user.email "bianbu-ci@spacemit.com"
+    - git config --global user.name "bianbu-ci"
+    # prepare bianbu-dev toolkits
+    - cd ..
+    - git clone git@gitlab.dc.com:bianbu/bianbu-devscripts.git
+    - export PATH=$(pwd)/bianbu-devscripts:$PATH
+    # prepare source code
+    - cd $CI_PROJECT_DIR
+    - git checkout ${CI_COMMIT_TAG%%/*}  
+    - bianbu-pkg -u local -w . -t $CI_COMMIT_TAG
+    - changes_file=$(find ../ -maxdepth 1 -type f -name "*.changes" | head -n 1)
+    # upload
+    - ssh-keyscan reprepro-headless-service.buildsystem.svc.cluster.local >> ~/.ssh/known_hosts
+    - bianbu-dev upload $changes_file # --suite mantic-spacemit
+    # change proxy as ssh
+    - git remote set-url origin git@gitlab.dc.com:$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME.git
+    - git push origin --all
+    # status
+    - echo "Bianbu devscripts successfully packaged."
+  <<: *rule-upload
+
+
+lint-test-job:   # This job also runs in the test stage.
+  stage: test    # It can run at the same time as unit-test-job (in parallel).
+  dependencies: []
+  script:
+    - echo "Linting code... This will take about 2 seconds."
+    - sleep 2
+    - echo "No lint issues found."
+  <<: *only-defaults
+
+deploy-job:      # This job runs in the deploy stage.
+  stage: deploy  # It only runs when *both* jobs in the test stage complete successfully.
+  environment: production
+  dependencies: []
+  script:
+    - echo "Deploying application..."
+    - echo "Application successfully deployed."
+  only:
+    - tags
+
+
+# SAST customization: https://docs.gitlab.com/ee/user/application_security/sast/#customizing-the-sast-settings
+# Secret Detection customization: https://docs.gitlab.com/ee/user/application_security/secret_detection/#customizing-settings
+# Dependency Scanning customization: https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#customizing-the-dependency-scanning-settings
+# Container Scanning customization: https://docs.gitlab.com/ee/user/application_security/container_scanning/#customizing-the-container-scanning-settings
+# Note that environment variables can be set in several places
+# See https://docs.gitlab.com/ee/ci/variables/#cicd-variable-precedence
+
+sast:
+  stage: test
+  dependencies: []
+
+include:
+  - template: Security/SAST.gitlab-ci.yml
+  - template: Security/Secret-Detection.gitlab-ci.yml

CMakeLists.txt

@@ -0,0 +1,21 @@
+cmake_minimum_required(VERSION 3.10)
+ 
+project(bianbuai)
+add_subdirectory(${CMAKE_SOURCE_DIR}/src)
+
+option(DEMO "option for Demo" OFF)
+if (DEMO)
+  set(BIANBUAI_HOME ${CMAKE_SOURCE_DIR})  # useless but necessary
+  add_subdirectory(${CMAKE_SOURCE_DIR}/demo)
+  install(DIRECTORY ${CMAKE_SOURCE_DIR}/rootfs/usr/bin DESTINATION .)
+  install(DIRECTORY ${CMAKE_SOURCE_DIR}/rootfs/usr/share DESTINATION .)
+  install(DIRECTORY ${CMAKE_SOURCE_DIR}/rootfs/etc DESTINATION ..)
+endif()
+# always install demo project with test data
+#install(DIRECTORY ${CMAKE_SOURCE_DIR}/demo DESTINATION .)
+#install(DIRECTORY ${CMAKE_SOURCE_DIR}/data DESTINATION demo)
+
+option(TEST "option for Test" ON)
+if (TEST)
+  add_subdirectory(${CMAKE_SOURCE_DIR}/src/tests)
+endif()

README.md

@@ -0,0 +1,51 @@
+# support
+
+
+## Getting started
+
+### install dependence
+gcc version: 14
+
+prepare opencv (at least version 4.2.0)
+```bash
+sudo apt-get install libopencv-dev
+```
+prepare onnxruntime
+
+visit [onnxruntime release](https://github.com/microsoft/onnxruntime/releases/tag/v1.15.1) and download proper version.
+
+
+### build demo with bash followed
+
+```bash
+ORT_HOME=${PATH_TO_ONNXRUNTIME}
+# Note: Add the installation prefix of "OpenCV" to CMAKE_PREFIX_PATH or set
+# "OpenCV_DIR" to a directory containing one of the following names:
+#   OpenCVConfig.cmake
+#   opencv-config.cmake
+OPENCV_DIR=${PATH_TO_OPENCV_CMAKE_DIR}
+
+mkdir build && pushd build
+cmake .. -DORT_HOME=${ORT_HOME} -DOpenCV_DIR=${OPENCV_DIR} -DCMAKE_BUILD_TYPE=Debug -DTEST=OFF -DDEMO=ON
+make install -j`nproc`
+popd
+
+# Or with cross compiler:
+CROSS_TOOL=${PATH_TO_COMPILER_PREFIX}-
+SYSROOT=${PATH_TO_SYSROOT}
+cmake .. -DORT_HOME=${ORT_HOME} -DOpenCV_DIR=${OPENCV_DIR} \
+   -DCMAKE_C_COMPILER=${CROSS_TOOL}gcc -DCMAKE_CXX_COMPILER=${CROSS_TOOL}g++ -DCMAKE_SYSROOT=${SYSROOT}
+```
+
+### run demo
+
+```bash
+./classification_demo <modelFilepath> <labelFilepath> <imageFilepath>
+./detection_stream_demo <configFilepath> <input> <inputType>
+./detection_demo <modelFilepath> <labelFilepath> <imageFilepath> <saveImgpath>
+or 
+./detection_demo <configFilepath> <imageFilepath> <saveImgpath>
+./detection_video_demo <configFilepath> <videoFilepath> <saveFilepath>
+./estimation_demo <detConfigFilepath> <poseConfigFilepath> <imageFilepath> <saveImgpath>
+./tracker_stream_demo <detConfigFilepath> <poseConfigFilepath> <input> <inputType>
+```

VERSION_NUMBER

@@ -0,0 +1 @@
+1.0.8

ci/README.md

@@ -0,0 +1,84 @@
+## CI Variables
+
+| Variable Name | Default Value | Description |
+| :--: | :--: | :--: |
+| CI_PKG_OPTION | --skip-py | options during ci `package` stage, choose values from [--skip-deb \| --skip-ort \| --skip-py \| --debug] |
+
+## Bianbu DevScripts
+
+* step by step
+
+```bash
+# preparation: download bianbu development toolkits
+git clone -b main git@gitlab.dc.com:bianbu/bianbu-devscripts.git && cd bianbu-devscripts
+# preparation: install dependencies
+apt update && apt install sudo vim-common -y
+bash install-scripts-depends.sh
+# preparation: install ca-certificates
+apt-get -y install ca-certificates wget && wget https://nexus.bianbu.xyz/repository/software/dc.com-CA-Root-Base64.crt -O /usr/local/share/ca-certificates/dc.com-CA-Root-Base64.crt --no-check-certificate && update-ca-certificates
+# preparation: add bianbu source list
+bianbu-dev chroot
+# add mantic-spacemit(similar for mantic-porting, mantic-customization)
+cat <<EOF >>/etc/apt/sources.list.d/bianbu.list
+# mantic-spacemit
+deb [signed-by=/usr/share/keyrings/bianbu-archive-keyring.gpg] https://archive.bianbu.xyz/bianbu-ports/ mantic-spacemit main universe multiverse restricted
+deb-src [signed-by=/usr/share/keyrings/bianbu-archive-keyring.gpg] https://archive.bianbu.xyz/bianbu-ports/ mantic-spacemit main universe multiverse restricted
+EOF
+apt update
+exit # quit chroot
+
+# quick start: check toolkit version
+PATH=$(pwd):$PATH
+bianbu-dev -v
+bianbu-pkg -h
+# quick start: default settings
+bianbu-dev set-default-dist bianbu-23.10
+bianbu-dev set-default-env bianbu-scripts-env-1.8.1
+# quick start: install docker container for development(optional)
+bianbu-dev install
+
+# -------------------------------------------------------------
+
+# quick import: example with gnome-shell
+# Note: To import-dsc, plz switch to any non-git repository directory
+apt source --download-only gnome-shell # gnome-shell_*-0ubuntu1.debian.tar.xz, gnome-shell_*-0ubuntu1.dsc, gnome-shell_*.orig.tar.xz
+bianbu-dev import-dsc gnome-shell_*-0ubuntu1.dsc
+# quick glance
+tree gnome-shell -L 1
+
+# quick import: with bianbu-ai-support
+cd bianbu-ai-support && git checkout main
+# Note: version number must start with digit(required by dpkg-buildpackage)
+bianbu-dev import . --suite mantic-spacemit --new-version 1.0.1
+
+# -------------------------------------------------------------
+
+# do sth ...
+git checkout -b bianbu-23.10 # switch to bianbu-23.10
+bianbu-dev tag
+bianbu-dev pack -a  # same as `bianbu-dev pack && bianbu-dev build` (pack 1st, then build)
+
+# -------------------------------------------------------------
+
+# enter develop env
+bianbu-dev chroot   # apt install onnxruntime, libopencv-dev
+
+# build package with dpkg(Note: install dependencies 1st plz, e.g. onnxruntime, opencv)
+apt-get build-dep -y $(pwd) # run in bianbu-ai-support directory
+dpkg-buildpackage -us -b -uc --no-pre-clean --no-post-clean # -tc
+# ..
+# |-- bianbu-ai-support-dbgsym_1.0.1_amd64.ddeb
+# |-- bianbu-ai-support_1.0.1_amd64.buildinfo
+# |-- bianbu-ai-support_1.0.1_amd64.changes
+# |-- bianbu-ai-support_1.0.1_amd64.deb
+
+# -------------------------------------------------------------
+
+# check https://archive.bianbu.xyz/bianbu-ports/logs/ for upload events
+bianbu-dev upload onnxruntime_1.15.1_amd64.deb --suite mantic-porting
+bianbu-dev upload onnxruntime_1.15.1_riscv64.deb --suite mantic-spacemit
+
+apt update
+apt show onnxruntime
+apt install onnxruntime
+```

ci/docker.ci.md

@@ -0,0 +1,84 @@
+## Auto Construction For Debian
+
+* create new container
+
+```bash
+IMAGE=harbor.bianbu.xyz/dev/bianbu-scripts-env:1.8.1
+NAME=xxx
+docker run --restart=always --net=host --privileged -itd --name $NAME -v $HOME:/mnt $IMAGE /bin/bash
+```
+
+* enter container && apply patch
+
+```bash
+docker exec -it $NAME /bin/bash
+# download qemu patch for bianbu-scripts-env:1.8.1(x86_64)
+wget https://nexus.bianbu.xyz/repository/software/qemu-user-static_8.0.4%2Bdfsg-1ubuntu3.23.10.1_amd64.deb
+dpkg -i qemu-user-static_8.0.4+dfsg-1ubuntu3.23.10.1_amd64.deb
+# re-register qemu-riscv64-static
+update-binfmts --disable qemu-riscv64-static
+update-binfmts --enable qemu-riscv64-static
+# quick glance(`mount binfmt_misc -t binfmt_misc /proc/sys/fs/binfmt_misc` if necessary)
+cat /proc/sys/fs/binfmt_misc/qemu-riscv64
+exit
+
+# enter with riscv64 rootfs
+docker exec -it $NAME schroot -r -c persistent-session
+```
+
+* update container
+
+```bash
+cat <<EOF >>/etc/apt/sources.list.d/bianbu.list
+# mantic-spacemit
+deb [signed-by=/usr/share/keyrings/bianbu-archive-keyring.gpg] https://archive.bianbu.xyz/bianbu-ports/ mantic-spacemit main universe multiverse restricted
+deb-src [signed-by=/usr/share/keyrings/bianbu-archive-keyring.gpg] https://archive.bianbu.xyz/bianbu-ports/ mantic-spacemit main universe multiverse restricted
+# mantic-porting
+deb [signed-by=/usr/share/keyrings/bianbu-archive-keyring.gpg] https://archive.bianbu.xyz/bianbu-ports/ mantic-porting main universe multiverse restricted
+deb-src [signed-by=/usr/share/keyrings/bianbu-archive-keyring.gpg] https://archive.bianbu.xyz/bianbu-ports/ mantic-porting main universe multiverse restricted
+# mantic-customization
+#deb [signed-by=/usr/share/keyrings/bianbu-archive-keyring.gpg] https://archive.bianbu.xyz/bianbu-ports/ mantic-customization main universe multiverse restricted
+#deb-src [signed-by=/usr/share/keyrings/bianbu-archive-keyring.gpg] https://archive.bianbu.xyz/bianbu-ports/ mantic-customization main universe multiverse restricted
+EOF
+
+apt update
+apt install -y debhelper cmake libopencv-dev onnxruntime
+
+# update /opt/smartentry/HEAD/pre-run* for smartentry(/sbin/smartentry.sh)
+vi /opt/smartentry/HEAD/pre-run
+# ...
+```
+
+* save/export container
+
+```bash
+docker commit $NAME <new-image>
+```
+
+* /opt/smartentry/HEAD/pre-run
+
+```bash
+# 此脚本会在docker run 时执行
+# 可以在这里做一些修改，补丁。执行完才会执行run脚本中的内容
+if [ -d "/var/lib/schroot/unpack/persistent-session" ]; then
+    echo "persistent-session exists."
+    schroot --recover-session -c persistent-session
+else
+    echo "persistent-session does not exist."
+    # start a new session with specified chroot named as persistent-session
+    schroot -b -c mantic-riscv64-sbuild -n persistent-session
+fi
+echo 'docker inited' > /root/.smartentry_status
+mount -o bind /mnt /var/run/schroot/mount/persistent-session/mnt
+```
+
+* smoke gitlab ci test
+
+```bash
+# run specified session
+cp -rdf $CI_PROJECT_DIR/.. /mnt/
+schroot -r -c persistent-session --directory /mnt/ai/support
+apt build-dep -y $(pwd)
+dpkg-buildpackage -us -b -uc --no-pre-clean --no-post-clean
+exit
+```

ci/dockerfile.riscv64

@@ -0,0 +1,9 @@
+FROM harbor.bianbu.xyz/spacemit-ai/bianbu-ai-support:v1.1.0
+
+# install spacemit-ai-sdk
+COPY spacemit-ai-sdk /opt/
+
+# Build/Update bianbu-ai-support:v1.0.1 image:
+# $ docker build -t bianbu-ai-support:v1.0.1 -f dockerfile.riscv64 .
+# $ docker tag bianbu-ai-support:v1.0.1 harbor.bianbu.xyz/spacemit-ai/bianbu-ai-support:v1.0.1
+# $ docker push harbor.bianbu.xyz/spacemit-ai/bianbu-ai-support:v1.0.1

ci/dockerfile.x86_64

@@ -0,0 +1,29 @@
+FROM ubuntu:18.04
+LABEL maintainer='ai_support <ai_support@spacemit.com>'
+
+WORKDIR /home/workspace
+ENV LANG=C.UTF-8 TZ=Asia/Shanghai
+
+# update source
+RUN sed -i'' 's/archive\.ubuntu\.com/us\.archive\.ubuntu\.com/' /etc/apt/sources.list
+RUN apt -y update
+
+# install tools for support library and ai toolkit cicd
+RUN DEBIAN_FRONTEND=noninteractive apt install -y \
+    cmake build-essential gcc g++ libglib2.0-0 \
+    libopencv-dev python3 python3-pip \
+    curl wget ca-certificates tree --fix-missing
+
+# install onnxruntime
+COPY onnxruntime-linux-x64-*.tgz /tmp/
+RUN tar xzf /tmp/onnxruntime-linux-x64-1.15.1.tgz -C . && rm /tmp/onnxruntime-linux-x64-1.15.1.tgz
+RUN tar xzf /tmp/onnxruntime-linux-x64-1.16.3.tgz -C . && rm /tmp/onnxruntime-linux-x64-1.16.3.tgz
+
+# install spacemit ca-certificates
+RUN wget https://nexus.bianbu.xyz/repository/software/dc.com-CA-Root-Base64.crt -O /usr/local/share/ca-certificates/dc.com-CA-Root-Base64.crt --no-check-certificate && update-ca-certificates
+
+# Build bianbu-ai-support:v1.0.0 image:
+# $ docker build -t bianbu-ai-support:v1.0.0 -f dockerfile.x86_64 .
+# $ docker tag bianbu-ai-support:v1.0.0 harbor.bianbu.xyz/spacemit-ai/bianbu-ai-support:v1.0.0
+# $ docker push harbor.bianbu.xyz/spacemit-ai/bianbu-ai-support:v1.0.0
+# v1.0.0: digest: sha256:3a8d345662fc0e264de5d630335e778e4ce86c8b79493bb84d63fc11d5aa2b7c size: 1790

ci/pack.sh

@@ -0,0 +1,93 @@
+#!/bin/bash
+#
+# Usage:
+# $ bash pack.sh <local_install> [--skip-deb|--skip-ort|--skip-py|--debug]
+
+set -e
+
+[[ ${PY_CMD} ]] || PY_CMD=python3
+REPO_DIR=$(readlink -f "$(dirname $0)/..")
+
+# check dpkg/python3/...
+function check() {
+  set +e
+  which $1 1>/dev/null 2>&1
+  if [[ 0 -ne $? ]]; then
+    echo -e "\033[5;31m[ERROR] Plz double check if '$1' is available or not!\033[0m"
+    exit 1
+  fi
+  set -e
+}
+
+# make debian package
+function make_deb_via_dpkg() {
+  check dpkg
+
+  PKG_NAME=bianbu-ai-support
+  PKG_VER=$(cat ${REPO_DIR}/VERSION_NUMBER)
+  PKG_DIR=${REPO_DIR}/${PKG_NAME}
+  PKG_DESC="bianbu ai support library"
+
+  INSTALL_LOCAL=$(readlink -f $1)
+  # check existance
+  if [[ ! -d $INSTALL_LOCAL ]]; then
+    echo -e "\033[0;31m[ERROR] local installed directory'$INSTALL_LOCAL' doesn't exist!\033[0m"
+    exit 2
+  fi
+
+  # get architecture
+  ARCH=${INSTALL_LOCAL##*.}
+  # update architecture for Debian && Ubuntu
+  if [[ "$ARCH" == "x86_64" ]]; then
+    ARCH="amd64"
+  fi
+
+  rm -rf ${PKG_DIR}
+  # create debian control
+  mkdir -p ${PKG_DIR}/DEBIAN
+  cat <<EOF > ${PKG_DIR}/DEBIAN/control
+Package: ${PKG_NAME}
+Version: ${PKG_VER}
+Section: Utils
+Priority: optional
+Architecture: ${ARCH}
+Build-Depends: cmake, debhelper-compat (= 12), onnxruntime, libopencv-dev
+Depends: onnxruntime, libopencv-dev
+Maintainer: -
+Description: ${PKG_DESC}.
+EOF
+
+  # copy bin, lib, include, etc.
+  mkdir -p ${PKG_DIR}/usr/
+  if [[ "$@" =~ "skip-ort" ]]; then
+    :
+  else
+    cp -rd ${INSTALL_LOCAL}/lib/3rdparty/onnxruntime/* ${PKG_DIR}/usr/
+  fi
+  cp -rdf ${INSTALL_LOCAL}/* ${PKG_DIR}/usr/
+  cp -rdf ${INSTALL_LOCAL}/../etc ${PKG_DIR}/
+  # post process
+  rm -rdf ${PKG_DIR}/usr/lib/3rdparty ${PKG_DIR}/usr/demo
+
+  # create debian package
+  dpkg -b ${PKG_DIR} ${PKG_NAME}-${PKG_VER}.deb
+}
+[[ "$@" =~ "skip-deb" ]] || make_deb_via_dpkg $@
+
+# make python package
+function make_python_dist_pkg() {
+  check ${PY_CMD}
+
+  # prepare temp python source directory to avoid inplace constraction
+  TMP_DIR=${REPO_DIR}/tmp
+  PY_DIR=${REPO_DIR}/python
+  rm -rf ${TMP_DIR} && cp -rdf ${PY_DIR} ${TMP_DIR}
+
+  # make python dist package
+  pushd ${TMP_DIR}
+  ${PY_CMD} ${TMP_DIR}/setup.py sdist --dist-dir=${REPO_DIR}/dist
+  ${PY_CMD} ${TMP_DIR}/setup.py bdist_wheel --dist-dir=${REPO_DIR}/dist
+  popd
+  [[ "$@" =~ "debug" ]] || rm -rf pushd ${TMP_DIR}
+}
+[[ "$@" =~ "skip-py" ]] || make_python_dist_pkg $@

ci/test.sh

@@ -0,0 +1,100 @@
+#!/bin/bash
+#
+# Author: hongjie.qin@spacemit.com
+# Brief:  test demos
+#
+# <test_package>
+# ├── bianbu-ai-support.riscv64
+# │   ├── bin
+# │   ├── include
+# │   │   └── bianbuai
+# │   └── lib
+# │       ├── 3rdparty
+# │       │   ├── onnxruntime -> spacemit-ort
+# │       │   ├── opencv -> opencv.v4.6.0.qt5
+# │       │   ├── opencv.v4.6.0.gtk2
+# │       │   ├── opencv.v4.6.0.qt5
+# │       │   └── spacemit-ort
+# │       ├── libbianbuai.so -> libbianbuai.so.1
+# │       ├── libbianbuai.so.1 -> libbianbuai.so.1.0.1
+# │       └── libbianbuai.so.1.0.1
+# ├── data
+# │   ├── config
+# │   │   └── nanodet.json
+# │   ├── imgs
+# │   ├── labels
+# │   ├── models
+# │   └── videos
+# │       └── test.mp4
+# └── test.sh
+
+#set -e
+#set -x
+
+function show_usage() {
+  echo "Usage:"
+  echo "  ${BASH_SOURCE[0]} --x86_64  # native test with  x86_64 outputs"
+  echo "  ${BASH_SOURCE[0]} --riscv64 # native test with riscv64 outputs"
+  echo "  ${BASH_SOURCE[0]} --qemu <path_to_sdk>  # test riscv64 outputs on x86_64 with qemu"
+}
+
+WORKSPACE=$(dirname $(readlink -f ${BASH_SOURCE[0]}))
+BIANBUAI_HOME=$WORKSPACE/bianbu-ai-support.riscv64
+
+if [[ $@ =~ "x86" ]]; then
+  BIANBUAI_HOME=$WORKSPACE/bianbu-ai-support.x86_64
+fi
+if [[ $@ =~ "--qemu" ]]; then
+  if [[ $@ =~ "x86" ]]; then
+    echo "[ERROR] invalid options" && show_usage
+    exit 0
+  fi
+  # set path for qemu and x86_64-riscv64-gcc
+  SDK=$(getopt -l --qemu -- $@ | tr -d "'") # e.g. $HOME/workspace/bianbu-ai-release/workspace
+  SDK=${SDK% --*}
+  if [[ -z "$SDK" ]]; then
+    echo "[ERROR] invalid options" && show_usage
+    exit 0
+  fi
+  SYSROOT=$SDK/spacemit-gcc/sysroot
+  QEMU_CMD="$SDK/spacemit-qemu/bin/qemu-riscv64 -L $SYSROOT"
+fi
+
+BIN_DIR=${BIANBUAI_HOME}/bin
+LIB_BIANBUAI=${BIANBUAI_HOME}/lib
+LIB_ONNXRUNTIME=${LIB_BIANBUAI}/3rdparty/onnxruntime/lib
+LIB_OPENCV=${LIB_BIANBUAI}/3rdparty/opencv/lib:${LIB_BIANBUAI}/3rdparty/opencv/lib/3rdparty
+
+task_prepare=(
+  # TODO: add md5sum checking
+  "if [[ ! -f data/models/squeezenet1.1-7.onnx ]]; then wget https://media.githubusercontent.com/media/onnx/models/main/archive/vision/classification/squeezenet/model/squeezenet1.1-7.onnx -O data/models/squeezenet1.1-7.onnx; fi"
+  "if [[ ! -f data/models/nanodet-plus-m_320.onnx ]]; then wget https://bj.bcebos.com/paddlehub/fastdeploy/nanodet-plus-m_320.onnx -O data/models/nanodet-plus-m_320.onnx; fi"
+)
+task_classification=(
+  "${BIN_DIR}/classification_demo data/models/squeezenet1.1-7.onnx data/labels/synset.txt data/imgs/dog.jpg"
+)
+task_detection=(
+  "${BIN_DIR}/detection_demo data/models/nanodet-plus-m_320.onnx data/imgs/person0.jpg result0.jpg data/labels/coco.txt"
+  "${BIN_DIR}/detection_demo data/models/nanodet-plus-m_320.int8.onnx data/imgs/person0.jpg result0.int8.jpg data/labels/coco.txt"
+  "${BIN_DIR}/detection_video_demo data/models/nanodet-plus-m_320.int8.onnx data/labels/coco.txt data/videos/test.mp4 test.avi"
+  "${BIN_DIR}/detection_stream_demo data/models/nanodet-plus-m_320.int8.onnx data/labels/coco.txt data/videos/test.mp4 video"
+  "${BIN_DIR}/detection_stream_demo data/models/nanodet-plus-m_320.int8.onnx data/labels/coco.txt 0 cameraId"
+)
+function smoke_test() {
+  # preparation(e.g. download models)
+  echo "[INFO] Prepare ..."
+  for cmd in "${task_prepare[@]}"; do eval "$cmd"; done
+  # image classification task test
+  echo "[INFO] Smoke test with image classification task ..."
+  for cmd in "${task_classification[@]}"; do
+    echo "[INFO] Run: $cmd"
+    env LD_LIBRARY_PATH=${LIB_BIANBUAI}:${LIB_ONNXRUNTIME}:${LIB_OPENCV}:$LD_LIBRARY_PATH ${QEMU_CMD} $cmd
+  done
+  # object detection task test
+  echo "[INFO] Smoke test with object detection task ..."
+  for cmd in "${task_detection[@]}"; do
+    echo "[INFO] Run: $cmd"
+    env LD_LIBRARY_PATH=${LIB_BIANBUAI}:${LIB_ONNXRUNTIME}:${LIB_OPENCV}:$LD_LIBRARY_PATH ${QEMU_CMD} $cmd
+  done
+}
+smoke_test

data/config/nanodet.json

@@ -0,0 +1,12 @@
+{
+    "name": "NanoDet",
+    "instance_name" : "object-detection-inference",
+    "model_path" : "../models/nanodet-plus-m_320.onnx",
+    "label_path" : "../labels/coco.txt",
+    "intra_threads_num" : 4,
+    "profiling_projects" : "",
+    "graph_optimization_level" : "ort_disable_all",
+    "disable_spcacemit_ep" : false,
+    "log_level" : 0,
+    "opt_model_path" : "nanodet-plus-m_320_opt.onnx"
+}

data/config/yolov4.json

@@ -0,0 +1,12 @@
+{
+    "name": "Yolov4",
+    "instance_name" : "object-detection-inference",
+    "model_path" : "../models/yolov4.onnx",
+    "label_path" : "../labels/coco.txt",
+    "intra_threads_num" : 4,
+    "profiling_projects" : "",
+    "graph_optimization_level" : "ort_disable_all",
+    "disable_spcacemit_ep" :false,
+    "log_level" : 0,
+    "opt_model_path" : "yolov4_opt.onnx"
+}

data/config/yolov6.json

@@ -0,0 +1,12 @@
+{
+    "name": "Yolov6",
+    "instance_name" : "object-detection-inference",
+    "model_path" : "../models/yolov6p5_t.onnx",
+    "label_path" : "../labels/coco.txt",
+    "intra_threads_num" : 4,
+    "profiling_projects" : "",
+    "graph_optimization_level" : "ort_disable_all",
+    "disable_spcacemit_ep" :false,
+    "log_level" : 0,
+    "opt_model_path" : "yolov6p5_t_opt.onnx"
+}

data/labels/coco.txt

@@ -0,0 +1,80 @@
+person
+bicycle
+car
+motorcycle
+airplane
+bus
+train
+truck
+boat
+traffic light
+fire hydrant
+stop sign
+parking meter
+bench
+bird
+cat
+dog
+horse
+sheep
+cow
+elephant
+bear
+zebra
+giraffe
+backpack
+umbrella
+handbag
+tie
+suitcase
+frisbee
+skis
+snowboard
+sports ball
+kite
+baseball bat
+baseball glove
+skateboard
+surfboard
+tennis racket
+bottle
+wine glass
+cup
+fork
+knife
+spoon
+bowl
+banana
+apple
+sandwich
+orange
+broccoli
+carrot
+hot dog
+pizza
+donut
+cake
+chair
+couch
+potted plant
+bed
+dining table
+toilet
+tv
+laptop
+mouse
+remote
+keyboard
+cell phone
+microwave
+oven
+toaster
+sink
+refrigerator
+book
+clock
+vase
+scissors
+teddy bear
+hair drier
+toothbrush

debian/README.Debian

@@ -0,0 +1,8 @@
+bianbu-ai-support for Debian
+
+Please edit this to provide information specific to
+this bianbu-ai-support Debian package.
+
+    (Automatically generated by debmake Version 4.3.2)
+
+ -- root <>  Thu, 04 Jan 2024 20:25:23 +0800

debian/bianbu.conf

@@ -0,0 +1,4 @@
+[package]
+upstream = False
+targetsuite = mantic-spacemit
+

debian/changelog

@@ -0,0 +1,55 @@
+bianbu-ai-support (1.0.8) mantic-spacemit; urgency=medium
+
+  [ bianbu-ci ]
+  * Sync change from bianbu-23.10/1.0.8
+
+ -- qinhongjie <hongjie.qin@spacemit.com>  Thu, 24 Feb 2024 13:16:48 +0800
+
+bianbu-ai-support (1.0.7) mantic-spacemit; urgency=medium
+
+  [ bianbu-ci ]
+  * Sync change from bianbu-23.10/1.0.7
+
+ -- qinhongjie <hongjie.qin@spacemit.com>  Thu,  1 Feb 2024 18:14:18 +0800
+
+bianbu-ai-support (1.0.6) mantic-spacemit; urgency=medium
+
+  [ bianbu-ci ]
+  * Sync change from bianbu-23.10/1.0.6
+
+ -- qinhongjie <hongjie.qin@spacemit.com>  Thu, 26 Jan 2024 12:52:18 +0800
+
+bianbu-ai-support (1.0.5) mantic-spacemit; urgency=medium
+
+  [ bianbu-ci ]
+  * Sync change from bianbu-23.10/1.0.5
+
+ -- qinhongjie <hongjie.qin@spacemit.com>  Thu, 25 Jan 2024 21:19:18 +0800
+
+bianbu-ai-support (1.0.4) mantic-spacemit; urgency=medium
+
+  [ bianbu-ci ]
+  * Sync change from bianbu-23.10/1.0.4
+
+ -- qinhongjie <hongjie.qin@spacemit.com>  Thu, 19 Jan 2024 17:26:18 +0800
+
+bianbu-ai-support (1.0.3) mantic-spacemit; urgency=medium
+
+  [ bianbu-ci ]
+  * Sync change from bianbu-23.10/1.0.3
+
+ -- qinhongjie <hongjie.qin@spacemit.com>  Thu, 18 Jan 2024 17:26:18 +0800
+
+bianbu-ai-support (1.0.2) mantic-spacemit; urgency=medium
+
+  [ bianbu-ci ]
+  * Sync change from bianbu-23.10/1.0.2
+
+ -- qinhongjie <hongjie.qin@spacemit.com>  Tue, 16 Jan 2024 21:00:18 +0800
+
+bianbu-ai-support (1.0.1) mantic-spacemit; urgency=medium
+
+  * Initial for bianbu-23.10
+
+ -- root <root@SW-Station>  Thu, 04 Jan 2024 20:25:23 +0800
+

debian/control

@@ -0,0 +1,15 @@
+Source: bianbu-ai-support
+Section: Utils
+Priority: optional
+Maintainer: bianbu-ai-support <bianbu-ai-support@spacemit.com>
+Build-Depends: cmake, debhelper-compat (= 12), onnxruntime, libopencv-dev
+Standards-Version: 4.5.0
+Homepage: https://gitlab.dc.com:8443/bianbu/ai/support
+
+Package: bianbu-ai-support
+Architecture: any
+Multi-Arch: foreign
+Depends: ${misc:Depends}, ${shlibs:Depends}, onnxruntime, libopencv-dev
+Description: auto-generated package by debmake
+ This Debian binary package was auto-generated by the
+ debmake(1) command provided by the debmake package.

debian/copyright

@@ -0,0 +1,73 @@
+Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: bianbu-ai-support
+Upstream-Contact: <preferred name and address to reach the upstream project>
+Source: <url://example.com>
+#
+# Please double check copyright with the licensecheck(1) command.
+
+Files:     .gitlab-ci.yml
+           CMakeLists.txt
+           README.md
+           ci/dockerfile.riscv64
+           ci/dockerfile.x86_64
+           ci/test.sh
+           data/config/nanodet.json
+           data/config/yolov4.json
+           data/config/yolov6.json
+           data/imgs/bird.jpeg
+           data/imgs/cat.jpg
+           data/imgs/dog.jpg
+           data/imgs/person0.jpg
+           data/imgs/person1.jpg
+           data/imgs/person2.jpeg
+           data/labels/coco.txt
+           data/labels/synset.txt
+           demo/CMakeLists.txt
+           demo/README.md
+           demo/build.sh
+           demo/image_classification_demo.cc
+           demo/object_detection_demo.cc
+           demo/object_detection_stream_demo.cc
+           demo/object_detection_video_demo.cc
+           demo/utils/box_utils.h
+           demo/utils/check_utils.h
+           include/task/vision/image_classification_task.h
+           include/task/vision/image_classification_types.h
+           include/task/vision/object_detection_task.h
+           include/task/vision/object_detection_types.h
+           include/utils/time.h
+           include/utils/utils.h
+           src/CMakeLists.txt
+           src/core/engine.cc
+           src/core/engine.h
+           src/core/ort_wrapper.cc
+           src/core/ort_wrapper.h
+           src/processor/classification_postprocessor.cc
+           src/processor/classification_postprocessor.h
+           src/processor/classification_preprocessor.cc
+           src/processor/classification_preprocessor.h
+           src/processor/detection_postprocessor.cc
+           src/processor/detection_postprocessor.h
+           src/processor/detection_preprocessor.cc
+           src/processor/detection_preprocessor.h
+           src/processor/processor.h
+           src/task/core/base_task_api.h
+           src/task/vision/base_vision_task_api.h
+           src/task/vision/imageclassification/image_classification.cc
+           src/task/vision/imageclassification/image_classification.h
+           src/task/vision/imageclassification/image_classification_task.cc
+           src/task/vision/objectdetection/object_detection.cc
+           src/task/vision/objectdetection/object_detection.h
+           src/task/vision/objectdetection/object_detection_task.cc
+           src/tests/CMakeLists.txt
+           src/tests/json_test.cc
+           src/utils/cv2_utils.h
+           src/utils/json.hpp
+           src/utils/nms_utils.h
+           src/utils/utils.h
+Copyright: __NO_COPYRIGHT_NOR_LICENSE__
+License:   __NO_COPYRIGHT_NOR_LICENSE__
+
+#----------------------------------------------------------------------------
+# Files marked as NO_LICENSE_TEXT_FOUND may be covered by the following
+# license/copyright files.

debian/postinst

@@ -0,0 +1,64 @@
+#!/bin/bash
+
+set -e
+#set -x
+
+export XDG_DATA_HOME=${XDG_DATA_HOME:-/usr/share/}
+
+function copy_install_to_desktop() {
+  if [ $# -ne 3 ]; then
+    return;
+  fi
+  local curFileName=$1
+  local aiDesktopName=$2
+  local curUserName=$3
+  if [ -e "/usr/share/applications/${aiDesktopName}" ]; then
+    cp -f /usr/share/applications/${aiDesktopName} "${curFileName}/${gDesktopName}/${wpsDesktopName}"
+    if test $? -eq 0; then
+      chmod +x "${curFileName}/${gDesktopName}/${aiDesktopName}"
+      chown ${curUserName} "${curFileName}/${gDesktopName}/${aiDesktopName}"
+    fi
+  fi
+}
+
+aiDesktop=(
+  "object-detection.desktop"
+  #"hand-tracker.desktop"
+  "pose-tracker.desktop"
+)
+function ai_config_desktop() {
+  gDesktopName="桌面"
+  if [ -d "/root/桌面" ]; then
+    gDesktopName="桌面"
+  elif [ -d "/root/Desktop" ]; then
+    gDesktopName="Desktop"
+  fi
+
+  for FILENAME in /home/*; do
+
+    if [ -f "${FILENAME}/.config/user-dirs.dirs" ]; then
+      if [ ! $HOME ]; then HOME=${FILENAME}; fi
+      source "${FILENAME}/.config/user-dirs.dirs"
+      if [ ! -d "${XDG_DESKTOP_DIR}" ]; then
+        mkdir -p "${XDG_DESKTOP_DIR}" >/dev/null 2>&1 || true
+      fi
+      gDesktopName="${XDG_DESKTOP_DIR//${HOME}\//}"
+    else
+      if [ -d "${FILENAME}/桌面" ]; then
+        gDesktopName="桌面"
+      elif [ -d "${FILENAME}/Desktop" ]; then
+        gDesktopName="Desktop"
+      fi
+    fi
+
+    if [ -d "${FILENAME}/${gDesktopName}" ]; then
+      local curUserName=$(echo ${FILENAME} | awk '{print substr($FILENAME, 7, 32)}')
+      for desktop in "${aiDesktop[@]}"; do
+        rm -rf ${FILENAME}/${gDesktopName}/${desktop}
+        copy_install_to_desktop ${FILENAME} ${desktop} ${curUserName}
+      done
+    fi
+  done
+}
+
+ai_config_desktop

debian/postrm

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+ai_xdg_dirs=/usr/share
+ai_xdg_dir=
+
+function ai_demo_init() {
+  set -e
+}
+
+function ai_demo_remove() {
+  if [ -x /usr/bin/update-mime-database ] ; then
+    : #update-mime-database "${ai_xdg_dir}/mime"
+  fi
+
+  if [ -x /usr/bin/update-desktop-database ] ; then
+    : #update-desktop-database -q "${ai_xdg_dir}/applications"
+  fi
+
+  for HOMEDIR in /home/*; do
+    :
+  done
+}
+
+function ai_demo_main() {
+  if [ $# -eq 0 ] ; then
+    return;
+  fi
+
+  ai_demo_init
+  case $1 in
+    remove | upgrade )  shift; ai_demo_remove $@;;
+    purge )             ;;
+    abort-install )     ;;
+    abort-upgrade )     ;;
+    failed-upgrade )    ;;
+  esac
+}
+
+args="$@"
+ai_demo_main $@

debian/preinst

@@ -0,0 +1 @@
+#!/bin/bash

debian/prerm

@@ -0,0 +1,65 @@
+#!/bin/bash
+
+export XDG_DATA_HOME=${XDG_DATA_HOME:-/usr/share/}
+
+function ai_prerm_init() {
+  set -e
+}
+
+aiDesktop=(
+  "object-detection.desktop"
+  "hand-tracker.desktop"
+  "pose-tracker.desktop"
+)
+function ai_prerm_uninstall_desktop() {
+  gDesktopName="桌面"
+  if [ -d "/root/桌面" ]; then
+    gDesktopName="桌面"
+  elif [ -d "/root/Desktop" ]; then
+    gDesktopName="Desktop"
+  fi
+
+  #desktop
+  if [ -d "/root/${gDesktopName}" ]; then
+    for desktop in "${aiDesktop[@]}"; do
+      rm -rf /root/${gDesktopName}/${desktop}
+    done
+  fi
+
+  for FILENAME in /home/*; do
+    if [ -f "${FILENAME}/.config/user-dirs.dirs" ]; then
+      source "${FILENAME}/.config/user-dirs.dirs"
+      if [ ! -d "${XDG_DESKTOP_DIR}" ]; then
+        mkdir -p "${XDG_DESKTOP_DIR}" >/dev/null 2>&1 || true
+      fi
+      gDesktopName="${XDG_DESKTOP_DIR//${HOME}\//}"
+    else
+      if [ -d "${FILENAME}/桌面" ]; then
+        gDesktopName="桌面"
+      elif [ -d "${FILENAME}/Desktop" ]; then
+        gDesktopName="Desktop"
+      fi
+    fi
+
+    for desktop in "${aiDesktop[@]}"; do
+      rm -rf "${FILENAME}/${gDesktopName}/${desktop}"
+    done
+    rm -rf ${FILENAME}/.config/bianbu-ai-support
+  done
+}
+
+function ai_prerm_main() {
+  if [ $# -eq 0 ]; then
+    return;
+  fi
+
+  ai_prerm_init
+  case $1 in
+    remove )            shift; ai_prerm_uninstall_desktop $@;;
+    upgrade )           shift;;
+    failed-upgrade )    ;;
+  esac
+}
+
+args="$@"
+ai_prerm_main $@

debian/rules

@@ -0,0 +1,15 @@
+#!/usr/bin/make -f
+# You must remove unused comment lines for the released package.
+#export DH_VERBOSE = 1
+#export DEB_BUILD_MAINT_OPTIONS = hardening=+all
+#export DEB_CFLAGS_MAINT_APPEND  = -Wall -pedantic
+#export DEB_LDFLAGS_MAINT_APPEND = -Wl,--as-needed
+
+%:
+	dh $@  
+
+override_dh_auto_configure:
+	dh_auto_configure -- -DORT_HOME=/usr -DDEMO=ON
+
+override_dh_shlibdeps:
+	dh_shlibdeps --dpkg-shlibdeps-params=--ignore-missing-info

debian/source/format

@@ -0,0 +1 @@
+3.0 (native)

debian/source/local-options

@@ -0,0 +1,2 @@
+#abort-on-upstream-changes
+#unapply-patches

debian/watch

@@ -0,0 +1,2 @@
+# You must remove unused comment lines for the released package.
+version=3

demo/CMakeLists.txt

@@ -0,0 +1,86 @@
+cmake_minimum_required(VERSION 3.10)
+
+project(bianbuai-demo)
+
+set(CMAKE_CXX_STANDARD 14)
+#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fpermissive")
+
+if (WIN32)
+  set(ext_src_pattern
+    "utils/win_getopt/mb/*.cc")
+  file(GLOB ext_src CONFIGURE_DEPENDS ${ext_src_pattern})
+  include_directories("utils/win_getopt/mb/include")
+else()
+  #set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Werror")
+endif()
+
+# TODO: update cc files with '#ifndef NDEBUG'
+set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DDEBUG")
+
+# To find OpenCV, one may need to set OpenCV_DIR variable to the
+# absolute path to the directory containing OpenCVConfig.cmake file.
+# Otherwise, try to set OPENCV_INC and OPENCV_LIB variables via the
+# command line or GUI.
+if ((NOT DEFINED OPENCV_INC OR OPENCV_INC STREQUAL "") OR ((NOT DEFINED OPENCV_LIB OR OPENCV_LIB STREQUAL "")))
+find_package(OpenCV REQUIRED COMPONENTS core imgproc imgcodecs videoio highgui)
+if (OpenCV_FOUND)
+  if (NOT DEFINED OPENCV_INC OR OPENCV_INC STREQUAL "")
+    set(OPENCV_INC "${OpenCV_INCLUDE_DIRS}")
+  endif()
+  if (NOT DEFINED OPENCV_LIB OR OPENCV_LIB STREQUAL "")
+    get_target_property(OpenCV_LIB_PATH opencv_core LOCATION)
+    get_filename_component(OPENCV_LIB ${OpenCV_LIB_PATH} DIRECTORY)
+    set(OPENCV_LIBS ${OpenCV_LIBS})
+  endif()
+endif()
+endif()
+
+# Check Required Env
+if (NOT DEFINED BIANBUAI_HOME OR BIANBUAI_HOME STREQUAL "")
+  message(FATAL_ERROR "Env 'BIANBUAI_HOME' not defined for platform ${CMAKE_GENERATOR_PLATFORM}")
+endif()
+if (NOT DEFINED OPENCV_INC OR OPENCV_INC STREQUAL "")
+  message(FATAL_ERROR "OpenCV include dirs not found for platform ${CMAKE_GENERATOR_PLATFORM}")
+endif()
+if (NOT DEFINED OPENCV_LIB OR OPENCV_LIB STREQUAL "")
+  message(FATAL_ERROR "OpenCV library dirs not found for platform ${CMAKE_GENERATOR_PLATFORM}")
+endif()
+if (NOT DEFINED ORT_HOME)
+  if (EXISTS ${BIANBUAI_HOME}/lib/3rdparty/onnxruntime)
+    set(ORT_HOME ${BIANBUAI_HOME}/lib/3rdparty/onnxruntime)
+  else()
+    message(FATAL_ERROR "Env 'ORT_HOME' not defined and OnnxRuntime library may not found for platform ${CMAKE_GENERATOR_PLATFORM}")
+  endif()
+endif()
+
+include_directories(${OPENCV_INC} ${BIANBUAI_HOME}/include/bianbuai)
+
+if (NOT WIN32)
+  set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-rpath-link,${BIANBUAI_HOME}/lib:${OPENCV_LIB}:${OPENCV_LIB}/3rdparty:${ORT_HOME}/lib")
+endif()
+set(TARGET_EXE_LINKER_FLAGS "-Wl,--as-needed")
+link_directories(${BIANBUAI_HOME}/lib ${ORT_HOME}/lib ${OPENCV_LIB} ${OPENCV_LIB}/3rdparty)
+
+add_executable(classification_demo image_classification_demo.cc ${ext_src})
+target_link_libraries(classification_demo PUBLIC bianbuai opencv_core opencv_imgcodecs)
+
+add_executable(detection_demo object_detection_demo.cc ${ext_src})
+target_link_libraries(detection_demo PUBLIC bianbuai opencv_core opencv_imgproc opencv_imgcodecs)
+
+add_executable(detection_video_demo object_detection_video_demo.cc ${ext_src})
+target_link_libraries(detection_video_demo PUBLIC bianbuai opencv_core opencv_imgproc opencv_videoio opencv_highgui)
+
+find_package(Threads REQUIRED)
+add_executable(detection_stream_demo object_detection_stream_demo.cc ${ext_src})
+target_link_libraries(detection_stream_demo PUBLIC bianbuai Threads::Threads opencv_core opencv_imgproc opencv_videoio opencv_imgcodecs opencv_highgui)
+
+add_executable(estimation_demo pose_estimation_demo.cc ${ext_src})
+target_link_libraries(estimation_demo PUBLIC bianbuai opencv_core opencv_imgproc opencv_imgcodecs)
+
+add_executable(tracker_stream_demo pose_tracker_stream_demo.cc ${ext_src})
+target_link_libraries(tracker_stream_demo PUBLIC bianbuai Threads::Threads opencv_core opencv_imgproc opencv_videoio opencv_imgcodecs opencv_highgui)
+
+install(TARGETS detection_demo classification_demo detection_stream_demo detection_video_demo estimation_demo tracker_stream_demo
+        RUNTIME DESTINATION bin
+        LIBRARY DESTINATION lib
+        ARCHIVE DESTINATION lib)

demo/README.md

@@ -0,0 +1,49 @@
+# User Guide
+
+## Quick Build
+
+```bash
+# Note: update the following settings if necessary
+SDK=${PATH_TO_SPACEMIT_AI_SDK} # e.g. /opt/spacemit-ai-sdk.v0.5.0
+
+# For cross compilation, try:
+CROSS_TOOL=$SDK/spacemit-gcc/bin/riscv64-unknown-linux-gnu-
+SYSROOT=$SDK/spacemit-gcc/sysroot
+BIANBUAI_HOME=$SDK/bianbu-ai-support
+ORT_HOME=$SDK/spacemit-ort
+OPENCV_DIR=$SDK/bianbu-ai-support/lib/3rdparty/opencv4/lib/cmake/opencv4
+
+# For native building, one may need to install opencv first, then try:
+#CROSS_TOOL=
+#SYSROOT=
+#BIANBUAI_HOME=$SDK/bianbu-ai-support
+#ORT_HOME=${PATH_TO_OFFICIAL_ONNXRUNTIME_RELEASE}
+#OPENCV_DIR=
+
+mkdir build && pushd build
+cmake .. -DBIANBUAI_HOME=${BIANBUAI_HOME} -DORT_HOME=${ORT_HOME} -DOpenCV_DIR=${OPENCV_DIR} -DCMAKE_C_COMPILER=${CROSS_TOOL}gcc -DCMAKE_CXX_COMPILER=${CROSS_TOOL}g++ -DCMAKE_SYSROOT=${SYSROOT}
+make -j4
+popd
+```
+
+## Quick Test
+
+```bash
+# Add qemu settings if necessary(e.g. run riscv64 demo on x86_64).
+QEMU_CMD="$SDK/qemu/bin/qemu-riscv64 -L $SYSROOT"
+# For native test, just let:
+#QEMU_CMD=
+
+# Smoke test with image classification
+env LD_LIBRARY_PATH=${ORT_HOME}/lib:$LD_LIBRARY_PATH ${QEMU_CMD} \
+  build/classification_demo data/models/squeezenet1.1-7.onnx data/labels/synset.txt data/imgs/dog.jpg
+
+# Smoke test with object detection
+env LD_LIBRARY_PATH=${ORT_HOME}/lib:$LD_LIBRARY_PATH ${QEMU_CMD} \
+  build/detection_demo data/models/nanodet-plus-m_320.onnx data/imgs/person0.jpg result0.jpg data/labels/coco.txt
+```
+
+* Model List
+
+4f22f9a64ab9612ca4372a0343b3879a  [data/models/nanodet-plus-m_320.onnx](https://bj.bcebos.com/paddlehub/fastdeploy/nanodet-plus-m_320.onnx)  
+497ad0774f4e0b59e4f2c77ae88fcdfc  [data/models/squeezenet1.1-7.onnx](https://github.com/onnx/models/blob/main/archive/vision/classification/squeezenet/model/squeezenet1.1-7.onnx)  

demo/build.sh

@@ -0,0 +1,84 @@
+#!/bin/bash
+#
+# Author: hongjie.qin@spacemit.com
+# Brief:  Build demos and run smoke test.
+#
+
+set -e
+
+# Note: update the following settings if necessary
+_NAME=bianbu # cuspace
+SDK=$(dirname $(which ${_NAME})) #$(dirname ${BASH_SOURCE[0]})
+
+function config_native() {
+  BIANBUAI_HOME=$SDK/bianbu-ai-support
+  # Plz update the following settings !!!
+  ORT_HOME=${PATH_TO_OFFICIAL_ONNXRUNTIME_RELEASE}
+}
+
+function config_x86_riscv64() {
+  CROSS_TOOL=$SDK/spacemit-gcc/bin/riscv64-unknown-linux-gnu-
+  SYSROOT=$SDK/spacemit-gcc/sysroot
+  BIANBUAI_HOME=$SDK/bianbu-ai-support
+  ORT_HOME=$SDK/spacemit-ort # cuspace-ai
+  OPENCV_DIR=$SDK/bianbu-ai-support/lib/3rdparty/opencv4/lib/cmake/opencv4
+  QEMU_CMD="$SDK/spacemit-qemu/bin/qemu-riscv64 -L $SYSROOT"
+}
+
+# config
+if [[ $@ =~ "--native" ]]; then
+  config_native
+else
+  config_x86_riscv64
+fi
+
+function build() {
+  mkdir build && pushd build
+  cmake .. -DBIANBUAI_HOME=${BIANBUAI_HOME} -DORT_HOME=${ORT_HOME} -DOpenCV_DIR=${OPENCV_DIR} -DCMAKE_C_COMPILER=${CROSS_TOOL}gcc -DCMAKE_CXX_COMPILER=${CROSS_TOOL}g++ -DCMAKE_SYSROOT=${SYSROOT}
+  make -j4
+  popd
+  echo "[INFO] Building demos done."
+}
+if [[ ! -d build ]]; then
+  build
+else
+  cd build && make && cd ..
+fi
+
+task_prepare=(
+  "if [[ ! -d data && -d ../data ]]; then ln -sf ../data .; fi"
+  "if [[ ! -d data ]]; then echo '[Error] Can not find data directory!'; exit 0; fi"
+  "mkdir -p data/models"
+  # TODO: add md5sum checking
+  "if [[ ! -f data/models/squeezenet1.1-7.onnx ]]; then wget https://media.githubusercontent.com/media/onnx/models/main/validated/vision/classification/squeezenet/model/squeezenet1.1-7.onnx -O data/models/squeezenet1.1-7.onnx; fi"
+  "if [[ ! -f data/models/nanodet-plus-m_320.onnx ]]; then wget https://bj.bcebos.com/paddlehub/fastdeploy/nanodet-plus-m_320.onnx -O data/models/nanodet-plus-m_320.onnx; fi"
+)
+task_classification=(
+  "build/classification_demo data/models/squeezenet1.1-7.onnx data/labels/synset.txt data/imgs/dog.jpg"
+)
+task_detection=(
+  "build/detection_demo data/models/nanodet-plus-m_320.onnx data/labels/coco.txt data/imgs/person0.jpg result0.jpg"
+)
+function smoke_test() {
+  # preparation(e.g. download models)
+  echo "[INFO] Prepare ..."
+  for cmd in "${task_prepare[@]}"; do eval "$cmd"; done
+  # image classification task test
+  echo "[INFO] Smoke test with image classification task ..."
+  for cmd in "${task_classification[@]}"; do
+    echo "[INFO] Run: $cmd"
+    env LD_LIBRARY_PATH=${ORT_HOME}/lib:$LD_LIBRARY_PATH ${QEMU_CMD} $cmd
+  done
+  # object detection task test
+  echo "[INFO] Smoke test with object detection task ..."
+  for cmd in "${task_detection[@]}"; do
+    echo "[INFO] Run: $cmd"
+    env LD_LIBRARY_PATH=${ORT_HOME}/lib:$LD_LIBRARY_PATH ${QEMU_CMD} $cmd
+  done
+}
+
+if [[ "$@" =~ "--test" ]]; then
+  smoke_test
+else
+  echo "[INFO] Try '${BASH_SOURCE[0]} $@ --test' to run the demos."
+fi

demo/dataloader.hpp

@@ -0,0 +1,194 @@
+#ifndef SUPPORT_DEMO_DATALOADER_HPP_
+#define SUPPORT_DEMO_DATALOADER_HPP_
+
+#include <cctype>  // for: std::isdigit
+#include <memory>
+#include <mutex>
+#include <queue>
+#include <string>
+/* opencv header files */
+#include "opencv2/opencv.hpp"
+/* bianbu-ai-support header files */
+#include "utils/utils.h"
+
+class DataLoader {
+ public:
+  DataLoader(const int& resize_height, const int& resize_width) {
+    enable = true;
+    resize_height_ = resize_height;
+    resize_width_ = resize_width;
+    preview_fps_ = 0;
+    detection_fps_ = 0;
+  }
+  ~DataLoader() {}
+  bool ifenable() { return enable; }
+  void set_disable() { enable = false; }
+  void set_preview_fps(int preview_fps) { preview_fps_ = preview_fps; }
+  void set_detection_fps(int detection_fps) { detection_fps_ = detection_fps; }
+  int get_preview_fps() { return preview_fps_; }
+  int get_detection_fps() { return detection_fps_; }
+  int get_resize_height() { return resize_height_; }
+  int get_resize_width() { return resize_width_; }
+  virtual cv::Mat fetch_frame() = 0;
+  virtual cv::Mat peek_frame() = 0;
+
+ private:
+  bool enable;
+  int resize_height_;
+  int resize_width_;
+  int preview_fps_;
+  int detection_fps_;
+};
+
+// 独占式
+class ExclusiveDataLoader : public DataLoader {
+ public:
+  ExclusiveDataLoader(const int& resize_height, const int& resize_width)
+      : DataLoader(resize_height, resize_width) {}
+  ~ExclusiveDataLoader() {}
+  int init(const std::string& path) {
+    capture_.open(path);
+    if (capture_.isOpened()) {
+      return 0;
+    } else {
+      std::cout << "Open video capture failed" << std::endl;
+      return -1;
+    }
+  }
+  int init(const int camera_id) {
+    capture_.open(camera_id);
+    if (capture_.isOpened()) {
+      return 0;
+    } else {
+      std::cout << "Open camera capture failed" << std::endl;
+      return -1;
+    }
+  }
+  cv::Mat fetch_frame() {
+    cv::Mat frame;
+    capture_.read(frame);
+    return frame;
+  }
+  cv::Mat peek_frame() { return fetch_frame(); }
+
+ private:
+  cv::VideoCapture capture_;
+};
+
+#ifndef _WIN32
+
+#include <fcntl.h>
+#include <linux/videodev2.h>
+#include <sys/ioctl.h>
+#include <unistd.h>  //for close
+
+static bool is_valid_camera(const std::string& path) {
+  int fd = open(path.c_str(), O_RDWR);
+  if (fd == -1) {
+    return false;
+  }
+  struct v4l2_capability cap;
+  if (ioctl(fd, VIDIOC_QUERYCAP, &cap) == -1) {
+    close(fd);
+    return false;
+  }
+  close(fd);
+  return (cap.capabilities & V4L2_CAP_VIDEO_CAPTURE) != 0;
+}
+
+#endif
+
+inline bool isNumber(const std::string& str) {
+  for (char const& c : str) {
+    if (std::isdigit(c) == 0) {
+      return false;
+    }
+  }
+  return true;
+}
+
+// 共享式
+class SharedDataLoader : public DataLoader {
+ public:
+  SharedDataLoader(const int& resize_height, const int& resize_width)
+      : DataLoader(resize_height, resize_width) {}
+  ~SharedDataLoader() {}
+
+  int init(const std::string& path) {
+    if (isNumber(path)) {
+      return init(std::stoi(path));
+    }
+    capture_.open(path);
+    if (capture_.isOpened()) {
+      int width = 1280;
+      int height = 720;
+      capture_.set(cv::CAP_PROP_FRAME_WIDTH, width);
+      capture_.set(cv::CAP_PROP_FRAME_HEIGHT, height);
+      return 0;
+    } else {
+      std::cout << "Open video capture failed" << std::endl;
+      return -1;
+    }
+  }
+
+  int init(int camera_id) {
+#ifndef _WIN32
+    capture_.open(camera_id);
+    if (!capture_.isOpened()) {
+      std::cout
+          << "Open camera capture failed, try to figure out right camera id"
+          << std::endl;
+      std::string path = "/dev/video";
+      for (int i = 0; i <= 100; ++i) {
+        std::string device_path = path + std::to_string(i);
+        if (is_valid_camera(device_path)) {
+          capture_.open(i);
+          if (capture_.isOpened()) {
+            break;
+          }
+        }
+      }
+    }
+#else
+    capture_.open(camera_id);
+#endif
+    if (capture_.isOpened()) {
+      int width = 640;
+      int height = 480;
+      capture_.set(cv::CAP_PROP_FRAME_WIDTH, width);
+      capture_.set(cv::CAP_PROP_FRAME_HEIGHT, height);
+      return 0;
+    } else {
+      std::cout << "Open camera capture failed" << std::endl;
+      return -1;
+    }
+  }
+
+  cv::Mat fetch_frame() {
+    cv::Mat frame, temp;
+    capture_.read(frame);
+    if (!frame.empty()) {
+      resize_unscale(frame, temp, get_resize_height(), get_resize_width());
+    }
+    frame_mutex_.lock();
+    frame_ = temp.clone();
+    frame_mutex_.unlock();
+    return frame;
+  }
+  cv::Mat peek_frame() {
+    cv::Mat frame;
+    frame_mutex_.lock();
+    frame = frame_.clone();  // 深拷贝
+    frame_mutex_.unlock();
+    return frame;
+  }
+
+ private:
+  std::shared_ptr<cv::Mat> frame;
+  cv::Mat frame_;
+  std::mutex frame_mutex_;
+  cv::VideoCapture capture_;
+  std::queue<cv::Mat> frame_queue_;
+};
+
+#endif  // SUPPORT_DEMO_DATALOADER_HPP_

demo/image_classification_demo.cc

@@ -0,0 +1,54 @@
+#include <iostream>
+
+#include "task/vision/image_classification_task.h"
+#include "utils/check_utils.h"
+#include "utils/time.h"
+#include "utils/utils.h"
+
+int main(int argc, char* argv[]) {
+  std::string filePath, labelFilepath, imageFilepath;
+  if (argc == 4) {
+    filePath = argv[1];
+    labelFilepath = argv[2];
+    imageFilepath = argv[3];
+  } else if (argc > 4) {
+    filePath = argv[1];
+    labelFilepath = argv[2];
+    imageFilepath = argv[3];
+    if (!checkImageFileExtension(imageFilepath)) {
+      std::cout << "[ ERROR ] The ImageFilepath is not correct. Make sure you "
+                   "are setting the path to an imgae file (.jpg/.jpeg/.png)"
+                << std::endl;
+      return -1;
+    }
+    if (!exists_check(imageFilepath)) {
+      std::cout << "[ ERROR ] The Image File does not exist. Make sure you are "
+                   "setting the correct path to the file"
+                << std::endl;
+      return -1;
+    }
+  } else {
+    std::cout << "run with " << argv[0]
+              << " <modelFilepath> <labelFilepath> <imageFilepath>"
+              << std::endl;
+    return -1;
+  }
+  cv::Mat imgRaw;
+  std::unique_ptr<imageClassificationTask> imageclassification =
+      std::unique_ptr<imageClassificationTask>(
+          new imageClassificationTask(filePath, labelFilepath));
+#ifdef DEBUG
+  std::cout << "." << std::endl;
+#endif
+  {
+#ifdef DEBUG
+    TimeWatcher t("|-- Load input data");
+#endif
+    imgRaw = cv::imread(imageFilepath);
+  }
+  if (!imageclassification->getInitFlag()) {
+    ImageClassificationResult result = imageclassification->Classify(imgRaw);
+    std::cout << "classify result: " << result.label_text << std::endl;
+  }
+  return 0;
+}

demo/object_detection.hpp

@@ -0,0 +1,28 @@
+#ifndef SUPPORT_DEMO_OBJECT_DETECTION_HPP_
+#define SUPPORT_DEMO_OBJECT_DETECTION_HPP_
+
+#include <string>
+#include <vector>
+
+#include "opencv2/opencv.hpp"
+#include "task/vision/object_detection_types.h"
+
+inline void draw_boxes_inplace(cv::Mat &mat_inplace,
+                               const std::vector<Boxi> &boxes) {
+  if (boxes.empty()) {
+    return;
+  }
+  for (const auto &box : boxes) {
+    if (box.flag) {
+      cv::rectangle(mat_inplace, box.rect(), cv::Scalar(255, 255, 0), 2);
+      if (box.label_text) {
+        std::string label_text(box.label_text);
+        label_text = label_text + ":" + std::to_string(box.score).substr(0, 4);
+        cv::putText(mat_inplace, label_text, box.tl(), cv::FONT_HERSHEY_SIMPLEX,
+                    .5f, cv::Scalar(0, 69, 255), 1);
+      }
+    }
+  }
+}
+
+#endif  // SUPPORT_DEMO_OBJECT_DETECTION_HPP_

demo/object_detection_demo.cc

@@ -0,0 +1,144 @@
+#include <iomanip>  // for: setprecision
+#include <iostream>
+
+#include "object_detection.hpp"
+#include "task/vision/object_detection_task.h"
+#include "utils/check_utils.h"
+#include "utils/time.h"
+#include "utils/utils.h"
+
+int main(int argc, char* argv[]) {
+  std::vector<Boxi> resultBoxes;
+  std::string filePath, modelFilepath, imageFilepath, saveImgpath,
+      labelFilepath, configFilepath;
+  cv::Mat imgRaw;
+#ifdef DEBUG
+  std::cout << "." << std::endl;
+#endif
+  if (argc == 4) {
+    filePath = argv[1];
+    imageFilepath = argv[2];
+    saveImgpath = argv[3];
+    if (!checkImageFileExtension(imageFilepath) ||
+        !checkImageFileExtension(saveImgpath)) {
+      std::cout << "[ ERROR ] The ImageFilepath is not correct. Make sure you "
+                   "are setting the path to an imgae file (.jpg/.jpeg/.png)"
+                << std::endl;
+      return -1;
+    }
+    if (!exists_check(imageFilepath)) {
+      std::cout << "[ ERROR ] The Image File does not exist. Make sure you are "
+                   "setting the correct path to the file"
+                << std::endl;
+      return -1;
+    }
+    {
+#ifdef DEBUG
+      TimeWatcher t("|-- Load input data");
+#endif
+      imgRaw = cv::imread(imageFilepath);
+    }
+    std::unique_ptr<objectDetectionTask> objectdetectiontask =
+        std::unique_ptr<objectDetectionTask>(new objectDetectionTask(filePath));
+    resultBoxes = objectdetectiontask->Detect(imgRaw).result_bboxes;
+    {
+#ifdef DEBUG
+      TimeWatcher t("|-- Output result");
+#endif
+      for (int i = 0; i < static_cast<int>(resultBoxes.size()); i++) {
+        if (resultBoxes[i].flag) {
+          std::cout << "bbox[" << std::setw(2) << i << "]"
+                    << " "
+                    << "x1y1x2y2: "
+                    << "(" << std::setw(4) << resultBoxes[i].x1 << ","
+                    << std::setw(4) << resultBoxes[i].y1 << "," << std::setw(4)
+                    << resultBoxes[i].x2 << "," << std::setw(4)
+                    << resultBoxes[i].y2 << ")"
+                    << ", "
+                    << "score: " << std::fixed << std::setprecision(3)
+                    << std::setw(4) << resultBoxes[i].score << ", "
+                    << "label_text: " << resultBoxes[i].label_text << std::endl;
+        }
+      }
+    }
+    {
+#ifdef DEBUG
+      TimeWatcher t("|-- Box drawing");
+#endif
+      draw_boxes_inplace(imgRaw, resultBoxes);
+    }
+
+    cv::imwrite(saveImgpath, imgRaw);
+    // cv::imshow("detected.jpg",imgRaw);
+    // cv::waitKey(0);
+  } else if (argc == 5) {
+    filePath = argv[1];
+    labelFilepath = argv[2];
+    imageFilepath = argv[3];
+    saveImgpath = argv[4];
+    if (!checkImageFileExtension(imageFilepath) ||
+        !checkImageFileExtension(saveImgpath)) {
+      std::cout << "[ ERROR ] The ImageFilepath is not correct. Make sure you "
+                   "are setting the path to an imgae file (.jpg/.jpeg/.png)"
+                << std::endl;
+      return -1;
+    }
+    if (!exists_check(imageFilepath)) {
+      std::cout << "[ ERROR ] The Image File does not exist. Make sure you are "
+                   "setting the correct path to the file"
+                << std::endl;
+      return -1;
+    }
+    {
+#ifdef DEBUG
+      TimeWatcher t("|-- Load input data");
+#endif
+      imgRaw = cv::imread(imageFilepath);
+    }
+    std::unique_ptr<objectDetectionTask> objectdetectiontask =
+        std::unique_ptr<objectDetectionTask>(
+            new objectDetectionTask(filePath, labelFilepath));
+    if (objectdetectiontask->getInitFlag() != 0) {
+      return -1;
+    }
+    resultBoxes = objectdetectiontask->Detect(imgRaw).result_bboxes;
+    {
+#ifdef DEBUG
+      TimeWatcher t("|-- Output result");
+#endif
+      for (int i = 0; i < static_cast<int>(resultBoxes.size()); i++) {
+        if (resultBoxes[i].flag) {
+          std::cout << "bbox[" << std::setw(2) << i << "]"
+                    << " "
+                    << "x1y1x2y2: "
+                    << "(" << std::setw(4) << resultBoxes[i].x1 << ","
+                    << std::setw(4) << resultBoxes[i].y1 << "," << std::setw(4)
+                    << resultBoxes[i].x2 << "," << std::setw(4)
+                    << resultBoxes[i].y2 << ")"
+                    << ", "
+                    << "score: " << std::fixed << std::setprecision(3)
+                    << std::setw(4) << resultBoxes[i].score << ", "
+                    << "label_text: " << resultBoxes[i].label_text << std::endl;
+        }
+      }
+    }
+    {
+#ifdef DEBUG
+      TimeWatcher t("|-- Box drawing");
+#endif
+      draw_boxes_inplace(imgRaw, resultBoxes);
+    }
+
+    cv::imwrite(saveImgpath, imgRaw);
+    // cv::imshow("detected.jpg",imgRaw);
+    // cv::waitKey(0);
+  } else {
+    std::cout
+        << "run with " << argv[0]
+        << " <modelFilepath> <labelFilepath> <imageFilepath> <saveImgpath> or "
+        << argv[0] << " <configFilepath> <imageFilepath> <saveImgpath>"
+        << std::endl;
+    return -1;
+  }
+  return 0;
+}

demo/object_detection_stream_demo.cc

@@ -0,0 +1,268 @@
+#include <pthread.h>
+#include <stdlib.h>
+#include <unistd.h>  // for: getopt
+
+#include <algorithm>  // for: swap
+#include <chrono>
+#include <cmath>
+#include <iostream>
+#include <mutex>
+#include <queue>
+#include <string>
+#include <thread>
+
+#include "dataloader.hpp"
+#include "object_detection.hpp"
+#include "opencv2/opencv.hpp"
+#include "task/vision/object_detection_task.h"
+#ifdef DEBUG
+#include "utils/time.h"
+#endif
+
+#include "utils/utils.h"
+
+class Detector {
+ public:
+  explicit Detector(const std::string& filePath) { filePath_ = filePath; }
+  ~Detector() {}
+  // 初始化/反初始化
+  int init() {
+    objectdetectiontask_ = std::unique_ptr<objectDetectionTask>(
+        new objectDetectionTask(filePath_));
+    return get_init_flag();
+  }
+
+  int uninit() { return 0; }
+
+  // 推理
+  int infer(cv::Mat frame) {
+    if (frame.empty()) {
+      return -1;
+    }
+    ObjectDetectionResult objs_temp = objectdetectiontask_->Detect(frame);
+    objs_mutex_.lock();
+    objs_array_.push(objs_temp);  // 直接替换掉当前的 objs_array_
+    objs_mutex_.unlock();
+    return objs_array_.size();
+  }
+
+  // 查询检测结果
+  int detected() { return objs_array_.size(); }
+  int get_init_flag() { return objectdetectiontask_->getInitFlag(); }
+  // 移走检测结果
+  ObjectDetectionResult get_object() {
+    ObjectDetectionResult objs_moved;
+    objs_mutex_.lock();
+    objs_moved = objs_array_.back();
+    std::queue<struct ObjectDetectionResult> empty;
+    std::swap(empty, objs_array_);
+    objs_mutex_.unlock();
+    return objs_moved;
+  }
+
+ private:
+  std::mutex objs_mutex_;
+  std::queue<struct ObjectDetectionResult> objs_array_;
+  std::unique_ptr<objectDetectionTask> objectdetectiontask_;
+  std::string filePath_;
+};
+
+// 检测线程
+void Detection(DataLoader& dataloader, Detector& detector) {
+  if (detector.init() != 0) {
+    std::cout << "[ERROR] detector init error" << std::endl;
+    dataloader.set_disable();
+  }
+  cv::Mat frame;
+  while (dataloader.ifenable()) {
+    auto start = std::chrono::steady_clock::now();
+    frame = dataloader.peek_frame();  // 取(拷贝)一帧数据
+    if ((frame).empty()) {
+      dataloader.set_disable();
+      continue;
+    }
+    int flag = detector.infer(frame);  // 推理并保存检测结果
+    auto end = std::chrono::steady_clock::now();
+    auto detection_duration =
+        std::chrono::duration_cast<std::chrono::milliseconds>(end - start);
+    dataloader.set_detection_fps(1000 / (detection_duration.count()));
+    if (flag == -1) {
+      std::cout << "[Error] infer frame failed" << std::endl;
+      break;  // 摄像头结束拍摄或者故障
+    }
+  }
+  std::cout << "detection thread quit" << std::endl;
+}
+
+// 预览线程
+void Preview(DataLoader& dataloader, Detector& detector) {
+  cv::Mat frame;
+  ObjectDetectionResult objs;
+  auto now = std::chrono::steady_clock::now();
+  objs.timestamp = now;
+  int count = 0;
+  int dur = 0;
+  int enable_show = 1;
+  const char* showfps = getenv("SHOWFPS");
+  const char* show = getenv("SHOW");
+  if (show && strcmp(show, "-1") == 0) {
+    enable_show = -1;
+  }
+  while (dataloader.ifenable()) {
+    auto start = std::chrono::steady_clock::now();
+    frame = dataloader.fetch_frame();  // 取(搬走)一帧数据
+    if ((frame).empty()) {
+      dataloader.set_disable();
+      break;
+    }
+    if (detector.detected())  // 判断原因: detector.detected 不用锁,
+                              // detector.get_object 需要锁;
+    {
+      // 是否有检测结果
+      objs = detector.get_object();  // 取(搬走)检测结果(移动赋值)
+      if (objs.result_bboxes.size()) {
+        int input_height = dataloader.get_resize_height();
+        int input_width = dataloader.get_resize_width();
+        int img_height = frame.rows;
+        int img_width = frame.cols;
+        float resize_ratio = std::min(
+            static_cast<float>(input_height) / static_cast<float>(img_height),
+            static_cast<float>(input_width) / static_cast<float>(img_width));
+        float dw = (input_width - resize_ratio * img_width) / 2;
+        float dh = (input_height - resize_ratio * img_height) / 2;
+        for (int i = 0; i < static_cast<int>(objs.result_bboxes.size()); i++) {
+          objs.result_bboxes[i].x1 =
+              (objs.result_bboxes[i].x1 - dw) / resize_ratio;
+          objs.result_bboxes[i].x2 =
+              (objs.result_bboxes[i].x2 - dw) / resize_ratio;
+          objs.result_bboxes[i].y1 =
+              (objs.result_bboxes[i].y1 - dh) / resize_ratio;
+          objs.result_bboxes[i].y2 =
+              (objs.result_bboxes[i].y2 - dh) / resize_ratio;
+        }
+      }
+      {
+#ifdef DEBUG
+        TimeWatcher t("|-- Output result");
+#endif
+        for (int i = 0; i < static_cast<int>(objs.result_bboxes.size()); i++) {
+          if (objs.result_bboxes[i].flag) {
+            std::cout << "bbox[" << std::setw(2) << i << "]"
+                      << " "
+                      << "x1y1x2y2: "
+                      << "(" << std::setw(4) << objs.result_bboxes[i].x1 << ","
+                      << std::setw(4) << objs.result_bboxes[i].y1 << ","
+                      << std::setw(4) << objs.result_bboxes[i].x2 << ","
+                      << std::setw(4) << objs.result_bboxes[i].y2 << ")"
+                      << ", "
+                      << "score: " << std::fixed << std::setprecision(3)
+                      << std::setw(4) << objs.result_bboxes[i].score << ", "
+                      << "label_text: " << objs.result_bboxes[i].label_text
+                      << std::endl;
+          }
+        }
+      }
+    }
+    // 调用 detector.detected 和 detector.get_object 期间,
+    // 检测结果依然可能被刷新
+    now = std::chrono::steady_clock::now();
+    auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(
+        now - objs.timestamp);
+    if (duration.count() < 1000) {
+      draw_boxes_inplace((frame), objs.result_bboxes);  // 画框
+    }
+    int preview_fps = dataloader.get_preview_fps();
+    int detection_fps = dataloader.get_detection_fps();
+    if (showfps != nullptr) {
+      cv::putText(frame, "preview fps: " + std::to_string(preview_fps),
+                  cv::Point(0, 15), cv::FONT_HERSHEY_SIMPLEX, 0.5f,
+                  cv::Scalar(0, 255, 0), 1, cv::LINE_AA);
+      cv::putText(frame, "detection fps: " + std::to_string(detection_fps),
+                  cv::Point(500, 15), cv::FONT_HERSHEY_SIMPLEX, 0.5f,
+                  cv::Scalar(0, 255, 0), 1, cv::LINE_AA);
+    }
+    if (enable_show != -1) {
+      cv::imshow("Detection", (frame));
+      cv::waitKey(10);
+    }
+    auto end = std::chrono::steady_clock::now();
+    auto preview_duration =
+        std::chrono::duration_cast<std::chrono::milliseconds>(end - start);
+    count++;
+    dur = dur + preview_duration.count();
+    if (dur >= 1000) {
+      dataloader.set_preview_fps(count);
+      dur = 0;
+      count = 0;
+    }
+    if (enable_show != -1) {
+      if (cv::getWindowProperty("Detection", cv::WND_PROP_VISIBLE) < 1) {
+        dataloader.set_disable();
+        break;
+      }
+    }
+  }
+  std::cout << "preview thread quit" << std::endl;
+  if (enable_show != -1) {
+    cv::destroyAllWindows();
+  }
+}
+
+#ifndef _WIN32
+void setThreadName(std::thread& thread, const char* name) {
+  pthread_setname_np(thread.native_handle(), name);
+}
+#endif
+
+int main(int argc, char* argv[]) {
+  std::string filePath, input, inputType;
+  int resize_height{320}, resize_width{320};
+  if (argc == 4) {
+    filePath = argv[1];
+    input = argv[2];
+    inputType = argv[3];
+  } else if (argc > 4) {
+    filePath = argv[1];
+    input = argv[2];
+    inputType = argv[3];
+    int o;
+    const char* optstring = "w:h:";
+    while ((o = getopt(argc, argv, optstring)) != -1) {
+      switch (o) {
+        case 'w':
+          resize_width = atoi(optarg);
+          break;
+        case 'h':
+          resize_height = atoi(optarg);
+          break;
+        case '?':
+          std::cout << "[ERROR] Unsupported usage" << std::endl;
+          break;
+      }
+    }
+  } else {
+    std::cout
+        << "run with " << argv[0]
+        << " <configFilepath> <input> <inputType> (video "
+           "or cameraId) option(-h <resize_height>) option(-w <resize_width>)"
+        << std::endl;
+    return -1;
+  }
+  Detector detector{filePath};
+  SharedDataLoader dataloader{resize_height, resize_width};
+  if (dataloader.init(input) != 0) {
+    std::cout << "[ERROR] dataloader init error" << std::endl;
+    return -1;
+  }
+
+  std::thread t1(Preview, std::ref(dataloader), std::ref(detector));
+  // std::this_thread::sleep_for(std::chrono::seconds(5));
+  std::thread t2(Detection, std::ref(dataloader), std::ref(detector));
+#ifndef _WIN32
+  setThreadName(t1, "PreviewThread");
+  setThreadName(t2, "DetectionThread");
+#endif
+  t1.join();
+  t2.join();
+  return 0;
+}

demo/object_detection_video_demo.cc

@@ -0,0 +1,93 @@
+#include <iostream>
+
+#include "object_detection.hpp"
+#include "task/vision/object_detection_task.h"
+#ifdef DEBUG
+#include "utils/time.h"
+#endif
+
+#include "utils/utils.h"
+int DetectVideo(const std::string &filepath, const std::string &videoPath,
+                const std::string &srcPath) {
+  std::unique_ptr<objectDetectionTask> objectdetectiontask =
+      std::unique_ptr<objectDetectionTask>(new objectDetectionTask(filepath));
+  if (objectdetectiontask->getInitFlag() != 0) {
+    return -1;
+  }
+  cv::VideoCapture capture(videoPath);
+  if (!capture.isOpened()) {
+    std::cout << "Open video capture failed" << std::endl;
+    return -1;
+  }
+  cv::Mat frame;
+  if (!capture.read(frame)) {
+    std::cout << "Read frame failed" << std::endl;
+    return -1;
+  }
+  double rate = capture.get(cv::CAP_PROP_FPS);
+  int delay = 1000 / rate;
+  int fps = rate;
+  int frameWidth = frame.rows;
+  int frameHeight = frame.cols;
+  cv::VideoWriter writer(srcPath, cv::VideoWriter::fourcc('D', 'I', 'V', 'X'),
+                         fps, cv::Size(frameHeight, frameWidth), 1);
+  while (true) {
+    capture >> frame;
+    if (frame.empty()) {
+      break;
+    }
+    std::vector<Boxi> resultBoxes =
+        objectdetectiontask->Detect(frame).result_bboxes;
+    {
+#ifdef DEBUG
+      TimeWatcher t("|-- Output result");
+#endif
+      for (int i = 0; i < static_cast<int>(resultBoxes.size()); i++) {
+        if (resultBoxes[i].flag) {
+          std::cout << "bbox[" << std::setw(2) << i << "]"
+                    << " "
+                    << "x1y1x2y2: "
+                    << "(" << std::setw(4) << resultBoxes[i].x1 << ","
+                    << std::setw(4) << resultBoxes[i].y1 << "," << std::setw(4)
+                    << resultBoxes[i].x2 << "," << std::setw(4)
+                    << resultBoxes[i].y2 << ")"
+                    << ", "
+                    << "score: " << std::fixed << std::setprecision(3)
+                    << std::setw(4) << resultBoxes[i].score << ", "
+                    << "label_text: " << resultBoxes[i].label_text << std::endl;
+        }
+      }
+    }
+    draw_boxes_inplace(frame, resultBoxes);
+    writer.write(frame);
+    cv::waitKey(
+        delay);  // 因为图像处理需要消耗一定时间,所以图片展示速度比保存视频要慢
+                 // cv::imshow("Detection", frame);
+  }
+  capture.release();
+  writer.release();
+  return 0;
+}
+
+int main(int argc, char *argv[]) {
+  std::string filepath, videoFilepath, dstFilepath;
+#ifdef DEBUG
+  std::cout << "." << std::endl;
+#endif
+  if (argc == 4) {
+    filepath = argv[1];
+    videoFilepath = argv[2];
+    dstFilepath = argv[3];
+    int flag = DetectVideo(filepath, videoFilepath, dstFilepath);
+    if (flag != 0) {
+      std::cout << "[Error] Detect fail" << std::endl;
+    }
+  } else {
+    std::cout << "run with " << argv[0]
+              << " <configFilepath> <videoFilepath> "
+                 "<dstFilepath> (end with .avi)"
+              << std::endl;
+    return -1;
+  }
+  return 0;
+}

demo/pose_estimation.hpp

@@ -0,0 +1,31 @@
+#ifndef SUPPORT_DEMO_POSE_ESTIMATION_HPP_
+#define SUPPORT_DEMO_POSE_ESTIMATION_HPP_
+
+#include <string>   // for std::string
+#include <utility>  // for std::pair
+#include <vector>
+
+#include "opencv2/opencv.hpp"
+#include "task/vision/pose_estimation_types.h"
+
+inline void draw_points_inplace(cv::Mat &img,
+                                const std::vector<PosePoint> &points) {
+  std::vector<std::pair<int, int>> coco_17_joint_links = {
+      {0, 1},   {0, 2},   {1, 3},   {2, 4},  {5, 7},  {7, 9},
+      {6, 8},   {8, 10},  {5, 6},   {5, 11}, {6, 12}, {11, 12},
+      {11, 13}, {13, 15}, {12, 14}, {14, 16}};
+  for (size_t i = 0; i < points.size(); ++i) {
+    cv::circle(img, cv::Point(points[i].x, points[i].y), 2,
+               cv::Scalar{0, 0, 255}, 2, cv::LINE_AA);
+  }
+  for (size_t i = 0; i < coco_17_joint_links.size(); ++i) {
+    std::pair<int, int> joint_links = coco_17_joint_links[i];
+    cv::line(
+        img,
+        cv::Point(points[joint_links.first].x, points[joint_links.first].y),
+        cv::Point(points[joint_links.second].x, points[joint_links.second].y),
+        cv::Scalar{0, 255, 0}, 2, cv::LINE_AA);
+  }
+}
+
+#endif  // SUPPORT_DEMO_POSE_ESTIMATION_HPP_

demo/pose_estimation_demo.cc

@@ -0,0 +1,106 @@
+#include <iomanip>  // for: setprecision
+#include <iostream>
+
+#include "task/vision/object_detection_task.h"
+#include "task/vision/pose_estimation_task.h"
+#include "utils/check_utils.h"
+#include "utils/time.h"
+#include "utils/utils.h"
+
+int main(int argc, char* argv[]) {
+  std::vector<std::pair<int, int>> coco_17_joint_links = {
+      {0, 1},   {0, 2},   {1, 3},   {2, 4},  {5, 7},  {7, 9},
+      {6, 8},   {8, 10},  {5, 6},   {5, 11}, {6, 12}, {11, 12},
+      {11, 13}, {13, 15}, {12, 14}, {14, 16}};
+  std::vector<PosePoint> resultPoints;
+  std::vector<Boxi> resultBoxes;
+  std::string detFilePath, poseFilePath, imageFilepath, saveImgpath;
+  cv::Mat imgRaw, img;
+#ifdef DEBUG
+  std::cout << "." << std::endl;
+#endif
+  if (argc == 5) {
+    detFilePath = argv[1];
+    poseFilePath = argv[2];
+    imageFilepath = argv[3];
+    saveImgpath = argv[4];
+    if (!checkImageFileExtension(imageFilepath) ||
+        !checkImageFileExtension(saveImgpath)) {
+      std::cout << "[ ERROR ] The ImageFilepath is not correct. Make sure you "
+                   "are setting the path to an imgae file (.jpg/.jpeg/.png)"
+                << std::endl;
+      return -1;
+    }
+    if (!exists_check(imageFilepath)) {
+      std::cout << "[ ERROR ] The Image File does not exist. Make sure you are "
+                   "setting the correct path to the file"
+                << std::endl;
+      return -1;
+    }
+    {
+#ifdef DEBUG
+      TimeWatcher t("|-- Load input data");
+#endif
+      imgRaw = cv::imread(imageFilepath);
+      resize_unscale(imgRaw, img, 320, 320);
+    }
+    std::unique_ptr<objectDetectionTask> objectdetectiontask =
+        std::unique_ptr<objectDetectionTask>(
+            new objectDetectionTask(detFilePath));
+    if (objectdetectiontask->getInitFlag() != 0) {
+      return -1;
+    }
+    resultBoxes = objectdetectiontask->Detect(img).result_bboxes;
+    std::unique_ptr<poseEstimationTask> poseestimationtask =
+        std::unique_ptr<poseEstimationTask>(
+            new poseEstimationTask(poseFilePath));
+    if (poseestimationtask->getInitFlag() != 0) {
+      return -1;
+    }
+    Boxi box;
+    for (int i = 0; i < static_cast<int>(resultBoxes.size()); i++) {
+      box = resultBoxes[i];
+      if (box.label != 0) {
+        continue;
+      }
+      resultPoints = poseestimationtask->Estimate(img, box).result_points;
+      if (resultPoints.size()) {
+        int input_height = 320;
+        int input_width = 320;
+        int img_height = imgRaw.rows;
+        int img_width = imgRaw.cols;
+        float resize_ratio = std::min(
+            static_cast<float>(input_height) / static_cast<float>(img_height),
+            static_cast<float>(input_width) / static_cast<float>(img_width));
+        float dw = (input_width - resize_ratio * img_width) / 2;
+        float dh = (input_height - resize_ratio * img_height) / 2;
+        for (int i = 0; i < static_cast<int>(resultPoints.size()); i++) {
+          resultPoints[i].x = (resultPoints[i].x - dw) / resize_ratio;
+          resultPoints[i].y = (resultPoints[i].y - dh) / resize_ratio;
+        }
+      }
+      for (int i = 0; i < static_cast<int>(resultPoints.size()); ++i) {
+        cv::circle(imgRaw, cv::Point(resultPoints[i].x, resultPoints[i].y), 2,
+                   cv::Scalar{0, 0, 255}, 2, cv::LINE_AA);
+      }
+
+      for (int i = 0; i < static_cast<int>(coco_17_joint_links.size()); ++i) {
+        std::pair<int, int> joint_links = coco_17_joint_links[i];
+        cv::line(imgRaw,
+                 cv::Point(resultPoints[joint_links.first].x,
+                           resultPoints[joint_links.first].y),
+                 cv::Point(resultPoints[joint_links.second].x,
+                           resultPoints[joint_links.second].y),
+                 cv::Scalar{0, 255, 0}, 2, cv::LINE_AA);
+      }
+    }
+    cv::imwrite(saveImgpath, imgRaw);
+  } else {
+    std::cout << "run with " << argv[0]
+              << " <detConfigFilepath> <poseConfigFilepath> <imageFilepath> "
+                 "<saveImgpath> "
+              << std::endl;
+    return -1;
+  }
+  return 0;
+}

demo/pose_tracker_stream_demo.cc

@@ -0,0 +1,275 @@
+#include <pthread.h>
+#include <stdlib.h>
+#include <unistd.h>  // for: getopt
+
+#include <algorithm>  // for: swap
+#include <chrono>
+#include <cmath>
+#include <iostream>
+#include <mutex>
+#include <queue>
+#include <string>
+#include <thread>
+
+#include "dataloader.hpp"
+#include "opencv2/opencv.hpp"
+#include "pose_estimation.hpp"
+#include "task/vision/object_detection_task.h"
+#include "task/vision/pose_estimation_task.h"
+#ifdef DEBUG
+#include "utils/time.h"
+#endif
+
+#include "utils/utils.h"
+
+class Tracker {
+ public:
+  Tracker(const std::string& detFilePath, const std::string& poseFilePath) {
+    detFilePath_ = detFilePath;
+    poseFilePath_ = poseFilePath;
+  }
+  ~Tracker() {}
+  // 初始化/反初始化
+  int init() {
+    objectdetectiontask_ = std::unique_ptr<objectDetectionTask>(
+        new objectDetectionTask(detFilePath_));
+    poseestimationtask_ = std::unique_ptr<poseEstimationTask>(
+        new poseEstimationTask(poseFilePath_));
+    return get_init_flag();
+  }
+
+  int get_init_flag() {
+    return (objectdetectiontask_->getInitFlag() ||
+            poseestimationtask_->getInitFlag());
+  }
+
+  int uninit() { return 0; }
+
+  // 推理
+  int infer(cv::Mat frame) {
+    if (frame.empty()) {
+      return -1;
+    }
+    ObjectDetectionResult objs_temp = objectdetectiontask_->Detect(frame);
+    int count{0}, flag{0};
+    for (count = 0; count < static_cast<int>(objs_temp.result_bboxes.size());
+         count++) {
+      if (objs_temp.result_bboxes[count].label == 0) {
+        flag = 1;
+        break;
+      }
+    }
+    if (flag) {
+      PoseEstimationResult poses_temp =
+          poseestimationtask_->Estimate(frame, objs_temp.result_bboxes[count]);
+      poses_mutex_.lock();
+      poses_array_.push(poses_temp);  // 直接替换掉当前的 objs_array_
+      poses_mutex_.unlock();
+      return poses_array_.size();
+    } else {
+      return 0;
+    }
+  }
+
+  // 查询检测结果
+  int estimated() { return poses_array_.size(); }
+
+  // 移走检测结果
+  struct PoseEstimationResult get_pose() {
+    struct PoseEstimationResult poses_moved;
+    poses_mutex_.lock();
+    poses_moved = poses_array_.back();
+    std::queue<struct PoseEstimationResult> empty;
+    std::swap(empty, poses_array_);
+    poses_mutex_.unlock();
+    return poses_moved;
+  }
+
+ private:
+  std::mutex poses_mutex_;
+  std::queue<struct PoseEstimationResult> poses_array_;
+  std::unique_ptr<objectDetectionTask> objectdetectiontask_;
+  std::unique_ptr<poseEstimationTask> poseestimationtask_;
+  std::string poseFilePath_;
+  std::string detFilePath_;
+  std::string labelFilepath_;
+};
+
+// 检测线程
+void Track(DataLoader& dataloader, Tracker& tracker) {
+  if (tracker.init() != 0) {
+    std::cout << "[ERROR] tracker init error" << std::endl;
+    return;
+  }
+  cv::Mat frame;
+  while (dataloader.ifenable()) {
+    auto start = std::chrono::steady_clock::now();
+    frame = dataloader.peek_frame();  // 取(拷贝)一帧数据
+    if ((frame).empty()) {
+      dataloader.set_disable();
+      continue;
+    }
+    int flag = tracker.infer(frame);  // 推理并保存检测结果
+    auto end = std::chrono::steady_clock::now();
+    auto detection_duration =
+        std::chrono::duration_cast<std::chrono::milliseconds>(end - start);
+    dataloader.set_detection_fps(1000 / (detection_duration.count()));
+    if (flag == 0) {
+      std::cout << "[Warning] unable to catch person" << std::endl;  // 无人
+    }
+    if (flag == -1) {
+      std::cout << "[Error] infer frame failed" << std::endl;
+      break;  // 摄像头结束拍摄或者故障
+    }
+  }
+  std::cout << "track thread quit" << std::endl;
+}
+
+// 预览线程
+void Preview(DataLoader& dataloader, Tracker& tracker) {
+  cv::Mat frame;
+  PoseEstimationResult poses;
+  auto now = std::chrono::steady_clock::now();
+  poses.timestamp = now;
+  int count = 0;
+  int dur = 0;
+  int enable_show = 1;
+  const char* showfps = getenv("SHOWFPS");
+  const char* show = getenv("SHOW");
+  if (show && strcmp(show, "-1") == 0) {
+    enable_show = -1;
+  }
+  while (dataloader.ifenable()) {
+    auto start = std::chrono::steady_clock::now();
+    frame = dataloader.fetch_frame();  // 取(搬走)一帧数据
+    if ((frame).empty()) {
+      dataloader.set_disable();
+      break;
+    }
+    if (tracker.estimated())  // 判断原因: detector.detected 不用锁,
+                              // detector.get_object 需要锁;
+    {
+      // 是否有检测结果
+      poses = tracker.get_pose();  // 取(搬走)检测结果(移动赋值)
+      if (poses.result_points.size()) {
+        int input_height = dataloader.get_resize_height();
+        int input_width = dataloader.get_resize_width();
+        int img_height = frame.rows;
+        int img_width = frame.cols;
+        float resize_ratio = std::min(
+            static_cast<float>(input_height) / static_cast<float>(img_height),
+            static_cast<float>(input_width) / static_cast<float>(img_width));
+        float dw = (input_width - resize_ratio * img_width) / 2;
+        float dh = (input_height - resize_ratio * img_height) / 2;
+        for (int i = 0; i < static_cast<int>(poses.result_points.size()); i++) {
+          poses.result_points[i].x =
+              (poses.result_points[i].x - dw) / resize_ratio;
+          poses.result_points[i].y =
+              (poses.result_points[i].y - dh) / resize_ratio;
+        }
+      }
+    }
+    // 调用 detector.detected 和 detector.get_object 期间,
+    // 检测结果依然可能被刷新
+    now = std::chrono::steady_clock::now();
+    auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(
+        now - poses.timestamp);
+    if (duration.count() < 1000 && poses.result_points.size()) {
+      draw_points_inplace((frame), poses.result_points);  // 画框
+    }
+    int preview_fps = dataloader.get_preview_fps();
+    int detection_fps = dataloader.get_detection_fps();
+    if (showfps != nullptr) {
+      cv::putText(frame, "preview fps: " + std::to_string(preview_fps),
+                  cv::Point(0, 15), cv::FONT_HERSHEY_SIMPLEX, 0.5f,
+                  cv::Scalar(0, 255, 0), 1, cv::LINE_AA);
+      cv::putText(frame, "detection fps: " + std::to_string(detection_fps),
+                  cv::Point(500, 15), cv::FONT_HERSHEY_SIMPLEX, 0.5f,
+                  cv::Scalar(0, 255, 0), 1, cv::LINE_AA);
+    }
+    if (enable_show != -1) {
+      cv::imshow("Track", (frame));
+      cv::waitKey(10);
+    }
+    auto end = std::chrono::steady_clock::now();
+    auto preview_duration =
+        std::chrono::duration_cast<std::chrono::milliseconds>(end - start);
+    count++;
+    dur = dur + preview_duration.count();
+    if (dur >= 1000) {
+      dataloader.set_preview_fps(count);
+      dur = 0;
+      count = 0;
+    }
+    if (enable_show != -1) {
+      if (cv::getWindowProperty("Track", cv::WND_PROP_VISIBLE) < 1) {
+        dataloader.set_disable();
+        break;
+      }
+    }
+  }
+  std::cout << "preview thread quit" << std::endl;
+  if (enable_show != -1) {
+    cv::destroyAllWindows();
+  }
+}
+
+#ifndef _WIN32
+void setThreadName(std::thread& thread, const char* name) {
+  pthread_setname_np(thread.native_handle(), name);
+}
+#endif
+
+int main(int argc, char* argv[]) {
+  std::string detFilePath, poseFilePath, input, inputType;
+  int resize_height{320}, resize_width{320};
+  if (argc == 5) {
+    detFilePath = argv[1];
+    poseFilePath = argv[2];
+    input = argv[3];
+    inputType = argv[4];
+  } else if (argc > 5) {
+    detFilePath = argv[1];
+    poseFilePath = argv[2];
+    input = argv[3];
+    inputType = argv[4];
+    int o;
+    const char* optstring = "w:h:";
+    while ((o = getopt(argc, argv, optstring)) != -1) {
+      switch (o) {
+        case 'w':
+          resize_width = atoi(optarg);
+          break;
+        case 'h':
+          resize_height = atoi(optarg);
+          break;
+        case '?':
+          std::cout << "[ERROR] Unsupported usage" << std::endl;
+          break;
+      }
+    }
+  } else {
+    std::cout << "run with " << argv[0]
+              << " <detFilepath> <poseFilepath> <input> <inputType> (video or "
+                 "cameraId option(-h <resize_height>) option(-w <resize_width>)"
+              << std::endl;
+    return -1;
+  }
+  Tracker tracker{detFilePath, poseFilePath};
+  SharedDataLoader dataloader{resize_height, resize_width};
+  if (dataloader.init(input) != 0) {
+    std::cout << "[ERROR] dataloader init error" << std::endl;
+    return -1;
+  }
+
+  std::thread t1(Preview, std::ref(dataloader), std::ref(tracker));
+  // std::this_thread::sleep_for(std::chrono::seconds(5));
+  std::thread t2(Track, std::ref(dataloader), std::ref(tracker));
+#ifndef _WIN32
+  setThreadName(t1, "PreviewThread");
+  setThreadName(t2, "TrackerThread");
+#endif
+  t1.join();
+  t2.join();
+  return 0;
+}

demo/utils/check_utils.h

@@ -0,0 +1,23 @@
+#ifndef SUPPORT_DEMO_UTILS_CHECK_UTILS_H_
+#define SUPPORT_DEMO_UTILS_CHECK_UTILS_H_
+#include <cstdlib>
+#include <fstream>
+#include <iostream>
+#include <string>
+
+static bool checkImageFileExtension(const std::string& filename) {
+  size_t pos = filename.rfind('.');
+  if (filename.empty()) {
+    std::cout << "[ ERROR ] The Image file path is empty" << std::endl;
+    return false;
+  }
+  if (pos == std::string::npos) return false;
+  std::string ext = filename.substr(pos + 1);
+  if (ext == "jpeg" || ext == "jpg" || ext == "png") {
+    return true;
+  } else {
+    return false;
+  }
+}
+
+#endif  // SUPPORT_DEMO_UTILS_CHECK_UTILS_H_

demo/utils/win_getopt/mb/getopt.cc

@@ -0,0 +1,117 @@
+/*	$NetBSD: getopt.c,v 1.29 2014/06/05 22:00:22 christos Exp $	*/
+
+/*-
+ * SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright (c) 1987, 1993, 1994
+ *	The Regents of the University of California.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+int opterr = 1, /* if error message should be printed */
+    optind = 1, /* index into parent argv vector */
+    optopt,     /* character checked for validity */
+    optreset;   /* reset getopt */
+char* optarg;   /* argument associated with option */
+
+#define BADCH (int)'?'
+#define BADARG (int)':'
+#define EMSG ""
+
+/*
+ * getopt --
+ *	Parse argc/argv argument vector.
+ */
+int getopt(int nargc, char* const nargv[], const char* ostr) {
+  static char* place = EMSG; /* option letter processing */
+  char* oli;                 /* option letter list index */
+
+  if (optreset || *place == 0) { /* update scanning pointer */
+    optreset = 0;
+    place = nargv[optind];
+    if (optind >= nargc || *place++ != '-') {
+      /* Argument is absent or is not an option */
+      place = EMSG;
+      return (-1);
+    }
+    optopt = *place++;
+    if (optopt == '-' && *place == 0) {
+      /* "--" => end of options */
+      ++optind;
+      place = EMSG;
+      return (-1);
+    }
+    if (optopt == 0) {
+      /* Solitary '-', treat as a '-' option
+                           if the program (eg su) is looking for it. */
+      place = EMSG;
+      if (strchr(ostr, '-') == NULL) return (-1);
+      optopt = '-';
+    }
+  } else
+    optopt = *place++;
+
+  /* See if option letter is one the caller wanted... */
+  if (optopt == ':' || (oli = (char*)strchr(ostr, optopt)) == NULL) {
+    if (*place == 0) ++optind;
+    if (opterr && *ostr != ':') (void)fprintf(stderr, "illegal option -- %c\n", optopt);
+    return (BADCH);
+  }
+
+  /* Does this option need an argument? */
+  if (oli[1] != ':') {
+    /* don't need argument */
+    optarg = NULL;
+    if (*place == 0) ++optind;
+  } else {
+    /* Option-argument is either the rest of this argument or the
+                   entire next argument. */
+    if (*place)
+      optarg = place;
+    else if (oli[2] == ':')
+      /*
+       * GNU Extension, for optional arguments if the rest of
+       * the argument is empty, we return NULL
+       */
+      optarg = NULL;
+    else if (nargc > ++optind)
+      optarg = nargv[optind];
+    else {
+      /* option-argument absent */
+      place = EMSG;
+      if (*ostr == ':') return (BADARG);
+      if (opterr) (void)fprintf(stderr, "option requires an argument -- %c\n", optopt);
+      return (BADCH);
+    }
+    place = EMSG;
+    ++optind;
+  }
+  return (optopt); /* return option letter */
+}

demo/utils/win_getopt/mb/include/getopt.h

@@ -0,0 +1,42 @@
+/*	$NetBSD: getopt.c,v 1.29 2014/06/05 22:00:22 christos Exp $	*/
+
+/*-
+ * SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright (c) 1987, 1993, 1994
+ *	The Regents of the University of California.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#pragma once
+
+extern int opterr,   /* if error message should be printed */
+    optind,          /* index into parent argv vector */
+    optopt,          /* character checked for validity */
+    optreset;        /* reset getopt */
+extern char* optarg; /* argument associated with option */
+
+int getopt(int nargc, char* const nargv[], const char* ostr);

demo/utils/win_getopt/mb/include/unistd.h

@@ -0,0 +1,5 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+#include "getopt.h"

demo/utils/win_getopt/wide/getopt.cc

@@ -0,0 +1,117 @@
+/*	$NetBSD: getopt.c,v 1.29 2014/06/05 22:00:22 christos Exp $	*/
+
+/*-
+ * SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright (c) 1987, 1993, 1994
+ *	The Regents of the University of California.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+int opterr = 1,        /* if error message should be printed */
+    optind = 1,        /* index into parent argv vector */
+    optopt,            /* character checked for validity */
+    optreset;          /* reset getopt */
+const wchar_t* optarg; /* argument associated with option */
+
+#define BADCH (int)L'?'
+#define BADARG (int)L':'
+#define EMSG L""
+
+/*
+ * getopt --
+ *	Parse argc/argv argument vector.
+ */
+int getopt(int nargc, wchar_t* const nargv[], const wchar_t* ostr) {
+  static const wchar_t* place = EMSG; /* option letter processing */
+  wchar_t* oli;                       /* option letter list index */
+
+  if (optreset || *place == 0) { /* update scanning pointer */
+    optreset = 0;
+    place = nargv[optind];
+    if (optind >= nargc || *place++ != L'-') {
+      /* Argument is absent or is not an option */
+      place = EMSG;
+      return (-1);
+    }
+    optopt = *place++;
+    if (optopt == L'-' && *place == 0) {
+      /* "--" => end of options */
+      ++optind;
+      place = EMSG;
+      return (-1);
+    }
+    if (optopt == 0) {
+      /* Solitary '-', treat as a '-' option
+                           if the program (eg su) is looking for it. */
+      place = EMSG;
+      if (wcschr(ostr, L'-') == NULL) return (-1);
+      optopt = '-';
+    }
+  } else
+    optopt = *place++;
+
+  /* See if option letter is one the caller wanted... */
+  if (optopt == L':' || (oli = (wchar_t*)wcschr(ostr, (wchar_t)optopt)) == NULL) {
+    if (*place == 0) ++optind;
+    if (opterr && *ostr != L':') (void)fprintf(stderr, "illegal option -- %c\n", optopt);
+    return (BADCH);
+  }
+
+  /* Does this option need an argument? */
+  if (oli[1] != L':') {
+    /* don't need argument */
+    optarg = NULL;
+    if (*place == 0) ++optind;
+  } else {
+    /* Option-argument is either the rest of this argument or the
+                   entire next argument. */
+    if (*place)
+      optarg = place;
+    else if (oli[2] == L':')
+      /*
+       * GNU Extension, for optional arguments if the rest of
+       * the argument is empty, we return NULL
+       */
+      optarg = NULL;
+    else if (nargc > ++optind)
+      optarg = nargv[optind];
+    else {
+      /* option-argument absent */
+      place = EMSG;
+      if (*ostr == L':') return (BADARG);
+      if (opterr) (void)fprintf(stderr, "option requires an argument -- %c\n", optopt);
+      return (BADCH);
+    }
+    place = EMSG;
+    ++optind;
+  }
+  return (optopt); /* return option letter */
+}

demo/utils/win_getopt/wide/include/getopt.h

@@ -0,0 +1,42 @@
+/*	$NetBSD: getopt.c,v 1.29 2014/06/05 22:00:22 christos Exp $	*/
+
+/*-
+ * SPDX-License-Identifier: BSD-3-Clause
+ *
+ * Copyright (c) 1987, 1993, 1994
+ *	The Regents of the University of California.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#pragma once
+
+extern int opterr,            /* if error message should be printed */
+    optind,                   /* index into parent argv vector */
+    optopt,                   /* character checked for validity */
+    optreset;                 /* reset getopt */
+extern const wchar_t* optarg; /* argument associated with option */
+
+int getopt(int nargc, wchar_t* const nargv[], const wchar_t* ostr);

include/task/vision/image_classification_task.h

@@ -0,0 +1,24 @@
+#ifndef SUPPORT_INCLUDE_TASK_VISION_IMAGE_CLASSIFICATION_TASK_H_
+#define SUPPORT_INCLUDE_TASK_VISION_IMAGE_CLASSIFICATION_TASK_H_
+
+#include <memory>  // for: shared_ptr
+#include <string>
+
+#include "opencv2/opencv.hpp"
+#include "task/vision/image_classification_types.h"
+
+class imageClassificationTask {
+ public:
+  imageClassificationTask(const std::string& filePath,
+                          const std::string& labelFilepath);
+  ~imageClassificationTask() = default;
+  ImageClassificationResult Classify(const cv::Mat& img_raw);
+  int getInitFlag();
+
+ private:
+  class impl;
+  std::shared_ptr<impl> pimpl_;
+  int init_flag_;
+};
+
+#endif  // SUPPORT_INCLUDE_TASK_VISION_IMAGE_CLASSIFICATION_TASK_H_

include/task/vision/image_classification_types.h

@@ -0,0 +1,12 @@
+#ifndef SUPPORT_INCLUDE_TASK_VISION_IMAGE_CLASSIFICATION_TYPES_H_
+#define SUPPORT_INCLUDE_TASK_VISION_IMAGE_CLASSIFICATION_TYPES_H_
+
+#include <chrono>
+#include <string>
+
+struct ImageClassificationResult {
+  std::string label_text;
+  int label;
+  float score;
+};
+#endif  // SUPPORT_INCLUDE_TASK_VISION_IMAGE_CLASSIFICATION_TYPES_H_

include/task/vision/object_detection_task.h

@@ -0,0 +1,30 @@
+#ifndef SUPPORT_INCLUDE_TASK_VISION_OBJECT_DETECTION_TASK_H_
+#define SUPPORT_INCLUDE_TASK_VISION_OBJECT_DETECTION_TASK_H_
+
+#include <memory>  // for: shared_ptr
+#include <string>
+#include <vector>  //for: vector
+
+#include "opencv2/opencv.hpp"
+#include "task/vision/object_detection_types.h"
+
+class objectDetectionTask {
+ public:
+  objectDetectionTask(const std::string &filePath,
+                      const std::string &labelFilepath);
+  explicit objectDetectionTask(const std::string &filePath);
+  ~objectDetectionTask() = default;
+  ObjectDetectionResult Detect(const cv::Mat &raw_img);
+  ObjectDetectionResult Detect(
+      const std::vector<std::vector<float>> &input_tensors,
+      const int img_height, const int img_width);
+  std::vector<std::vector<float>> Process(const cv::Mat &img_raw);
+  int getInitFlag();
+
+ private:
+  class impl;
+  std::shared_ptr<impl> pimpl_;
+  int init_flag_;
+};
+
+#endif  // SUPPORT_INCLUDE_TASK_VISION_OBJECT_DETECTION_TASK_H_

include/task/vision/object_detection_types.h

@@ -0,0 +1,165 @@
+#ifndef SUPPORT_INCLUDE_TASK_VISION_OBJECT_DETECTION_TYPES_H_
+#define SUPPORT_INCLUDE_TASK_VISION_OBJECT_DETECTION_TYPES_H_
+
+#include <chrono>
+#include <limits>  // for numeric_limits<>
+#include <type_traits>
+#include <vector>
+
+#include "opencv2/opencv.hpp"
+
+template <typename _T1 = float, typename _T2 = float>
+static inline void __assert_type() {
+  static_assert(
+      std::is_pod<_T1>::value && std::is_pod<_T2>::value &&
+          std::is_floating_point<_T2>::value &&
+          (std::is_integral<_T1>::value || std::is_floating_point<_T1>::value),
+      "not support type.");
+}  // only support for some specific types. check at compile-time.
+
+// bounding box.
+template <typename T1 = float, typename T2 = float>
+struct BoundingBoxType {
+  typedef T1 value_type;
+  typedef T2 score_type;
+  value_type x1;
+  value_type y1;
+  value_type x2;
+  value_type y2;
+  score_type score;
+  const char *label_text;
+  unsigned int label;  // for general object detection.
+  bool flag;           // future use.
+  // convert type.
+  template <typename O1, typename O2 = score_type>
+  BoundingBoxType<O1, O2> convert_type() const;
+
+  template <typename O1, typename O2 = score_type>
+  value_type iou_of(const BoundingBoxType<O1, O2> &other) const;
+
+  value_type width() const;
+
+  value_type height() const;
+
+  value_type area() const;
+
+  cv::Rect rect() const;
+
+  cv::Point2i tl() const;
+
+  cv::Point2i bl() const;
+
+  cv::Point2i rb() const;
+
+  BoundingBoxType()
+      : x1(static_cast<value_type>(0)),
+        y1(static_cast<value_type>(0)),
+        x2(static_cast<value_type>(0)),
+        y2(static_cast<value_type>(0)),
+        score(static_cast<score_type>(0)),
+        label_text(nullptr),
+        label(0),
+        flag(false) {
+    __assert_type<value_type, score_type>();
+  }
+};  // End BoundingBox.
+
+typedef BoundingBoxType<int, float> Boxi;
+typedef BoundingBoxType<float, float> Boxf;
+typedef BoundingBoxType<double, double> Boxd;
+
+/* implementation for 'BoundingBox'. */
+template <typename T1, typename T2>
+template <typename O1, typename O2>
+inline BoundingBoxType<O1, O2> BoundingBoxType<T1, T2>::convert_type() const {
+  typedef O1 other_value_type;
+  typedef O2 other_score_type;
+  __assert_type<other_value_type, other_score_type>();
+  __assert_type<value_type, score_type>();
+  BoundingBoxType<other_value_type, other_score_type> other;
+  other.x1 = static_cast<other_value_type>(x1);
+  other.y1 = static_cast<other_value_type>(y1);
+  other.x2 = static_cast<other_value_type>(x2);
+  other.y2 = static_cast<other_value_type>(y2);
+  other.score = static_cast<other_score_type>(score);
+  other.label_text = label_text;
+  other.label = label;
+  other.flag = flag;
+  return other;
+}
+
+template <typename T1, typename T2>
+template <typename O1, typename O2>
+inline typename BoundingBoxType<T1, T2>::value_type
+BoundingBoxType<T1, T2>::iou_of(const BoundingBoxType<O1, O2> &other) const {
+  BoundingBoxType<value_type, score_type> tbox =
+      other.template convert_type<value_type, score_type>();
+  value_type inner_x1 = x1 > tbox.x1 ? x1 : tbox.x1;
+  value_type inner_y1 = y1 > tbox.y1 ? y1 : tbox.y1;
+  value_type inner_x2 = x2 < tbox.x2 ? x2 : tbox.x2;
+  value_type inner_y2 = y2 < tbox.y2 ? y2 : tbox.y2;
+  value_type inner_h = inner_y2 - inner_y1 + static_cast<value_type>(1.0f);
+  value_type inner_w = inner_x2 - inner_x1 + static_cast<value_type>(1.0f);
+  if (inner_h <= static_cast<value_type>(0.f) ||
+      inner_w <= static_cast<value_type>(0.f))
+    return std::numeric_limits<value_type>::min();
+  value_type inner_area = inner_h * inner_w;
+  return static_cast<value_type>(inner_area /
+                                 (area() + tbox.area() - inner_area));
+}
+
+template <typename T1, typename T2>
+inline cv::Rect BoundingBoxType<T1, T2>::rect() const {
+  __assert_type<value_type, score_type>();
+  auto boxi = this->template convert_type<int>();
+  return cv::Rect(boxi.x1, boxi.y1, boxi.width(), boxi.height());
+}
+
+template <typename T1, typename T2>
+inline cv::Point2i BoundingBoxType<T1, T2>::tl() const {
+  __assert_type<value_type, score_type>();
+  auto boxi = this->template convert_type<int>();
+  return cv::Point2i(boxi.x1, boxi.y1 + 10);
+}
+
+template <typename T1, typename T2>
+inline cv::Point2i BoundingBoxType<T1, T2>::bl() const {
+  __assert_type<value_type, score_type>();
+  auto boxi = this->template convert_type<int>();
+  return cv::Point2i(boxi.x1, boxi.y2);
+}
+
+template <typename T1, typename T2>
+inline cv::Point2i BoundingBoxType<T1, T2>::rb() const {
+  __assert_type<value_type, score_type>();
+  auto boxi = this->template convert_type<int>();
+  return cv::Point2i(boxi.x2, boxi.y2);
+}
+
+template <typename T1, typename T2>
+inline typename BoundingBoxType<T1, T2>::value_type
+BoundingBoxType<T1, T2>::width() const {
+  __assert_type<value_type, score_type>();
+  return (x2 - x1 + static_cast<value_type>(1));
+}
+
+template <typename T1, typename T2>
+inline typename BoundingBoxType<T1, T2>::value_type
+BoundingBoxType<T1, T2>::height() const {
+  __assert_type<value_type, score_type>();
+  return (y2 - y1 + static_cast<value_type>(1));
+}
+
+template <typename T1, typename T2>
+inline typename BoundingBoxType<T1, T2>::value_type
+BoundingBoxType<T1, T2>::area() const {
+  __assert_type<value_type, score_type>();
+  return std::abs<value_type>(width() * height());
+}
+
+struct ObjectDetectionResult {
+  std::vector<Boxi> result_bboxes;
+  std::chrono::time_point< std::chrono::steady_clock > timestamp;
+};
+
+#endif  // SUPPORT_INCLUDE_TASK_VISION_OBJECT_DETECTION_TYPES_H_

include/task/vision/pose_estimation_task.h

@@ -0,0 +1,24 @@
+#ifndef SUPPORT_INCLUDE_TASK_VISION_POSE_ESTIMATION_TASK_H_
+#define SUPPORT_INCLUDE_TASK_VISION_POSE_ESTIMATION_TASK_H_
+
+#include <memory>  // for: shared_ptr
+#include <string>
+
+#include "opencv2/opencv.hpp"
+#include "task/vision/object_detection_types.h"
+#include "task/vision/pose_estimation_types.h"
+
+class poseEstimationTask {
+ public:
+  explicit poseEstimationTask(const std::string &filePath);
+  ~poseEstimationTask() = default;
+  int getInitFlag();
+  PoseEstimationResult Estimate(const cv::Mat &raw_img, const Boxi &box);
+
+ private:
+  class impl;
+  std::shared_ptr<impl> pimpl_;
+  int init_flag_;
+};
+
+#endif  // SUPPORT_INCLUDE_TASK_VISION_POSE_ESTIMATION_TASK_H_

include/task/vision/pose_estimation_types.h

@@ -0,0 +1,26 @@
+#ifndef SUPPORT_INCLUDE_TASK_VISION_POSE_ESTIMATION_TYPES_H_
+#define SUPPORT_INCLUDE_TASK_VISION_POSE_ESTIMATION_TYPES_H_
+
+#include <chrono>  //for chrono
+#include <vector>  //for vector
+
+struct PosePoint {
+  int x;
+  int y;
+  float score;
+
+  PosePoint() {
+    x = 0;
+    y = 0;
+    score = 0.0;
+  }
+};
+
+typedef PosePoint Vector2D;
+
+struct PoseEstimationResult {
+  std::vector<PosePoint> result_points;
+  std::chrono::time_point< std::chrono::steady_clock > timestamp;
+};
+
+#endif  // SUPPORT_INCLUDE_TASK_VISION_POSE_ESTIMATION_TYPES_H_

include/utils/time.h

@@ -0,0 +1,30 @@
+#ifndef SUPPORT_INCLUDE_UTILS_TIME_H_
+#define SUPPORT_INCLUDE_UTILS_TIME_H_
+
+#include <chrono>
+#include <iomanip>
+#include <iostream>
+#include <string>
+
+class TimeWatcher {
+ public:
+  TimeWatcher(const std::string& name) : msg_(name) {
+    start_ = std::chrono::steady_clock::now();
+  }
+  int64_t DurationWithMicroSecond() {
+    return std::chrono::duration_cast<std::chrono::milliseconds>(end_ - start_)
+        .count();
+  }
+  ~TimeWatcher() {
+    end_ = std::chrono::steady_clock::now();
+    std::cout << msg_ << " consumes " << std::fixed << std::setprecision(0)
+              << DurationWithMicroSecond() << " ms" << std::endl;
+  }
+
+ private:
+  std::string msg_;
+  std::chrono::steady_clock::time_point start_;
+  std::chrono::steady_clock::time_point end_;
+};
+
+#endif  // SUPPORT_INCLUDE_UTILS_TIME_H_

include/utils/utils.h

@@ -0,0 +1,16 @@
+#ifndef SUPPORT_INCLUDE_UTILS_UTILS_H_
+#define SUPPORT_INCLUDE_UTILS_UTILS_H_
+
+#include <sys/stat.h>
+
+#include <algorithm>
+#include <string>
+
+#include "opencv2/opencv.hpp"
+
+bool exists_check(const std::string &name);
+
+void resize_unscale(const cv::Mat &mat, cv::Mat &mat_rs, int target_height,
+                    int target_width);
+
+#endif  // SUPPORT_INCLUDE_UTILS_UTILS_H_

rootfs/etc/xdg/autostart/object-detection.desktop

@@ -0,0 +1,7 @@
+[Desktop Entry]
+Type=Application
+Name=object-detection
+Exec=/usr/bin/bianbu-ai-autotools desktop /usr/share/applications/object-detection.desktop
+Terminal=false
+NoDisplay=true
+StartupNotify=false

rootfs/etc/xdg/autostart/pose-tracker.desktop

@@ -0,0 +1,7 @@
+[Desktop Entry]
+Type=Application
+Name=pose-tracker
+Exec=/usr/bin/bianbu-ai-autotools desktop /usr/share/applications/pose-tracker.desktop
+Terminal=false
+NoDisplay=true
+StartupNotify=false

rootfs/usr/bin/bianbu-ai-autotools

@@ -0,0 +1,52 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Copyright (c) 2023, spacemit.com, Inc. All Rights Reserved
+#
+# =============================================================
+#
+# Author: hongjie.qin@spacemit.com
+# Brief:  Bianbu AI Auto Toolkit.
+#
+
+#set -e
+#set -x
+
+cmd=$1
+shift
+
+#xdg-user-dirs-update
+test -f ${XDG_CONFIG_HOME:-~/.config}/user-dirs.dirs && . ${XDG_CONFIG_HOME:-~/.config}/user-dirs.dirs
+
+function auto_desktop() {
+  desktop=$1
+  shortcut=$(basename $desktop)
+  if [[ ! -f ${XDG_CONFIG_HOME:-~/.config}/bianbu-ai-support/applications/${shortcut%.desktop}-initial-setup-done ]]; then
+    xdg-desktop-icon install --novendor ${desktop}
+    gio set $(xdg-user-dir DESKTOP)/${shortcut} metadata::trusted true
+  fi
+  if [[ -f $(xdg-user-dir DESKTOP)/${shortcut} ]]; then
+    mkdir -p ${XDG_CONFIG_HOME:-~/.config}/bianbu-ai-support/applications
+    touch ${XDG_CONFIG_HOME:-~/.config}/bianbu-ai-support/applications/${shortcut%.desktop}-initial-setup-done
+  fi
+}
+
+if [[ $cmd =~ "desktop" ]]; then
+  auto_desktop $@
+fi

rootfs/usr/share/ai-support/models/coco.txt

@@ -0,0 +1,80 @@
+person
+bicycle
+car
+motorcycle
+airplane
+bus
+train
+truck
+boat
+traffic light
+fire hydrant
+stop sign
+parking meter
+bench
+bird
+cat
+dog
+horse
+sheep
+cow
+elephant
+bear
+zebra
+giraffe
+backpack
+umbrella
+handbag
+tie
+suitcase
+frisbee
+skis
+snowboard
+sports ball
+kite
+baseball bat
+baseball glove
+skateboard
+surfboard
+tennis racket
+bottle
+wine glass
+cup
+fork
+knife
+spoon
+bowl
+banana
+apple
+sandwich
+orange
+broccoli
+carrot
+hot dog
+pizza
+donut
+cake
+chair
+couch
+potted plant
+bed
+dining table
+toilet
+tv
+laptop
+mouse
+remote
+keyboard
+cell phone
+microwave
+oven
+toaster
+sink
+refrigerator
+book
+clock
+vase
+scissors
+teddy bear
+hair drier
+toothbrush

rootfs/usr/share/ai-support/models/rtmpose.json

@@ -0,0 +1,8 @@
+{
+    "name": "Rtmpose",
+    "instance_name": "pose-estimation-inference",
+    "model_path": "/usr/share/ai-support/models/rtmpose-t.q.onnx",
+    "label_path": "/usr/share/ai-support/models/coco.txt",
+    "intra_threads_num": 2,
+    "graph_optimization_level": ""
+}

rootfs/usr/share/ai-support/models/yolov6.json

@@ -0,0 +1,9 @@
+{
+    "name": "Yolov6",
+    "instance_name": "object-detection-inference",
+    "model_path": "/usr/share/ai-support/models/yolov6p5_n.q.onnx",
+    "label_path": "/usr/share/ai-support/models/coco.txt",
+    "intra_threads_num": 2,
+    "graph_optimization_level": "",
+    "score_threshold": 0.3
+}

rootfs/usr/share/applications/object-detection.desktop

@@ -0,0 +1,8 @@
+[Desktop Entry]
+Type=Application
+#Encoding=UTF-8
+Version=1.0.3
+Name=object-detection
+Exec=/usr/bin/detection_stream_demo /usr/share/ai-support/models/yolov6.json 0 cameraId
+Terminal=true
+Icon=/usr/share/icons/bianbu-ai-demo/object-detection.png

rootfs/usr/share/applications/pose-tracker.desktop

@@ -0,0 +1,8 @@
+[Desktop Entry]
+Type=Application
+#Encoding=UTF-8
+Version=1.0.3
+Name=pose-tracker
+Exec=/usr/bin/tracker_stream_demo /usr/share/ai-support/models/yolov6.json /usr/share/ai-support/models/rtmpose.json 0 cameraId
+Terminal=true
+Icon=/usr/share/icons/bianbu-ai-demo/pose-tracker.png

src/CMakeLists.txt

@@ -0,0 +1,91 @@
+file(READ ${CMAKE_SOURCE_DIR}/VERSION_NUMBER VERSION_CONTENT)
+string(STRIP "${VERSION_CONTENT}" VERSION_CONTENT)
+project(bianbuai-support-library VERSION ${VERSION_CONTENT})
+
+set(CMAKE_CXX_STANDARD 14)
+#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fpermissive")
+
+if (WIN32)
+  set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
+else()
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Werror")
+endif()
+
+# TODO: update cc files with '#ifndef NDEBUG'
+set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DDEBUG")
+
+set(OPENCV_LIBS opencv_core opencv_imgproc)
+# To find OpenCV, one may need to set OpenCV_DIR variable to the
+# absolute path to the directory containing OpenCVConfig.cmake file.
+# Otherwise, try to set OPENCV_INC and OPENCV_LIB variables via the
+# command line or GUI.
+if ((NOT DEFINED OPENCV_INC OR OPENCV_INC STREQUAL "") OR ((NOT DEFINED OPENCV_LIB OR OPENCV_LIB STREQUAL "")))
+find_package(OpenCV REQUIRED COMPONENTS ${OPENCV_LIBS})
+if (OpenCV_FOUND)
+  if (NOT DEFINED OPENCV_INC OR OPENCV_INC STREQUAL "")
+    set(OPENCV_INC "${OpenCV_INCLUDE_DIRS}")
+  endif()
+  if (NOT DEFINED OPENCV_LIB OR OPENCV_LIB STREQUAL "")
+    get_target_property(OpenCV_LIB_PATH opencv_core LOCATION)
+    get_filename_component(OPENCV_LIB ${OpenCV_LIB_PATH} DIRECTORY)
+  endif()
+endif()
+endif()
+
+# Check Required Env
+if (NOT DEFINED ORT_HOME OR ORT_HOME STREQUAL "")
+  message(FATAL_ERROR "Env 'ORT_HOME' not defined for platform ${CMAKE_GENERATOR_PLATFORM}")
+endif()
+if (NOT DEFINED OPENCV_INC OR OPENCV_INC STREQUAL "")
+  message(FATAL_ERROR "OpenCV include dirs not found for platform ${CMAKE_GENERATOR_PLATFORM}")
+endif()
+if (NOT DEFINED OPENCV_LIB OR OPENCV_LIB STREQUAL "")
+  message(FATAL_ERROR "OpenCV library dirs not found for platform ${CMAKE_GENERATOR_PLATFORM}")
+endif()
+
+link_directories(${OPENCV_LIB})
+link_directories(${ORT_HOME}/lib)
+
+file(GLOB_RECURSE SRC_FILES "./*.cpp" "./*.c" "./*.cc")
+add_library(bianbuai SHARED ${SRC_FILES})
+set_target_properties(bianbuai PROPERTIES VERSION ${PROJECT_VERSION} SOVERSION ${PROJECT_VERSION_MAJOR})
+
+if (EXISTS "${ORT_HOME}/lib/libspacemit_ep.so")
+  add_definitions(-DHAS_SPACEMIT_EP)
+  set(SPACEMITEP_LIB "spacemit_ep")
+endif()
+
+target_include_directories(bianbuai PUBLIC  ${CMAKE_SOURCE_DIR}/include)
+target_include_directories(bianbuai PRIVATE ${CMAKE_SOURCE_DIR})
+target_include_directories(bianbuai SYSTEM PUBLIC  ${OPENCV_INC})
+target_include_directories(bianbuai SYSTEM PRIVATE ${ORT_HOME}/include ${ORT_HOME}/include/onnxruntime)
+
+set(HIDE_SYMBOLS_LINKER_FLAGS "-Wl,--exclude-libs,ALL")
+# Note: 'target_link_options' with 'PRIVATE' keyword would be cleaner
+# but it's not available until CMake 3.13. Switch to 'target_link_options'
+# once minimum CMake version is bumped up to 3.13 or above.
+target_link_libraries(bianbuai PRIVATE ${HIDE_SYMBOLS_LINKER_FLAGS})
+
+set(TARGET_SHARED_LINKER_FLAGS "-Wl,--as-needed")
+# Note: As modern cmake suggests to give the preference to the property settings before the global ones,
+# so instead of 'CMAKE_SHARED_LINKER_FLAGS' that has a global scope, the 'target_link_options' or
+# 'target_link_libraries' should be used here:
+#set(CMAKE_SHARED_LINKER_FLAGS "-Wl,--as-needed")
+target_link_libraries(bianbuai PRIVATE ${TARGET_SHARED_LINKER_FLAGS})
+target_link_libraries(bianbuai PRIVATE ${SPACEMITEP_LIB} onnxruntime ${OPENCV_LIBS})
+
+install(TARGETS bianbuai
+        RUNTIME DESTINATION bin
+        LIBRARY DESTINATION lib
+        ARCHIVE DESTINATION lib)
+
+install(DIRECTORY ${CMAKE_SOURCE_DIR}/include/task ${CMAKE_SOURCE_DIR}/include/utils
+        DESTINATION include/bianbuai
+        FILES_MATCHING PATTERN "*.h")
+
+#install(DIRECTORY ${ORT_HOME}/include ${ORT_HOME}/lib
+#        DESTINATION lib/3rdparty/onnxruntime
+#        FILES_MATCHING PATTERN "*.h" PATTERN "*onnxruntime*.so*" PATTERN "*spacemit*.so*")
+#if (EXISTS "${ORT_HOME}/bin")
+#  install(DIRECTORY ${ORT_HOME}/bin DESTINATION lib/3rdparty/onnxruntime)
+#endif()

src/core/engine.cc

@@ -0,0 +1,26 @@
+#include "src/core/engine.h"
+
+#ifdef _WIN32
+#include <codecvt>
+inline std::wstring to_wstring(const std::string& input) {
+  std::wstring_convert<std::codecvt_utf8<wchar_t>> converter;
+  return converter.from_bytes(input);
+}
+#endif /* _WIN32 */
+
+int Engine::Init(std::string instanceName, std::string modelFilepath) {
+  return ortwrapper_.Init(instanceName,
+#ifdef _WIN32
+                          to_wstring(modelFilepath)
+#else
+                          modelFilepath
+#endif /* _WIN32 */
+  );
+}
+
+int Engine::Init(json config) { return ortwrapper_.Init(config); }
+
+std::vector<Ort::Value> Engine::Interpreter(
+    std::vector<std::vector<float>>& input_values_handler) {
+  return ortwrapper_.Invoke(input_values_handler);
+}

src/core/engine.h

@@ -0,0 +1,33 @@
+#ifndef SUPPORT_SRC_CORE_ENGINE_H_
+#define SUPPORT_SRC_CORE_ENGINE_H_
+
+#include <string>
+#include <vector>
+
+#include "opencv2/opencv.hpp"
+#include "src/core/ort_wrapper.h"
+#include "src/utils/json.hpp"
+using json = nlohmann::json;
+class Engine {
+ public:
+  Engine() { OrtWrapper ortwrapper_; }
+  ~Engine() {}
+  int Init(std::string instanceName, std::string modelFilepath);
+  int Init(json config);
+  size_t GetInputCount() { return ortwrapper_.GetInputCount(); }
+  size_t GetOutputCount() { return ortwrapper_.GetOutputCount(); }
+  std::vector<std::vector<int64_t>> GetInputDims() {
+    return ortwrapper_.GetInputDims();
+  }
+  std::vector<std::vector<int64_t>> GetOutputDims() {
+    return ortwrapper_.GetOutputDims();
+  }
+  std::vector<Ort::Value> Interpreter(
+      std::vector<std::vector<float>> &input_values_handler);
+
+ protected:
+ private:
+  OrtWrapper ortwrapper_;
+};
+
+#endif  // SUPPORT_SRC_CORE_ENGINE_H_

src/core/ort_wrapper.cc

@@ -0,0 +1,216 @@
+#include "src/core/ort_wrapper.h"
+
+#include <utility>  // for move
+
+#include "utils/time.h"
+#ifdef HAS_SPACEMIT_EP
+#include "spacemit_ort_env.h"
+#endif
+
+int OrtWrapper::Init(std::string instanceName,
+                     std::basic_string<ORTCHAR_T> modelFilepath) {
+  std::unique_ptr<Ort::Env> env(new Ort::Env(
+      OrtLoggingLevel::ORT_LOGGING_LEVEL_WARNING, instanceName.c_str()));
+  // Creation: The Ort::Session is created here
+  env_ = std::move(env);
+  sessionOptions_.SetIntraOpNumThreads(2);
+  sessionOptions_.AddConfigEntry("session.intra_op.allow_spinning", "0");
+  sessionOptions_.SetInterOpNumThreads(2);
+  sessionOptions_.AddConfigEntry("session.inter_op.allow_spinning", "0");
+#ifdef HAS_SPACEMIT_EP
+  SessionOptionsSpaceMITEnvInit(sessionOptions_);
+  // auto providers = Ort::GetAvailableProviders();
+  std::cout << "Enable spacemit ep now" << std::endl;
+#else
+  std::cout << "Disable spacemit ep now" << std::endl;
+#endif
+  // Sets graph optimization level
+  // Available levels are
+  // ORT_DISABLE_ALL -> To disable all optimizations
+  // ORT_ENABLE_BASIC -> To enable basic optimizations (Such as redundant node
+  // removals) ORT_ENABLE_EXTENDED -> To enable extended optimizations
+  // (Includes level 1 + more complex optimizations like node fusions)
+  // ORT_ENABLE_ALL -> To Enable All possible optimizations
+  // sessionOptions_.SetGraphOptimizationLevel(
+  // GraphOptimizationLevel::ORT_DISABLE_ALL);
+  std::unique_ptr<Ort::Session> session(
+      new Ort::Session(*env_, modelFilepath.c_str(), sessionOptions_));
+  session_ = std::move(session);
+  return 0;
+}
+
+int OrtWrapper::Init(json config) {
+  std::string instanceName;
+  if (config.contains("instance_name")) {
+    instanceName = config["instance_name"];
+  }
+  std::basic_string<ORTCHAR_T> modelFilepath = config["model_path"];
+  std::unique_ptr<Ort::Env> env(new Ort::Env(
+      OrtLoggingLevel::ORT_LOGGING_LEVEL_WARNING, instanceName.c_str()));
+  // Creation: The Ort::Session is created here
+  env_ = std::move(env);
+  if (!config.contains("disable_spcacemit_ep") ||
+      config["disable_spcacemit_ep"] == false) {
+#ifdef HAS_SPACEMIT_EP
+    SessionOptionsSpaceMITEnvInit(sessionOptions_);
+    // auto providers = Ort::GetAvailableProviders();
+    std::cout << "Enable spacemit ep now" << std::endl;
+#else
+    std::cout << "[Warning] Unsupport spacemit ep now" << std::endl;
+#endif
+  } else {
+    std::cout << "Disable spacemit ep now" << std::endl;
+  }
+  if (config.contains("intra_threads_num")) {
+    int intraThreadsnum = config["intra_threads_num"];
+    sessionOptions_.SetIntraOpNumThreads(intraThreadsnum);
+    sessionOptions_.AddConfigEntry("session.intra_op.allow_spinning", "0");
+  } else {
+    sessionOptions_.SetIntraOpNumThreads(4);
+    sessionOptions_.AddConfigEntry("session.intra_op.allow_spinning", "0");
+  }
+  sessionOptions_.SetInterOpNumThreads(1);
+  sessionOptions_.AddConfigEntry("session.inter_op.allow_spinning", "0");
+  if (config.contains("profiling_projects")) {
+    std::basic_string<ORTCHAR_T> profiling_projects =
+        config["profiling_projects"];
+    if (profiling_projects.size()) {
+      sessionOptions_.EnableProfiling(profiling_projects.c_str());
+    }
+  }
+  if (config.contains("opt_model_path")) {
+    std::basic_string<ORTCHAR_T> opt_model_path = config["opt_model_path"];
+    if (opt_model_path.size()) {
+      sessionOptions_.SetOptimizedModelFilePath(opt_model_path.c_str());
+    }
+  }
+  if (config.contains("log_level")) {
+    int log_level = config["log_level"];
+    if (log_level >= 0 && log_level <= 4) {
+      sessionOptions_.SetLogSeverityLevel(log_level);
+    }
+  }
+  // Sets graph optimization level
+  // Available levels are
+  // ORT_DISABLE_ALL -> To disable all optimizations
+  // ORT_ENABLE_BASIC -> To enable basic optimizations (Such as redundant node
+  // removals) ORT_ENABLE_EXTENDED -> To enable extended optimizations
+  // (Includes level 1 + more complex optimizations like node fusions)
+  // ORT_ENABLE_ALL -> To Enable All possible optimizations
+  if (config.contains("graph_optimization_level")) {
+    if (config["graph_optimization_level"] == "ort_disable_all") {
+      sessionOptions_.SetGraphOptimizationLevel(
+          GraphOptimizationLevel::ORT_DISABLE_ALL);
+    } else if (config["graph_optimization_level"] == "ort_enable_basic") {
+      sessionOptions_.SetGraphOptimizationLevel(
+          GraphOptimizationLevel::ORT_ENABLE_BASIC);
+    } else if (config["graph_optimization_level"] == "ort_enable_extended") {
+      sessionOptions_.SetGraphOptimizationLevel(
+          GraphOptimizationLevel::ORT_ENABLE_EXTENDED);
+    } else {
+      sessionOptions_.SetGraphOptimizationLevel(
+          GraphOptimizationLevel::ORT_ENABLE_ALL);
+    }
+  }
+  std::unique_ptr<Ort::Session> session(
+      new Ort::Session(*env_, modelFilepath.c_str(), sessionOptions_));
+  session_ = std::move(session);
+  return 0;
+}
+
+std::vector<std::vector<int64_t>> OrtWrapper::GetInputDims() {
+  size_t num_inputs = session_->GetInputCount();
+  std::vector<std::vector<int64_t>> input_node_dims;
+  input_node_dims.resize(num_inputs);
+  for (size_t i = 0; i < num_inputs; ++i) {
+    Ort::TypeInfo input_type_info = session_->GetInputTypeInfo(i);
+    auto input_tensor_info = input_type_info.GetTensorTypeAndShapeInfo();
+    auto input_dims = input_tensor_info.GetShape();
+    input_dims[0] = abs(input_dims[0]);
+    input_dims[1] = abs(input_dims[1]);
+    input_node_dims[i] = input_dims;
+  }
+  return input_node_dims;
+}
+
+std::vector<std::vector<int64_t>> OrtWrapper::GetOutputDims() {
+  size_t num_outputs = session_->GetOutputCount();
+  std::vector<std::vector<int64_t>> output_node_dims;
+  output_node_dims.resize(num_outputs);
+  for (size_t i = 0; i < num_outputs; ++i) {
+    Ort::TypeInfo output_type_info = session_->GetOutputTypeInfo(i);
+    auto output_tensor_info = output_type_info.GetTensorTypeAndShapeInfo();
+    auto output_dims = output_tensor_info.GetShape();
+    output_node_dims[i] = output_dims;
+  }
+  return output_node_dims;
+}
+
+std::vector<Ort::Value> OrtWrapper::Invoke(
+    std::vector<std::vector<float>> &input_tensor_values) {
+#ifdef DEBUG
+  TimeWatcher t("|-- Infer tensor");
+#endif
+  // init onnxruntime allocator.
+  Ort::AllocatorWithDefaultOptions allocator;
+
+  // input names initial and build
+  std::vector<const char *> input_node_names;
+  std::vector<std::string> input_names;
+  size_t num_inputs = session_->GetInputCount();
+  input_node_names.resize(num_inputs);
+  for (size_t i = 0; i < num_inputs; i++) {
+    input_names.push_back(std::string(""));
+  }
+
+  for (size_t i = 0; i < num_inputs; ++i) {
+    auto input_name = session_->GetInputNameAllocated(i, allocator);
+    input_names[i].append(input_name.get());
+    input_node_names[i] = input_names[i].c_str();
+  }
+
+  // input node dims and input dims
+  auto input_node_dims = GetInputDims();
+
+  // input tensor size
+  std::vector<size_t> input_tensor_size;
+  input_tensor_size.resize(input_node_dims.size());
+  for (size_t i = 0; i < num_inputs; ++i) {
+    input_tensor_size[i] = 1;
+    for (size_t j = 0; j < input_node_dims[i].size(); ++j) {
+      input_tensor_size[i] *= input_node_dims[i][j];
+    }
+  }
+
+  // output names initial and build
+  std::vector<const char *> output_node_names;
+  std::vector<std::string> output_names;
+  size_t num_outputs = session_->GetOutputCount();
+  output_node_names.resize(num_outputs);
+  for (size_t i = 0; i < num_outputs; i++) {
+    output_names.push_back(std::string(""));
+  }
+
+  for (size_t i = 0; i < num_outputs; ++i) {
+    auto output_name = session_->GetOutputNameAllocated(i, allocator);
+    output_names[i].append(output_name.get());
+    output_node_names[i] = output_names[i].c_str();
+  }
+
+  // init and build input tensors
+  std::vector<Ort::Value> input_tensors;
+  Ort::MemoryInfo memoryInfo = Ort::MemoryInfo::CreateCpu(
+      OrtAllocatorType::OrtArenaAllocator, OrtMemType::OrtMemTypeDefault);
+
+  for (int i = 0; i < static_cast<int>(num_inputs); i++) {
+    input_tensors.push_back(Ort::Value::CreateTensor<float>(
+        memoryInfo, input_tensor_values[i].data(), input_tensor_size[i],
+        input_node_dims[i].data(), input_node_dims[i].size()));
+  }
+
+  // run model
+  auto outputTensors = session_->Run(
+      Ort::RunOptions{nullptr}, input_node_names.data(), input_tensors.data(),
+      num_inputs, output_node_names.data(), num_outputs);
+  return outputTensors;
+}

src/core/ort_wrapper.h

@@ -0,0 +1,37 @@
+#ifndef SUPPORT_SRC_CORE_ORT_WRAPPER_H_
+#define SUPPORT_SRC_CORE_ORT_WRAPPER_H_
+
+#include <cmath>
+#include <iostream>
+#include <memory>
+#include <numeric>
+#include <string>
+#include <vector>
+
+#include "onnxruntime_cxx_api.h"
+#include "opencv2/opencv.hpp"
+#include "src/utils/json.hpp"
+using json = nlohmann::json;
+
+class OrtWrapper {
+ public:
+  OrtWrapper() {}
+  ~OrtWrapper() {}
+  int Init(std::string instanceName,
+           std::basic_string<ORTCHAR_T> modelFilepath);
+  int Init(json config);
+  size_t GetInputCount() { return session_->GetInputCount(); }
+  size_t GetOutputCount() { return session_->GetOutputCount(); }
+  std::vector<std::vector<int64_t>> GetInputDims();
+  std::vector<std::vector<int64_t>> GetOutputDims();
+
+  std::vector<Ort::Value> Invoke(
+      std::vector<std::vector<float>>& input_tensor_values);
+
+ protected:
+ private:
+  std::unique_ptr<Ort::Env> env_;
+  Ort::SessionOptions sessionOptions_;
+  std::unique_ptr<Ort::Session> session_;
+};
+#endif  // SUPPORT_SRC_CORE_ORT_WRAPPER_H_

src/processor/classification_postprocessor.cc

@@ -0,0 +1,41 @@
+#include "src/processor/classification_postprocessor.h"
+
+#include <limits>  // for numeric_limits<>
+
+#include "utils/time.h"
+
+float ClassificationPostprocessor::division(float num, float den) {
+  if (den == 0) {
+    throw std::runtime_error(
+        "[ ERROR ] Math error: Attempted to divide by Zero\n");
+  }
+  return (num / den);
+}
+
+ImageClassificationResult ClassificationPostprocessor::Postprocess(
+    std::vector<Ort::Value> output_tensors, std::vector<std::string> &labels) {
+#ifdef DEBUG
+  TimeWatcher t("|-- Postprocess");
+#endif
+  int predId = 0;
+  float activation = 0;
+  float maxActivation = std::numeric_limits<float>::lowest();
+  float expSum = 0;
+  /* The inference result could be found in the buffer for the output tensors,
+  which are usually the buffer from std::vector instances. */
+  Ort::Value &pred = output_tensors.at(0);
+  const float *output_pred_ptr = pred.GetTensorData<float>();
+  for (int i = 0; i < static_cast<int>(labels.size()); i++) {
+    activation = output_pred_ptr[i];
+    expSum += std::exp(activation);
+    if (activation > maxActivation) {
+      predId = i;
+      maxActivation = activation;
+    }
+  }
+  ImageClassificationResult result;
+  result.label = predId;
+  result.label_text = labels.at(predId);
+  result.score = std::exp(maxActivation) / expSum;
+  return result;
+}

src/processor/classification_postprocessor.h

@@ -0,0 +1,26 @@
+#ifndef SUPPORT_SRC_PROCESSOR_CLASSIFICATION_POSTPROCESSOR_H_
+#define SUPPORT_SRC_PROCESSOR_CLASSIFICATION_POSTPROCESSOR_H_
+
+#include <cmath>
+#include <iostream>
+#include <stdexcept>  // To use runtime_error
+#include <string>
+#include <vector>
+
+#include "onnxruntime_cxx_api.h"
+#include "src/processor/processor.h"
+#include "task/vision/image_classification_types.h"
+
+class ClassificationPostprocessor : public Postprocessor {
+ public:
+  ClassificationPostprocessor() {}
+  ~ClassificationPostprocessor() {}
+  // Function to validate the input image file extension.
+  ImageClassificationResult Postprocess(std::vector<Ort::Value> output_tensors,
+                                        std::vector<std::string> &labels);
+
+  // Handling divide by zero
+  float division(float num, float den);
+};
+
+#endif  // SUPPORT_SRC_PROCESSOR_CLASSIFICATION_POSTPROCESSOR_H_

src/processor/classification_preprocessor.cc

@@ -0,0 +1,41 @@
+#include "src/processor/classification_preprocessor.h"
+
+#include "utils/time.h"
+
+void ClassificationPreprocessor::Preprocess(
+    const cv::Mat& imageBGR, std::vector<std::vector<int64_t>> inputDims,
+    std::vector<std::vector<float>>& input_tensor_values) {
+  cv::Mat resizedImageBGR, resizedImageRGB, resizedImage, preprocessedImage;
+  {
+#ifdef DEBUG
+    TimeWatcher t("| |-- Resize image");
+#endif
+    cv::resize(imageBGR, resizedImageBGR,
+               cv::Size(static_cast<int>(inputDims[0][3]),
+                        static_cast<int>(inputDims[0][2])),
+               cv::InterpolationFlags::INTER_CUBIC);
+  }
+
+  // step 3: Convert the image to HWC RGB UINT8 format.
+  cv::cvtColor(resizedImageBGR, resizedImageRGB,
+               cv::ColorConversionCodes::COLOR_BGR2RGB);
+  // step 4: Convert the image to HWC RGB float format by dividing each pixel by
+  // 255.
+  resizedImageRGB.convertTo(resizedImage, CV_32F, 1.0 / 255);
+
+  // step 5: Split the RGB channels from the image.
+  cv::Mat channels[3];
+  cv::split(resizedImage, channels);
+  const float mean_vals[3] = {0.485f, 0.456f, 0.406f};
+  const float scale_vals[3] = {0.229f, 0.224f, 0.225f};
+
+  int channel = 3;
+  std::vector<float> input_tensor_value;
+  for (int i = 0; i < channel; i++) {
+    channels[i] = (channels[i] - mean_vals[i]) / (scale_vals[i]);
+    std::vector<float> data = std::vector<float>(channels[i].reshape(1, 1));
+    input_tensor_value.insert(input_tensor_value.end(), data.begin(),
+                              data.end());
+  }
+  input_tensor_values.push_back(input_tensor_value);
+}

src/processor/classification_preprocessor.h

@@ -0,0 +1,22 @@
+#ifndef SUPPORT_SRC_PROCESSOR_CLASSIFICATION_PREPROCESSOR_H_
+#define SUPPORT_SRC_PROCESSOR_CLASSIFICATION_PREPROCESSOR_H_
+
+#include <fstream>
+#include <iostream>
+#include <stdexcept>
+#include <string>
+#include <vector>
+
+#include "opencv2/opencv.hpp"
+#include "src/processor/processor.h"
+
+class ClassificationPreprocessor : public Preprocessor {
+ public:
+  ClassificationPreprocessor() {}
+  ~ClassificationPreprocessor() {}
+  void Preprocess(const cv::Mat& imageBGR,
+                  std::vector<std::vector<int64_t>> inputDims,
+                  std::vector<std::vector<float>>& input_tensor_values);
+};
+
+#endif  // SUPPORT_SRC_PROCESSOR_CLASSIFICATION_PREPROCESSOR_H_

src/processor/detection_postprocessor.cc

@@ -0,0 +1,341 @@
+#include "src/processor/detection_postprocessor.h"
+
+#include "utils/time.h"
+#include "utils/utils.h"
+
+void DetectionPostprocessor::Postprocess(
+    std::vector<Ort::Value> output_tensors, std::vector<Boxi> &result_boxes,
+    std::vector<std::vector<int64_t>> &input_dims, int img_height,
+    int img_width, std::vector<std::string> &labels, float score_threshold,
+    float iou_threshold, unsigned int topk, unsigned int nms_type) {
+#ifdef DEBUG
+  TimeWatcher t("|-- Postprocess");
+#endif
+  if (score_threshold == -1.f) {
+    score_threshold = 0.25f;
+  }
+  if (iou_threshold == -1.f) {
+    iou_threshold = 0.45f;
+  }
+  int STRIDES[3] = {8, 16, 32};
+  float XYSCALE[3] = {1.2f, 1.1f, 1.05f};
+  int anchors[3][3][2] = {{{12, 16}, {19, 36}, {40, 28}},
+                          {{36, 75}, {76, 55}, {72, 146}},
+                          {{142, 110}, {192, 243}, {459, 401}}};
+  std::vector<Boxf> bbox_collection;
+  bbox_collection.clear();
+  unsigned int count = 0;
+  const float input_height = static_cast<float>(input_dims[0][2]);  // e.g 640
+  const float input_width = static_cast<float>(input_dims[0][1]);   // e.g 640
+  const float resize_ratio =
+      std::min(input_height / img_height, input_width / img_width);
+  for (int s = 0; s < static_cast<int>(output_tensors.size()); s++) {
+    Ort::Value &pred = output_tensors.at(s);  // batch*13*13*3*85
+    auto outputInfo = pred.GetTensorTypeAndShapeInfo();
+    auto pred_dims = outputInfo.GetShape();
+    const auto num_classes = pred_dims.at(4) - 5;  // 80
+    for (auto i = 0; i < pred_dims[1]; ++i) {
+      for (auto j = 0; j < pred_dims[2]; ++j) {
+        int grid_x = j;
+        int grid_y = i;
+        for (auto k = 0; k < pred_dims[3]; ++k) {
+          float obj_conf = pred.At<float>({0, i, j, k, 4});
+          if (obj_conf < score_threshold) continue;  // filter first.
+
+          float cls_conf = pred.At<float>({0, i, j, k, 5});
+          unsigned int label = 0;
+          for (auto h = 0; h < num_classes; ++h) {
+            float tmp_conf = pred.At<float>({0, i, j, k, h + 5});
+            if (tmp_conf > cls_conf) {
+              cls_conf = tmp_conf;
+              label = h;
+            }
+          }
+          float conf = obj_conf * cls_conf;      // cls_conf (0.,1.)
+          if (conf < score_threshold) continue;  // filter
+          float cx = (sigmoid(pred.At<float>({0, i, j, k, 0})) * XYSCALE[s] -
+                      0.5f * (XYSCALE[s] - 1) + grid_x) *
+                     STRIDES[s];
+          float cy = (sigmoid(pred.At<float>({0, i, j, k, 1})) * XYSCALE[s] -
+                      0.5f * (XYSCALE[s] - 1) + grid_y) *
+                     STRIDES[s];
+          float w = exp(pred.At<float>({0, i, j, k, 2})) * anchors[s][k][0];
+          float h = exp(pred.At<float>({0, i, j, k, 3})) * anchors[s][k][1];
+          Boxf box;
+          float dw = (input_width - resize_ratio * img_width) / 2;
+          float dh = (input_height - resize_ratio * img_height) / 2;
+          box.x1 = (cx - w / 2.f - dw) / resize_ratio;
+          box.x1 = std::max(box.x1, .0f);
+          box.y1 = (cy - h / 2.f - dh) / resize_ratio;
+          box.y1 = std::max(box.y1, .0f);
+          box.x2 = (cx + w / 2.f - dw) / resize_ratio;
+          box.x2 = std::min(box.x2, static_cast<float>(img_width - 1));
+          box.y2 = (cy + h / 2.f - dh) / resize_ratio;
+          box.y2 = std::min(box.y2, static_cast<float>(img_height - 1));
+          box.score = conf;
+          box.label = label;
+          box.label_text = labels[label].c_str();
+          box.flag = true;
+          bbox_collection.push_back(box);
+          count += 1;  // limit boxes for nms.
+          if (count > max_nms) break;
+        }
+      }
+    }
+  }
+  std::vector<Boxf> detected_boxes;
+
+  // 4. hard|blend|offset nms with topk.
+  nms(bbox_collection, detected_boxes, iou_threshold, topk, nms_type);
+
+  int detected_boxes_num = detected_boxes.size();
+  for (auto i = 0; i < static_cast<int>(detected_boxes_num); i++) {
+    Boxi result_box;
+    result_box.x1 = static_cast<int>(detected_boxes[i].x1);
+    result_box.y1 = static_cast<int>(detected_boxes[i].y1);
+    result_box.x2 = static_cast<int>(detected_boxes[i].x2);
+    result_box.y2 = static_cast<int>(detected_boxes[i].y2);
+    result_box.label = detected_boxes[i].label;
+    result_box.score = detected_boxes[i].score;
+    result_box.label_text = detected_boxes[i].label_text;
+    result_box.flag = detected_boxes[i].flag;
+    result_boxes.push_back(result_box);
+  }
+}
+
+void DetectionPostprocessor::PostprocessYolov6(
+    std::vector<Ort::Value> output_tensors, std::vector<Boxi> &result_boxes,
+    std::vector<std::vector<int64_t>> &input_dims, int img_height,
+    int img_width, std::vector<std::string> &labels, float &score_threshold) {
+#ifdef DEBUG
+  TimeWatcher t("|-- Postprocess");
+#endif
+  if (score_threshold == -1.f) {
+    score_threshold = 0.59f;
+  }
+  std::vector<Boxf> bbox_collection;
+  bbox_collection.clear();
+  const float input_height = static_cast<float>(input_dims[0][2]);
+  const float input_width = static_cast<float>(input_dims[0][3]);
+  const float resize_ratio =
+      std::min(input_height / img_height, input_width / img_width);
+  // Ort::Value &num_dets = output_tensors.at(0);
+  Ort::Value &boxes = output_tensors.at(1);
+  Ort::Value &scores = output_tensors.at(2);
+  Ort::Value &output_labels = output_tensors.at(3);
+  auto outputInfo = boxes.GetTensorTypeAndShapeInfo();
+  auto pred_dims = outputInfo.GetShape();
+  float dw = (input_width - resize_ratio * img_width) / 2;
+  float dh = (input_height - resize_ratio * img_height) / 2;
+
+  auto num = pred_dims[1];
+  for (auto i = 0; i < num; i++) {
+    Boxi result_box;
+    result_box.score = scores.At<float>({0, i});
+    if (result_box.score < score_threshold) {
+      continue;
+    }
+    result_box.x1 =
+        static_cast<int>((boxes.At<float>({0, i, 0}) - dw) / resize_ratio);
+    result_box.y1 =
+        static_cast<int>((boxes.At<float>({0, i, 1}) - dh) / resize_ratio);
+    result_box.x2 =
+        static_cast<int>((boxes.At<float>({0, i, 2}) - dw) / resize_ratio);
+    result_box.y2 =
+        static_cast<int>((boxes.At<float>({0, i, 3}) - dh) / resize_ratio);
+    if (output_labels.At<int>({0, i}) < 0) {
+      continue;
+    }
+    result_box.label = output_labels.At<int>({0, i});
+    result_box.label_text = labels[result_box.label].c_str();
+    result_box.flag = true;
+    result_boxes.push_back(result_box);
+  }
+}
+
+void DetectionPostprocessor::PostprocessNanoDetPlus(
+    std::vector<Ort::Value> output_tensors, std::vector<Boxi> &result_boxes,
+    std::vector<std::vector<int64_t>> &input_dims, int img_height,
+    int img_width, std::vector<std::string> &labels, float &score_threshold,
+    float &nms_threshold) {
+#ifdef DEBUG
+  TimeWatcher t("|-- Postprocess");
+#endif
+  if (score_threshold == -1.f) {
+    score_threshold = 0.4f;
+  }
+  if (nms_threshold == -1.f) {
+    nms_threshold = 0.5f;
+  }
+  std::vector<Boxf> bbox_collection;
+  bbox_collection.clear();
+  const int cls_num = 80;
+  const float input_height = static_cast<float>(input_dims[0][2]);
+  const float input_width = static_cast<float>(input_dims[0][3]);
+  const float resize_ratio =
+      std::min(input_height / img_height, input_width / img_width);
+  Ort::Value &pred = output_tensors.at(0);
+  const float *output_pred_ptr = pred.GetTensorData<float>();
+  auto outputInfo = pred.GetTensorTypeAndShapeInfo();
+  auto pred_dims = outputInfo.GetShape();
+  std::vector<int> hw = {40, 20, 10, 5};
+  std::vector<int> strides = {8, 16, 32, 64};
+  int num = -1;
+  for (auto i = 0; i < 4; i++) {
+    for (auto y = 0; y < hw[i]; y++) {
+      for (auto x = 0; x < hw[i]; x++) {
+        num++;
+        int ct_x = x;
+        int ct_y = y;
+        const float *scores = output_pred_ptr + num * 112;  // row ptr
+        float cls_conf = pred.At<float>({0, num, 0});
+        unsigned int label = 0;
+        for (auto h = 0; h < cls_num; h++) {
+          float tmp_conf = scores[h];
+          if (tmp_conf > cls_conf) {
+            cls_conf = tmp_conf;
+            label = h;
+          }
+        }
+        if (cls_conf < score_threshold) continue;
+        std::vector<float> dis_pred(4, .0);
+        for (int s = 0; s < 4; s++) {
+          float alptha = .0;
+          int cur_num = cls_num + s * 8;
+          std::vector<float> dst(8);
+          for (int j = 0; j < 8; j++) {
+            alptha = std::max(alptha, scores[cur_num + j]);
+          }
+          float sum = .0f;
+          for (int j = 0; j < 8; j++) {
+            dst[j] = fast_exp(scores[cur_num + j] - alptha);
+            sum = sum + dst[j];
+          }
+          for (int j = 0; j < 8; j++) {
+            dis_pred[s] = dis_pred[s] + j * (dst[j] / sum);
+          }
+        }
+
+        Boxf box;
+        float dw = (input_width - resize_ratio * img_width) / 2;
+        float dh = (input_height - resize_ratio * img_height) / 2;
+        box.label = label;
+        box.label_text = labels[label].c_str();
+        box.score = cls_conf;
+        box.flag = true;
+        box.x1 = ((ct_x - dis_pred[0]) * strides[i] - dw) / resize_ratio;
+        box.x1 = std::max(box.x1, .0f);
+        box.y1 = ((ct_y - dis_pred[1]) * strides[i] - dh) / resize_ratio;
+        box.y1 = std::max(box.y1, .0f);
+        box.x2 = ((ct_x + dis_pred[2]) * strides[i] - dw) / resize_ratio;
+        box.x2 = std::min(box.x2, static_cast<float>(img_width - 1));
+        box.y2 = ((ct_y + dis_pred[3]) * strides[i] - dh) / resize_ratio;
+        box.y2 = std::min(box.y2, static_cast<float>(img_height - 1));
+        bbox_collection.push_back(box);
+      }
+    }
+  }
+  std::vector<Boxf> detected_boxes;
+
+  // 4. hard|blend|offset nms with topk.
+  nms(bbox_collection, detected_boxes, nms_threshold, 100, OFFSET);
+
+  size_t detected_boxes_num = detected_boxes.size();
+  for (size_t i = 0; i < detected_boxes_num; i++) {
+    Boxi result_box;
+    result_box.x1 = static_cast<int>(detected_boxes[i].x1);
+    result_box.y1 = static_cast<int>(detected_boxes[i].y1);
+    result_box.x2 = static_cast<int>(detected_boxes[i].x2);
+    result_box.y2 = static_cast<int>(detected_boxes[i].y2);
+    result_box.label = detected_boxes[i].label;
+    result_box.score = detected_boxes[i].score;
+    result_box.label_text = detected_boxes[i].label_text;
+    result_box.flag = detected_boxes[i].flag;
+    result_boxes.push_back(result_box);
+  }
+}
+void DetectionPostprocessor::PostprocessRtmDet(
+    std::vector<Ort::Value> output_tensors, std::vector<Boxi> &result_boxes,
+    std::vector<std::vector<int64_t>> &input_dims, int img_height,
+    int img_width, std::vector<std::string> &labels, float &score_threshold,
+    float &nms_threshold) {
+#ifdef DEBUG
+  TimeWatcher t("|-- Postprocess");
+#endif
+  if (score_threshold == -1.f) {
+    score_threshold = 0.2f;
+  }
+  if (nms_threshold == -1.f) {
+    nms_threshold = 0.6f;
+  }
+  std::vector<Boxf> bbox_collection;
+  bbox_collection.clear();
+  const float input_height = static_cast<float>(input_dims[0][2]);  // e.g 320
+  const float input_width = static_cast<float>(input_dims[0][3]);   // e.g 320
+  const float resize_ratio =
+      std::min(input_height / img_height, input_width / img_width);
+  Ort::Value &boxes = output_tensors.at(0);
+  Ort::Value &output_labels = output_tensors.at(1);
+  std::vector<int64_t> det_result_dims =
+      boxes.GetTensorTypeAndShapeInfo().GetShape();
+  std::vector<int64_t> label_result_dims =
+      output_labels.GetTensorTypeAndShapeInfo().GetShape();
+
+  assert(det_result_dims.size() == 3 && label_result_dims.size() == 2);
+  int64_t num_dets =
+      det_result_dims[1] == label_result_dims[1] ? det_result_dims[1] : 0;
+  int64_t reshap_dims = det_result_dims[2];
+  const float *det_result = boxes.GetTensorData<float>();
+  const int *label_result = output_labels.GetTensorData<int>();
+  float dw = (input_width - resize_ratio * img_width) / 2;
+  float dh = (input_height - resize_ratio * img_height) / 2;
+
+  for (int64_t i = 0; i < num_dets; ++i) {
+    int classes = label_result[i];
+    if (classes != 0) continue;
+    Boxf box;
+    box.flag = true;
+    box.score = det_result[i * reshap_dims + 4];
+    if (box.score < score_threshold) continue;  // filter
+    box.x1 = (det_result[i * reshap_dims] - dw) / resize_ratio;
+    box.x1 = std::max(box.x1, .0f);
+    box.y1 = (det_result[i * reshap_dims + 1] - dh) / resize_ratio;
+    box.y1 = std::max(box.y1, .0f);
+    box.x2 = (det_result[i * reshap_dims + 2] - dw) / resize_ratio;
+    box.x2 = std::min(box.x2, static_cast<float>(img_width - 1));
+    box.y2 = (det_result[i * reshap_dims + 3] - dh) / resize_ratio;
+    box.y2 = std::min(box.y2, static_cast<float>(img_height - 1));
+    box.label = label_result[i];
+    box.label_text = labels[box.label].c_str();
+    bbox_collection.push_back(box);
+  }
+
+  std::vector<Boxf> detected_boxes;
+  // 4. hard|blend|offset nms with topk.
+  nms(bbox_collection, detected_boxes, nms_threshold, 100, OFFSET);
+
+  size_t detected_boxes_num = detected_boxes.size();
+  for (size_t i = 0; i < detected_boxes_num; i++) {
+    Boxi result_box;
+    result_box.x1 = static_cast<int>(detected_boxes[i].x1);
+    result_box.y1 = static_cast<int>(detected_boxes[i].y1);
+    result_box.x2 = static_cast<int>(detected_boxes[i].x2);
+    result_box.y2 = static_cast<int>(detected_boxes[i].y2);
+    result_box.label = detected_boxes[i].label;
+    result_box.score = detected_boxes[i].score;
+    result_box.label_text = detected_boxes[i].label_text;
+    result_box.flag = detected_boxes[i].flag;
+    result_boxes.push_back(result_box);
+  }
+}
+
+void DetectionPostprocessor::nms(std::vector<Boxf> &input,
+                                 std::vector<Boxf> &output, float iou_threshold,
+                                 unsigned int topk, unsigned int nms_type) {
+  if (nms_type == BLEND)
+    blending_nms(input, output, iou_threshold, topk);
+  else if (nms_type == OFFSET)
+    offset_nms(input, output, iou_threshold, topk);
+  else
+    hard_nms(input, output, iou_threshold, topk);
+}

src/processor/detection_postprocessor.h

@@ -0,0 +1,52 @@
+#ifndef SUPPORT_SRC_PROCESSOR_DETECTION_POSTPROCESSOR_H_
+#define SUPPORT_SRC_PROCESSOR_DETECTION_POSTPROCESSOR_H_
+
+#include <algorithm>
+#include <iostream>
+#include <string>
+#include <vector>
+
+#include "onnxruntime_cxx_api.h"
+#include "src/processor/processor.h"
+#include "src/utils/nms_utils.h"
+#include "src/utils/utils.h"
+#include "task/vision/object_detection_types.h"
+
+class DetectionPostprocessor : public Postprocessor {
+ public:
+  DetectionPostprocessor() {}
+
+  void Postprocess(std::vector<Ort::Value> output_tensors,
+                   std::vector<Boxi> &result_boxes,
+                   std::vector<std::vector<int64_t>> &input_dims,
+                   int img_height, int img_width,
+                   std::vector<std::string> &labels, float score_threshold,
+                   float iou_threshold, unsigned int topk = 100,
+                   unsigned int nms_type = OFFSET);
+
+  void PostprocessYolov6(std::vector<Ort::Value> output_tensors,
+                         std::vector<Boxi> &result_boxes,
+                         std::vector<std::vector<int64_t>> &input_dims,
+                         int img_height, int img_width,
+                         std::vector<std::string> &labels,
+                         float &score_threshold);
+
+  void PostprocessNanoDetPlus(std::vector<Ort::Value> output_tensors,
+                              std::vector<Boxi> &result_boxes,
+                              std::vector<std::vector<int64_t>> &input_dims,
+                              int img_height, int img_width,
+                              std::vector<std::string> &labels,
+                              float &score_threshold, float &nms_threshold);
+
+  void PostprocessRtmDet(std::vector<Ort::Value> output_tensors,
+                         std::vector<Boxi> &result_boxes,
+                         std::vector<std::vector<int64_t>> &input_dims,
+                         int img_height, int img_width,
+                         std::vector<std::string> &labels,
+                         float &score_threshold, float &nms_threshold);
+
+ private:
+  void nms(std::vector<Boxf> &input, std::vector<Boxf> &output,
+           float iou_threshold, unsigned int topk, unsigned int nms_type);
+};
+#endif  // SUPPORT_SRC_PROCESSOR_DETECTION_POSTPROCESSOR_H_

src/processor/detection_preprocessor.cc

@@ -0,0 +1,125 @@
+#include "src/processor/detection_preprocessor.h"
+
+#include "utils/time.h"
+#include "utils/utils.h"
+
+void DetectionPreprocessor::PreprocessNanoDetPlus(
+    const cv::Mat& mat, std::vector<std::vector<int64_t>>& input_node_dims,
+    std::vector<std::vector<float>>& input_tensor_values) {
+  const int input_height = (int)input_node_dims[0][2];
+  const int input_width = (int)input_node_dims[0][3];
+  cv::Mat resizedImageBGR, resizedImage, preprocessedImage;
+  {
+#ifdef DEBUG
+    TimeWatcher t("| |-- Resize unscale");
+#endif
+    if (input_height != mat.cols || input_width != mat.rows) {
+      resize_unscale(mat, resizedImage, input_height, input_width);
+    } else {
+      resizedImage = mat;
+    }
+  }
+  {
+#ifdef DEBUG
+    TimeWatcher t("| |-- Convert to fp32");
+#endif
+    resizedImage.convertTo(resizedImage, CV_32F, 1.0);
+  }
+
+  {
+#ifdef DEBUG
+    TimeWatcher t("| |-- Normalize");
+#endif
+    cv::Mat channels[3];
+    cv::split(resizedImage, channels);
+    const float mean_vals[3] = {103.53f, 116.28f, 123.675f};
+    const float scale_vals[3] = {57.375f, 57.12f, 58.395f};
+    int channel = 3;
+    std::vector<float> input_tensor_value;
+    for (int i = 0; i < channel; i++) {
+      channels[i] = (channels[i] - mean_vals[i]) / (scale_vals[i]);
+      std::vector<float> data = std::vector<float>(channels[i].reshape(1, 1));
+      input_tensor_value.insert(input_tensor_value.end(), data.begin(),
+                                data.end());
+    }
+    input_tensor_values.push_back(input_tensor_value);
+  }
+}
+
+void DetectionPreprocessor::Preprocess(
+    const cv::Mat& mat, std::vector<std::vector<int64_t>>& input_node_dims,
+    std::vector<std::vector<float>>& input_tensor_values,
+    unsigned int data_format) {
+  if (mat.empty()) return;
+  if (data_format == 1) {
+    const int input_height = static_cast<int>(input_node_dims[0][1]);
+    const int input_width = static_cast<int>(input_node_dims[0][2]);
+
+    // resize & unscale
+    cv::Mat resizedImageBGR, resizedImageRGB, resizedImage, preprocessedImage;
+    {
+#ifdef DEBUG
+      TimeWatcher t("| |-- Resize unscale");
+#endif
+      if (input_height != mat.cols || input_width != mat.rows) {
+        resize_unscale(mat, resizedImageBGR, input_height, input_width);
+      } else {
+        resizedImageBGR = mat;
+      }
+    }
+    cv::cvtColor(resizedImageBGR, resizedImageRGB, cv::COLOR_BGR2RGB);
+    resizedImageRGB.convertTo(resizedImage, CV_32F, 1.0 / 255);
+    const unsigned int target_tensor_size = 3 * input_height * input_width;
+    std::vector<float> input_tensor_value;
+    input_tensor_value.resize(target_tensor_size);
+    std::memcpy(input_tensor_value.data(), resizedImage.data,
+                target_tensor_size * sizeof(float));
+    input_tensor_values.push_back(input_tensor_value);
+  } else {
+    const int input_height = static_cast<int>(input_node_dims[0][2]);
+    const int input_width = static_cast<int>(input_node_dims[0][3]);
+
+    cv::Mat resizedImageBGR, resizedImageRGB, resizedImage, preprocessedImage;
+    {
+#ifdef DEBUG
+      TimeWatcher t("| |-- Resize unscale");
+#endif
+      if (input_height != mat.cols || input_width != mat.rows) {
+        resize_unscale(mat, resizedImageBGR, input_height, input_width);
+      } else {
+        resizedImageBGR = mat;
+      }
+    }
+    {
+#ifdef DEBUG
+      TimeWatcher t("| |-- Convert to RGB");
+#endif
+      // step 3: Convert the image to HWC RGB UINT8 format.
+      cv::cvtColor(resizedImageBGR, resizedImageRGB, cv::COLOR_BGR2RGB);
+    }
+    {
+#ifdef DEBUG
+      TimeWatcher t("| |-- Convert to fp32");
+#endif
+      resizedImageRGB.convertTo(resizedImage, CV_32F, 1.0 / 255);
+    }
+    {
+#ifdef DEBUG
+      TimeWatcher t("| |-- Normalize");
+#endif
+      cv::Mat channels[3];
+      cv::split(resizedImage, channels);
+      // const float mean_vals[3] = {116.28f, 116.28f, 116.28f};
+      // const float scale_vals[3] = {0.017429f, 0.017429f, 0.017429f};
+      int channel = 3;
+      std::vector<float> input_tensor_value;
+      for (int i = 0; i < channel; i++) {
+        // channels[i] = (channels[i] - mean_vals[i]) * scale_vals[i];
+        std::vector<float> data = std::vector<float>(channels[i].reshape(1, 1));
+        input_tensor_value.insert(input_tensor_value.end(), data.begin(),
+                                  data.end());
+      }
+      input_tensor_values.push_back(input_tensor_value);
+    }
+  }
+}

src/processor/detection_preprocessor.h

@@ -0,0 +1,27 @@
+#ifndef SUPPORT_SRC_PROCESSOR_DETECTION_PREPROCESSOR_H_
+#define SUPPORT_SRC_PROCESSOR_DETECTION_PREPROCESSOR_H_
+
+#include <chrono>
+#include <string>
+#include <vector>
+
+#include "opencv2/opencv.hpp"
+#include "src/processor/processor.h"
+#include "src/utils/cv2_utils.h"
+#include "src/utils/nms_utils.h"
+
+class DetectionPreprocessor : public Preprocessor {
+ public:
+  DetectionPreprocessor() {}
+  ~DetectionPreprocessor() {}
+  // Function to validate the input image file extension.
+  void Preprocess(const cv::Mat& mat,
+                  std::vector<std::vector<int64_t>>& input_node_dims,
+                  std::vector<std::vector<float>>& input_tensor_values,
+                  unsigned int data_format);
+  void PreprocessNanoDetPlus(
+      const cv::Mat& mat, std::vector<std::vector<int64_t>>& input_node_dims,
+      std::vector<std::vector<float>>& input_tensor_values);
+};
+
+#endif  // SUPPORT_SRC_PROCESSOR_DETECTION_PREPROCESSOR_H_

src/processor/estimation_postprocessor.cc

@@ -0,0 +1,73 @@
+#include "src/processor/estimation_postprocessor.h"
+
+#include "utils/time.h"
+
+void EstimationPostprocessor::Postprocess(
+    std::vector<Ort::Value> output_tensors,
+    std::pair<cv::Mat, cv::Mat> crop_result_pair,
+    std::vector<PosePoint> &result_points) {
+#ifdef DEBUG
+  TimeWatcher t("|-- Postprocess");
+#endif
+  std::vector<int64_t> simcc_x_dims =
+      output_tensors[0].GetTensorTypeAndShapeInfo().GetShape();
+  std::vector<int64_t> simcc_y_dims =
+      output_tensors[1].GetTensorTypeAndShapeInfo().GetShape();
+
+  assert(simcc_x_dims.size() == 3 && simcc_y_dims.size() == 3);
+
+  // int batch_size = simcc_x_dims[0] == simcc_y_dims[0] ? simcc_x_dims[0] : 0;
+  int joint_num = simcc_x_dims[1] == simcc_y_dims[1] ? simcc_x_dims[1] : 0;
+  int extend_width = simcc_x_dims[2];
+  int extend_height = simcc_y_dims[2];
+
+  float *simcc_x_result = output_tensors[0].GetTensorMutableData<float>();
+  float *simcc_y_result = output_tensors[1].GetTensorMutableData<float>();
+
+  for (int i = 0; i < joint_num; ++i) {
+    // find the maximum and maximum indexes in the value of each Extend_width
+    // length
+    auto x_biggest_iter =
+        std::max_element(simcc_x_result + i * extend_width,
+                         simcc_x_result + i * extend_width + extend_width);
+    int max_x_pos =
+        std::distance(simcc_x_result + i * extend_width, x_biggest_iter);
+    int pose_x = max_x_pos / 2;
+    float score_x = *x_biggest_iter;
+
+    // find the maximum and maximum indexes in the value of each exten_height
+    // length
+    auto y_biggest_iter =
+        std::max_element(simcc_y_result + i * extend_height,
+                         simcc_y_result + i * extend_height + extend_height);
+    int max_y_pos =
+        std::distance(simcc_y_result + i * extend_height, y_biggest_iter);
+    int pose_y = max_y_pos / 2;
+    float score_y = *y_biggest_iter;
+
+    // float score = (score_x + score_y) / 2;
+    float score = std::max(score_x, score_y);
+
+    PosePoint temp_point;
+    temp_point.x = static_cast<int>(pose_x);
+    temp_point.y = static_cast<int>(pose_y);
+    temp_point.score = score;
+    result_points.emplace_back(temp_point);
+    // char index[8];
+    // sprintf(index, "%d", i);
+    // cv::putText(input_mat_copy_rgb, index, cv::Point(pose_result[i].x,
+    // pose_result[i].y), 1, 1, cv::Scalar{ 0, 0, 255 }, 1);
+  }
+  // cv::imwrite("pose.jpg", input_mat_copy_rgb);
+  // anti affine transformation to obtain the coordinates on the original
+  // picture
+  cv::Mat affine_transform_reverse = crop_result_pair.second;
+  for (int i = 0; i < static_cast<int>(result_points.size()); ++i) {
+    cv::Mat origin_point_Mat = cv::Mat::ones(3, 1, CV_64FC1);
+    origin_point_Mat.at<double>(0, 0) = result_points[i].x;
+    origin_point_Mat.at<double>(1, 0) = result_points[i].y;
+    cv::Mat temp_result_mat = affine_transform_reverse * origin_point_Mat;
+    result_points[i].x = temp_result_mat.at<double>(0, 0);
+    result_points[i].y = temp_result_mat.at<double>(1, 0);
+  }
+}

src/processor/estimation_postprocessor.h

@@ -0,0 +1,23 @@
+#ifndef SUPPORT_SRC_PROCESSOR_ESTIMATION_POSTPROCESSOR_H_
+#define SUPPORT_SRC_PROCESSOR_ESTIMATION_POSTPROCESSOR_H_
+
+#include <algorithm>
+#include <iostream>
+#include <string>
+#include <vector>
+
+#include "onnxruntime_cxx_api.h"
+#include "src/processor/processor.h"
+#include "src/utils/nms_utils.h"
+#include "src/utils/utils.h"
+#include "task/vision/pose_estimation_types.h"
+
+class EstimationPostprocessor : public Postprocessor {
+ public:
+  EstimationPostprocessor() {}
+  void Postprocess(std::vector<Ort::Value> output_tensors,
+                   std::pair<cv::Mat, cv::Mat> crop_result_pair,
+                   std::vector<PosePoint> &result_points);
+};
+
+#endif  // SUPPORT_SRC_PROCESSOR_ESTIMATION_POSTPROCESSOR_H_

src/processor/estimation_preprocessor.cc

@@ -0,0 +1,106 @@
+#include "src/processor/estimation_preprocessor.h"
+
+#include "src/utils/cv2_utils.h"
+#include "src/utils/nms_utils.h"
+#include "utils/time.h"
+#include "utils/utils.h"
+
+void EstimationPreprocessor::Preprocess(
+    const cv::Mat& mat, const Boxi& box,
+    std::vector<std::vector<float>>& input_tensor_values,
+    std::pair<cv::Mat, cv::Mat>& crop_result_pair, unsigned int data_format) {
+  if (mat.empty()) return;
+  if (data_format == 0) {
+    // const int input_height = input_node_dims[0][2];
+    // const int input_width = input_node_dims[0][3];
+    crop_result_pair = CropImageByDetectBox(mat, box);
+
+    cv::Mat crop_matBGR = crop_result_pair.first;
+    cv::Mat affine_transform_reverse = crop_result_pair.second;
+    cv::Mat crop_mat, crop_matRGB;
+
+    {
+#ifdef DEBUG
+      TimeWatcher t("| |-- Convert to RGB");
+#endif
+      // step 3: Convert the image to HWC RGB UINT8 format.
+      cv::cvtColor(crop_matBGR, crop_matRGB, cv::COLOR_BGR2RGB);
+    }
+    {
+#ifdef DEBUG
+      TimeWatcher t("| |-- Convert to fp32");
+#endif
+      crop_matRGB.convertTo(crop_mat, CV_32F, 1.0 / 255);
+    }
+    {
+#ifdef DEBUG
+      TimeWatcher t("| |-- Normalize");
+#endif
+      cv::Mat channels[3];
+      cv::split(crop_mat, channels);
+      int channel = 3;
+      std::vector<float> input_tensor_value;
+      for (int i = 0; i < channel; i++) {
+        // channels[i] = (channels[i] - mean_vals[i]) * scale_vals[i];
+        std::vector<float> data = std::vector<float>(channels[i].reshape(1, 1));
+        input_tensor_value.insert(input_tensor_value.end(), data.begin(),
+                                  data.end());
+      }
+      input_tensor_values.push_back(input_tensor_value);
+    }
+  }
+}
+
+std::pair<cv::Mat, cv::Mat> EstimationPreprocessor::CropImageByDetectBox(
+    const cv::Mat& input_image, const Boxi& box) {
+  // auto time_start = std::chrono::steady_clock::now();
+  std::pair<cv::Mat, cv::Mat> result_pair;
+
+  if (!input_image.data) {
+    return result_pair;
+  }
+
+  if (!box.flag) {
+    return result_pair;
+  }
+
+  // deep copy
+  cv::Mat input_mat_copy;
+  input_image.copyTo(input_mat_copy);
+
+  // calculate the width, height and center points of the human detection box
+  int box_width = box.x2 - box.x1;
+  int box_height = box.y2 - box.y1;
+  int box_center_x = box.x1 + box_width / 2;
+  int box_center_y = box.y1 + box_height / 2;
+
+  float aspect_ratio = 192.0 / 256.0;
+
+  // adjust the width and height ratio of the size of the picture in the RTMPOSE
+  // input
+  if (box_width > (aspect_ratio * box_height)) {
+    box_height = box_width / aspect_ratio;
+  } else if (box_width < (aspect_ratio * box_height)) {
+    box_width = box_height * aspect_ratio;
+  }
+
+  float scale_image_width = box_width * 1.3;
+  float scale_image_height = box_height * 1.3;
+
+  // get the affine matrix
+  cv::Mat affine_transform =
+      GetAffineTransform(box_center_x, box_center_y, scale_image_width,
+                         scale_image_height, 192, 256);
+
+  cv::Mat affine_transform_reverse =
+      GetAffineTransform(box_center_x, box_center_y, scale_image_width,
+                         scale_image_height, 192, 256, true);
+
+  // affine transform
+  cv::Mat affine_image;
+  cv::warpAffine(input_mat_copy, affine_image, affine_transform,
+                 cv::Size(192, 256), cv::INTER_LINEAR);
+
+  result_pair = std::make_pair(affine_image, affine_transform_reverse);
+  return result_pair;
+}

src/processor/estimation_preprocessor.h

@@ -0,0 +1,28 @@
+#ifndef SUPPORT_SRC_PROCESSOR_ESTIMATION_PREPROCESSOR_H_
+#define SUPPORT_SRC_PROCESSOR_ESTIMATION_PREPROCESSOR_H_
+
+#include <chrono>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "opencv2/opencv.hpp"
+#include "src/processor/processor.h"
+#include "task/vision/object_detection_types.h"
+
+class EstimationPreprocessor : public Preprocessor {
+ public:
+  EstimationPreprocessor() {}
+  ~EstimationPreprocessor() {}
+  // Function to validate the input image file extension.
+  void Preprocess(const cv::Mat& mat, const Boxi& box,
+                  std::vector<std::vector<float>>& input_tensor_values,
+                  std::pair<cv::Mat, cv::Mat>& crop_result_pair,
+                  unsigned int data_format);
+
+ protected:
+  std::pair<cv::Mat, cv::Mat> CropImageByDetectBox(const cv::Mat& input_image,
+                                                   const Boxi& box);
+};
+
+#endif  // SUPPORT_SRC_PROCESSOR_ESTIMATION_PREPROCESSOR_H_

src/processor/processor.h

@@ -0,0 +1,24 @@
+#ifndef SUPPORT_SRC_PROCESSOR_PROCESSOR_H_
+#define SUPPORT_SRC_PROCESSOR_PROCESSOR_H_
+
+class Processor {
+ public:
+  Processor() = default;
+  virtual ~Processor() = default;
+
+  // Processor is neither copyable nor movable.
+  Processor(const Processor&) = delete;
+  Processor& operator=(const Processor&) = delete;
+};
+
+class Preprocessor : public Processor {
+ protected:
+  using Processor::Processor;
+};
+
+class Postprocessor : public Processor {
+ protected:
+  using Processor::Processor;
+};
+
+#endif  // SUPPORT_SRC_PROCESSOR_PROCESSOR_H_

src/task/core/base_task_api.h

@@ -0,0 +1,46 @@
+#ifndef SUPPORT_SRC_TASK_CORE_BASE_TASK_API_H_
+#define SUPPORT_SRC_TASK_CORE_BASE_TASK_API_H_
+
+#include <memory>
+#include <vector>
+
+#include "onnxruntime_cxx_api.h"
+#include "opencv2/opencv.hpp"
+#include "src/core/engine.h"
+
+class BaseUntypedTaskApi {
+ public:
+  BaseUntypedTaskApi() { engine_ = std::unique_ptr<Engine>(new Engine()); }
+  virtual ~BaseUntypedTaskApi() = default;
+
+ protected:
+  Engine* GetEngine() { return engine_.get(); }
+  std::unique_ptr<Engine> engine_;
+};
+
+template <class OutputType, class... InputTypes>
+class BaseTaskApi : public BaseUntypedTaskApi {
+ public:
+  BaseTaskApi() : BaseUntypedTaskApi() {}
+  ~BaseTaskApi() {}
+  // BaseTaskApi is neither copyable nor movable.
+  BaseTaskApi(const BaseTaskApi&) = delete;
+  BaseTaskApi& operator=(const BaseTaskApi&) = delete;
+  std::vector<std::vector<int64_t>> GetInputShape() {
+    return GetEngine()->GetInputDims();
+  }
+  void Cancel() {}
+
+ protected:
+  // Subclasses need to populate input_tensors from api_inputs.
+  virtual void Preprocess(InputTypes... api_inputs) = 0;
+  // Subclasses need to construct OutputType object from output_tensors.
+  // Original inputs are also provided as they may be needed.
+  virtual OutputType Postprocess() = 0;
+  std::vector<Ort::Value> Infer(
+      std::vector<std::vector<float>>& input_tensors) {
+    return GetEngine()->Interpreter(input_tensors);
+  }
+};
+
+#endif  // SUPPORT_SRC_TASK_CORE_BASE_TASK_API_H_

src/task/vision/base_vision_task_api.h

@@ -0,0 +1,24 @@
+#ifndef _BASE_VISION_TASK_API_H_
+#define _BASE_VISION_TASK_API_H_
+
+#include <string>
+#include <vector>
+
+#include "opencv2/opencv.hpp"
+#include "src/task/core/base_task_api.h"
+#include "task/vision/object_detection_types.h"
+
+template <class OutputType>
+class BaseVisionTaskApi : public BaseTaskApi<OutputType, const cv::Mat&> {
+ public:
+  BaseVisionTaskApi() : BaseTaskApi<OutputType, const cv::Mat&>() {}
+  ~BaseVisionTaskApi() {}
+  // BaseVisionTaskApi is neither copyable nor movable.
+  BaseVisionTaskApi(const BaseVisionTaskApi&) = delete;
+  BaseVisionTaskApi& operator=(const BaseVisionTaskApi&) = delete;
+
+ protected:
+  virtual void Preprocess(const cv::Mat& img_raw) = 0;
+};
+
+#endif

src/task/vision/imageclassification/image_classification.cc

@@ -0,0 +1,41 @@
+#include "src/task/vision/imageclassification/image_classification.h"
+
+#include "utils/time.h"
+
+int imageClassification::Init(const std::string modelFilepath,
+                              const std::string labelFilepath) {
+  instanceName_ = "image-classification-inference";
+  modelFilepath_ = modelFilepath;
+  labelFilepath_ = labelFilepath;
+  labels_ = readLabels(labelFilepath_);
+  initFlag_ = GetEngine()->Init(instanceName_, modelFilepath_);
+  return initFlag_;
+}
+
+void imageClassification::Preprocess(const cv::Mat &img_raw) {
+  auto input_dims = GetInputShape();
+  preprocessor_.Preprocess(img_raw, input_dims, input_tensors_);
+}
+
+ImageClassificationResult imageClassification::Postprocess() {
+  return postprocessor_.Postprocess(Infer(input_tensors_), labels_);
+}
+
+ImageClassificationResult imageClassification::Classify(
+    const cv::Mat &img_raw) {
+  if (initFlag_ != 0) {
+    std::cout << "[ ERROR ] Init fail return empty result" << std::endl;
+    ImageClassificationResult empty_result{"", -1, .0f};
+    return empty_result;
+  } else {
+    img_raw_ = img_raw;
+    {
+#ifdef DEBUG
+      std::cout << "|-- Preprocess" << std::endl;
+      TimeWatcher t("|--");
+#endif
+      Preprocess(img_raw_);
+    }
+    return Postprocess();
+  }
+}

src/task/vision/imageclassification/image_classification.h

@@ -0,0 +1,49 @@
+#ifndef SUPPORT_SRC_TASK_VISION_IMAGECLASSIFICATION_IMAGE_CLASSIFICATION_H_
+#define SUPPORT_SRC_TASK_VISION_IMAGECLASSIFICATION_IMAGE_CLASSIFICATION_H_
+
+#include <cmath>
+#include <exception>
+#include <fstream>
+#include <iostream>
+#include <limits>
+#include <memory>
+#include <numeric>
+#include <string>
+#include <vector>
+
+#include "opencv2/opencv.hpp"
+#include "src/core/engine.h"
+#include "src/processor/classification_postprocessor.h"
+#include "src/processor/classification_preprocessor.h"
+#include "src/task/vision/base_vision_task_api.h"
+#include "src/utils/utils.h"
+#include "task/vision/image_classification_types.h"
+
+class imageClassification
+    : public BaseVisionTaskApi<ImageClassificationResult> {
+ public:
+  imageClassification() : BaseVisionTaskApi<ImageClassificationResult>() {
+    initFlag_ = -1;
+  }
+  ~imageClassification() {}
+  int Init(const std::string modelFilepath, const std::string labelFilepath);
+  ImageClassificationResult Classify(const cv::Mat& img_raw);
+
+ protected:
+  void Preprocess(const cv::Mat& img_raw) override;
+  ImageClassificationResult Postprocess() override;
+
+ private:
+  ClassificationPreprocessor preprocessor_;
+  ClassificationPostprocessor postprocessor_;
+  std::string instanceName_;
+  std::string modelFilepath_;
+  cv::Mat img_raw_;
+  std::string labelFilepath_;
+  std::vector<std::string> labels_;
+  std::vector<Ort::Value> output_tensors_;
+  std::vector<std::vector<float>> input_tensors_;
+  int initFlag_ = false;
+};
+
+#endif  // SUPPORT_SRC_TASK_VISION_IMAGECLASSIFICATION_IMAGE_CLASSIFICATION_H_