bianbu-repo-mirror/k1x-ai-support
1
0
Fork 0
mirror of https://gitee.com/bianbu-linux/ai-support synced 2025-04-22 05:08:39 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions
k1x-ai-support/.gitlab-ci.yml
James Deng 1974ea511a Update for v1.0.8
2024-07-16 19:26:24 +08:00

355 lines
No EOL
16 KiB
YAML
Raw Permalink Blame History

# This file is a template, and might need editing before it works on your project.
# This is a sample GitLab CI/CD configuration file that should run without any modifications.
# It demonstrates a basic 3 stage CI/CD pipeline. Instead of real tests or scripts,
# it uses echo commands to simulate the pipeline execution.
#
# A pipeline is composed of independent jobs that run scripts, grouped into stages.
# Stages run in sequential order, but jobs within stages run in parallel.
#
# For more information, see: https://docs.gitlab.com/ee/ci/yaml/index.html#stages
#
# You can copy and paste this template into a new `.gitlab-ci.yml` file.
# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword.
#
# To contribute improvements to CI/CD templates, please follow the Development guide at:
# https://docs.gitlab.com/ee/development/cicd/templates.html
# This specific template is located at:
# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml
stages: # List of stages for jobs, and their order of execution
- build
- test
- package
- upload
- deploy
image: harbor.bianbu.xyz/spacemit-ai/bianbu-ai-support:v1.1.1
.only-defaults: &only-defaults
only:
- tags
- main
- cicd
- merge_requests
- schedules
.rule-upload: &rule-upload
rules:
- if: '$CI_COMMIT_TAG =~ /^bianbu-.*\/.*$/'
when: on_success
variables:
SDK: "/opt"
DATA: "rootfs/usr/share/ai-support"
CROSS_TOOL: "$SDK/spacemit-gcc/bin/riscv64-unknown-linux-gnu-"
SYSROOT: "$SDK/spacemit-gcc/sysroot"
QEMU_CMD: "$SDK/qemu/bin/qemu-riscv64 -L $SYSROOT" # TODO: update with lastest spacemit-ai-sdk
ORT_HOME_X86: "/home/workspace/onnxruntime-linux-x64-1.15.1"
CI_NEXUS_URL: "https://nexus.bianbu.xyz/service/rest/repository/browse/bianbu-ai/onnxruntime/"
CI_INSTALL_OPTION: "--with-demo --with-ort"
KUBERNETES_MEMORY_LIMIT: "6Gi"
KUBERNETES_MEMORY_REQUEST: "6Gi"
build-job-x86_64: # This job runs in the build stage, which runs first.
stage: build
variables:
BUILD_DIR: "build.x86_64"
ORT_HOME: "${ORT_HOME_X86}"
INSTALL_PREFIX: "bianbu-ai-support.x86_64"
before_script:
- echo "Before scripts ..."
script:
- echo "Compiling the code ..."
- |
mkdir ${BUILD_DIR}
pushd ${BUILD_DIR}
cmake .. -DORT_HOME=${ORT_HOME} -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} -DDEMO=ON -DXDG=OFF
make install -j`nproc` VERBOSE=1
popd
# copy demo and data
- if [[ "${CI_INSTALL_OPTION}" =~ "--with-demo" ]]; then cp -rdf demo ${BUILD_DIR}/${INSTALL_PREFIX}; fi
# copy onnxruntime
- if [[ "${CI_INSTALL_OPTION}" =~ "--with-ort" ]]; then mkdir -p ${BUILD_DIR}/${INSTALL_PREFIX}/lib/3rdparty/onnxruntime && cp -rdf ${ORT_HOME}/* ${BUILD_DIR}/${INSTALL_PREFIX}/lib/3rdparty/onnxruntime; fi
- |
pushd demo
mkdir ${BUILD_DIR}
pushd ${BUILD_DIR}
cmake .. -DBIANBUAI_HOME=$(pwd)/../../${BUILD_DIR}/${INSTALL_PREFIX} -DORT_HOME=${ORT_HOME}
make -j`nproc` VERBOSE=1
popd
- echo "Compile complete."
artifacts:
paths:
- ${BUILD_DIR}
- demo/${BUILD_DIR}
<<: *only-defaults
build-job-riscv64: # This job runs in the build stage, which runs second.
stage: build
variables:
BUILD_DIR: "build.riscv64"
ORT_HOME: "${CI_PROJECT_DIR}/spacemit-ort"
# TODO: fix opencv.v4.6.0 output precision error
OPENCV_SHARED_DIR: "$SDK/bianbu-ai-support/3rdparty/opencv.v4.7.0.shared" # v4.6.0.shared.qt5
OPENCV_STATIC_DIR: "$SDK/bianbu-ai-support/3rdparty/opencv.v4.7.0.static"
INSTALL_PREFIX: "bianbu-ai-support.riscv64"
before_script:
- echo "Before scripts ..."
#- set -x
#- echo "CI_BUILDS_DIR ${CI_BUILDS_DIR}"
#- echo "CI_PROJECT_DIR ${CI_PROJECT_DIR}"
- echo "Downloading latest spacemit-ort ..."
- wget $(curl -X GET ${CI_NEXUS_URL} | grep -oP 'https:[^>]*.riscv64.[\d\.]*tar.gz' | sort -V | tail -n 1) -O spacemit-ort.latest.tar.gz --no-check-certificate
- |
mkdir spacemit-ort
tar xzf spacemit-ort.latest.tar.gz -C spacemit-ort --strip-components 1
tree -L 3 .
script:
- echo "Compiling the code ..."
# Note: To use opencv static libraries, one'd better set OpenCV_DIR variable instead of OPENCV_INC and OPENCV_LIB variables.
- |
mkdir ${BUILD_DIR}
pushd ${BUILD_DIR}
cmake .. -DORT_HOME=${ORT_HOME} -DOpenCV_DIR=${OPENCV_STATIC_DIR}/lib/cmake/opencv4 -DCMAKE_C_COMPILER=${CROSS_TOOL}gcc -DCMAKE_CXX_COMPILER=${CROSS_TOOL}g++ -DCMAKE_SYSROOT=${SYSROOT} -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} -DDEMO=ON -DXDG=OFF
make install -j`nproc` VERBOSE=1
popd
# copy demo and data
- if [[ "${CI_INSTALL_OPTION}" =~ "--with-demo" ]]; then cp -rdf demo ${BUILD_DIR}/${INSTALL_PREFIX}; fi
# copy onnxruntime
- if [[ "${CI_INSTALL_OPTION}" =~ "--with-ort" ]]; then mkdir -p ${BUILD_DIR}/${INSTALL_PREFIX}/lib/3rdparty/onnxruntime && cp -rdf ${ORT_HOME}/* ${BUILD_DIR}/${INSTALL_PREFIX}/lib/3rdparty/onnxruntime; fi
# - mkdir ${BUILD_DIR} && pushd ${BUILD_DIR} && cmake .. -DORT_HOME=${ORT_HOME} -DOPENCV_INC=${OPENCV_SHARED_DIR}/include/opencv4 -DOPENCV_LIB=${OPENCV_SHARED_DIR}/lib -DCMAKE_C_COMPILER=${CROSS_TOOL}gcc -DCMAKE_CXX_COMPILER=${CROSS_TOOL}g++ -DCMAKE_SYSROOT=${SYSROOT} -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} -DDEMO=ON -DUSE_OPENCV_SHARED_LIBS=ON && make install -j`nproc` VERBOSE=1 && popd
- |
pushd demo
mkdir ${BUILD_DIR}
pushd ${BUILD_DIR}
cmake .. -DBIANBUAI_HOME=$(pwd)/../../${BUILD_DIR}/${INSTALL_PREFIX} -DORT_HOME=${ORT_HOME} -DOPENCV_INC=${OPENCV_SHARED_DIR}/include/opencv4 -DOPENCV_LIB=${OPENCV_SHARED_DIR}/lib -DCMAKE_C_COMPILER=${CROSS_TOOL}gcc -DCMAKE_CXX_COMPILER=${CROSS_TOOL}g++ -DCMAKE_SYSROOT=${SYSROOT}
make -j`nproc` VERBOSE=1
popd
# - echo "Compile complete."
artifacts:
paths:
- ${BUILD_DIR}
- spacemit-ort
- demo/${BUILD_DIR}
<<: *only-defaults
build-job-riscv64-deb:
stage: build
image: harbor.bianbu.xyz/spacemit-ai/bianbu-ai-support:debian
dependencies: []
script:
- echo "Building debian package for tag $CI_COMMIT_TAG"
- |
set -x
mount -o bind $CI_PROJECT_DIR/.. /mnt/
schroot -r -c persistent-session --directory /mnt/ai/support
apt build-dep -y $(pwd)
dpkg-buildpackage -us -b -uc --no-pre-clean --no-post-clean
exit
mkdir bianbu-dev
mv ../bianbu-ai-support-* bianbu-dev
mv ../bianbu-ai-support_* bianbu-dev
artifacts:
paths:
- bianbu-dev
only:
- deb
x86_64-test-job: # This job runs in the test stage.
stage: test # It only starts when the job in the build stage completes successfully.
dependencies:
- build-job-x86_64
script:
- echo "Running unit tests ... (This will take several seconds.)"
- export SUPPORT_SHOW=-1
# smoke test for image classification
- export LD_LIBRARY_PATH=${ORT_HOME_X86}/lib:$LD_LIBRARY_PATH
- result=`demo/build.x86_64/classification_demo $DATA/models/squeezenet1.1-7.onnx $DATA/models/synset.txt $DATA/imgs/dog.jpg`
- $([[ ${result:0-30:8} == "Pembroke" ]])
# smoke test for object detection
- demo/build.x86_64/detection_demo $DATA/models/yolov6p5_n.q.onnx $DATA/models/coco.txt $DATA/imgs/person.jpg result0.jpg
- $([[ "b604ad08d6283995ec8dcdbddc2482c2" == "$(md5sum result0.jpg | awk '{print $1}')" ]])
- sed -i 's/\/usr/rootfs\/usr/g' $DATA/models/yolov6.json
- demo/build.x86_64/detection_demo $DATA/models/yolov6.json $DATA/imgs/person.jpg result0.jpg
- $([[ "b604ad08d6283995ec8dcdbddc2482c2" == "$(md5sum result0.jpg | awk '{print $1}')" ]])
- demo/build.x86_64/detection_video_demo $DATA/models/yolov6p5_n.q.onnx $DATA/models/coco.txt $DATA/videos/test.mp4 test.avi
- $([[ "760c5bb692fe302621265126f70bd7e9" == "$(md5sum test.avi | awk '{print $1}')" ]])
- demo/build.x86_64/detection_stream_demo $DATA/models/yolov6p5_n.q.onnx $DATA/models/coco.txt $DATA/videos/test.mp4
- demo/build.x86_64/detection_stream_demo $DATA/models/yolov6.json $DATA/videos/test.mp4
# smoke test for human pose estimation
- demo/build.x86_64/estimation_demo $DATA/models/yolov6p5_n.q.onnx $DATA/models/coco.txt $DATA/models/rtmpose-t.q.onnx $DATA/imgs/person.jpg result1.jpg
- $([[ "aa7a9492408bc41917178e242478a15c" == "$(md5sum result1.jpg | awk '{print $1}')" ]])
- sed -i 's/\/usr/rootfs\/usr/g' $DATA/models/rtmpose.json
- demo/build.x86_64/estimation_demo $DATA/models/yolov6.json $DATA/models/rtmpose.json $DATA/imgs/person.jpg result1.jpg
- $([[ "aa7a9492408bc41917178e242478a15c" == "$(md5sum result1.jpg | awk '{print $1}')" ]])
- demo/build.x86_64/tracker_stream_demo $DATA/models/yolov6p5_n.q.onnx $DATA/models/coco.txt $DATA/models/rtmpose-t.q.onnx $DATA/videos/test.mp4
- demo/build.x86_64/tracker_stream_demo $DATA/models/yolov6.json $DATA/models/rtmpose.json $DATA/videos/test.mp4
- echo "Running x86_64 tests done!"
<<: *only-defaults
riscv64-test-job: # This job runs in the test stage.
stage: test # It only starts when the job in the build stage completes successfully.
dependencies:
- build-job-riscv64
script:
- echo "Running unit tests ... (This will take several seconds.)"
# smoke test for image classification
- result=`${QEMU_CMD} demo/build.riscv64/classification_demo $DATA/models/squeezenet1.1-7.onnx $DATA/models/synset.txt $DATA/imgs/dog.jpg`
- $([[ ${result:0-30:8} == "Pembroke" ]])
# smoke test for object detection
- ${QEMU_CMD} demo/build.riscv64/detection_demo $DATA/models/yolov6p5_n.q.onnx $DATA/models/coco.txt $DATA/imgs/person.jpg result0.jpg
- $([[ "81d50314ee64d46c9d4c9941711820ba" == "$(md5sum result0.jpg | awk '{print $1}')" ]])
- sed -i 's/\/usr/rootfs\/usr/g' $DATA/models/yolov6.json
- ${QEMU_CMD} demo/build.riscv64/detection_demo $DATA/models/yolov6.json $DATA/imgs/person.jpg result0.jpg
- $([[ "81d50314ee64d46c9d4c9941711820ba" == "$(md5sum result0.jpg | awk '{print $1}')" ]])
# smoke test for human pose estimation
- ${QEMU_CMD} demo/build.riscv64/estimation_demo $DATA/models/yolov6p5_n.q.onnx $DATA/models/coco.txt $DATA/models/rtmpose-t.q.onnx $DATA/imgs/person.jpg result1.jpg
- $([[ "2becf268d0c8da27207ee1dec934e80b" == "$(md5sum result1.jpg | awk '{print $1}')" ]])
- sed -i 's/\/usr/rootfs\/usr/g' $DATA/models/rtmpose.json
- ${QEMU_CMD} demo/build.riscv64/estimation_demo $DATA/models/yolov6.json $DATA/models/rtmpose.json $DATA/imgs/person.jpg result0.jpg
- $([[ "2becf268d0c8da27207ee1dec934e80b" == "$(md5sum result0.jpg | awk '{print $1}')" ]])
- echo "Running riscv64 tests done!"
<<: *only-defaults
package-daily-dpkg:
stage: package
variables:
CI_PKG_OPTION: "--skip-py" # "--skip-ort --skip-py"
CI_PKG_VERSION: "latest"
dependencies:
- build-job-x86_64
- build-job-riscv64
script:
#- echo $(date +"%Y%m%d_%H%M%S")
- mkdir -p output
# make deb ball
- |
bash ci/pack.sh build.x86_64/bianbu-ai-support.x86_64 ${CI_PKG_OPTION}
mv bianbu-ai-support-*.deb output/bianbu-ai-support_${CI_PKG_VERSION}_amd64.deb
- |
bash ci/pack.sh build.riscv64/bianbu-ai-support.riscv64 ${CI_PKG_OPTION}
mv bianbu-ai-support-*.deb output/bianbu-ai-support_${CI_PKG_VERSION}_riscv64.deb
#- rm -rf build.x86_64 build.riscv64 demo
artifacts:
paths:
- output/bianbu-ai-support_*.deb
<<: *only-defaults
package-daily-prebuilt:
stage: package
dependencies:
- build-job-x86_64
- build-job-riscv64
script:
- mkdir -p output
# make tar ball
- |
pushd build.x86_64
tar czf ../output/bianbu-ai-support.x86_64.$(date +"%Y%m%d_%H%M%S").tar.gz bianbu-ai-support.x86_64
popd
- |
pushd build.riscv64
tar czf ../output/bianbu-ai-support.riscv64.$(date +"%Y%m%d_%H%M%S").tar.gz bianbu-ai-support.riscv64
popd
#- rm -rf build.x86_64 build.riscv64 demo
artifacts:
paths:
- output/bianbu-ai-support.*.tar.gz
<<: *only-defaults
upload-archive-nexus:
stage: upload
dependencies:
- build-job-x86_64
- build-job-riscv64
script:
- tag=v${CI_COMMIT_REF_NAME#*/}
- |
pushd build.x86_64
tar czf bianbu-ai-support.x86_64.${tag}.tar.gz bianbu-ai-support.x86_64
http_code=$(curl -k -u $NEXUS_USERNAME:$NEXUS_PASSWORD --upload-file bianbu-ai-support.x86_64.${tag}.tar.gz https://nexus.bianbu.xyz/repository/bianbu-ai/support-library/bianbu-ai-support.x86_64.${tag}.tar.gz -w %{http_code})
[[ ${http_code} == "201" ]]
popd
- |
pushd build.riscv64
tar czf bianbu-ai-support.riscv64.${tag}.tar.gz bianbu-ai-support.riscv64
http_code=$(curl -k -u $NEXUS_USERNAME:$NEXUS_PASSWORD --upload-file bianbu-ai-support.riscv64.${tag}.tar.gz https://nexus.bianbu.xyz/repository/bianbu-ai/support-library/bianbu-ai-support.riscv64.${tag}.tar.gz -w %{http_code})
[[ ${http_code} == "201" ]]
popd
<<: *rule-upload
upload-archive-bianbu:
stage: upload
image: harbor.bianbu.xyz/gitlab/ci-pack
dependencies: []
script:
# TODO: replace by shell script
- echo "Building for tag $CI_COMMIT_TAG"
- eval $(ssh-agent -s)
- echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
- mkdir -p ~/.ssh
- chmod 700 ~/.ssh
- ssh-keyscan gitlab.dc.com >> ~/.ssh/known_hosts
- chmod 644 ~/.ssh/known_hosts
- git config --global user.email "bianbu-ci@spacemit.com"
- git config --global user.name "bianbu-ci"
# prepare bianbu-dev toolkits
- cd ..
- git clone git@gitlab.dc.com:bianbu/bianbu-devscripts.git
- export PATH=$(pwd)/bianbu-devscripts:$PATH
# prepare source code
- cd $CI_PROJECT_DIR
- git checkout ${CI_COMMIT_TAG%%/*}
- bianbu-pkg -u local -w . -t $CI_COMMIT_TAG
- changes_file=$(find ../ -maxdepth 1 -type f -name "*.changes" | head -n 1)
# upload
- ssh-keyscan reprepro-headless-service.buildsystem.svc.cluster.local >> ~/.ssh/known_hosts
- bianbu-dev set-default-dist bianbu-23.10
- bianbu-dev upload $changes_file --suite mantic-porting
- bianbu-dev set-default-dist bianbu-24.04
- bianbu-dev upload $changes_file --suite noble-porting
# change proxy as ssh
- git remote set-url origin git@gitlab.dc.com:$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME.git
- git push origin --all
# status
- echo "Bianbu devscripts successfully packaged."
<<: *rule-upload
lint-test-job: # This job also runs in the test stage.
stage: test # It can run at the same time as unit-test-job (in parallel).
dependencies: []
script:
- echo "Linting code... This will take about 2 seconds."
- sleep 2
- echo "No lint issues found."
<<: *only-defaults
deploy-job: # This job runs in the deploy stage.
stage: deploy # It only runs when *both* jobs in the test stage complete successfully.
environment: production
dependencies: []
script:
- echo "Deploying application..."
- echo "Application successfully deployed."
only:
- tags
# SAST customization: https://docs.gitlab.com/ee/user/application_security/sast/#customizing-the-sast-settings
# Secret Detection customization: https://docs.gitlab.com/ee/user/application_security/secret_detection/#customizing-settings
# Dependency Scanning customization: https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#customizing-the-dependency-scanning-settings
# Container Scanning customization: https://docs.gitlab.com/ee/user/application_security/container_scanning/#customizing-the-container-scanning-settings
# Note that environment variables can be set in several places
# See https://docs.gitlab.com/ee/ci/variables/#cicd-variable-precedence
sast:
stage: test
dependencies: []
include:
- template: Security/SAST.gitlab-ci.yml
- template: Security/Secret-Detection.gitlab-ci.yml
Reference in a new issue View git blame Copy permalink