1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67

import triton_python_backend_utils as pb_utils

class TritonPythonModel:
    """
	Python 모델 `model.py`는 항상 TritonPythonModel클래스명을 가져야 한다.
    """

    @staticmethod
    def auto_complete_config(auto_complete_model_config):
		"""
		만약, 직접 config.pbtxt를 작성할 예정이고, --disable-auto-complete-config 옵션을 항상 킬 예정이라면, 이 부분을 구현할 필요가 없다.
		"""

    def initialize(self, args):
		"""
		`initialize`는 모델이 로딩되었을 때, 단 한 번만 실행된다.
		이 모델의 관련된 상태를 불러오는 데 사용된다.
		이 부분도 optional하다.

        Parameters
        ----------
        args : dict
		  key, value모두 string이다.
          * model_config: 모델 config를 담는 JSON문자열
          * model_instance_kind: 모델 인스턴스의 종류("cpu", "cuda" 등)
          * model_instance_device_id: 모델 인스턴스의 디바이스 id
          * model_repository: 모델 레포지토리 경로
          * model_version: 모델 버전
          * model_name: 모델 이름
        """
        print('Initialized...')

    def execute(self, requests):
		"""
		`execute`는 반드시 구현되어야 하는 함수이다.
		argument로 pb_utils.InferenceRequest의 리스트를 받는다.
		이 함수는 추론 요청이 올 때 실행된다.

        Parameters
        ----------
        requests : list
          pb_utils.InferenceRequest의 리스트

        Returns
        -------
        list
		  pb_utils.InferenceResponse 리스트.
		  응답의 길이는 `requests`와 같아야 한다.
        """

        responses = []

		# 모든 Python Backend는 리스트의 아이템을 정확히 한 번 다뤄야한다.
        for request in requests:
			# 여기에서 각 요청에 대한 추론이 이뤄지도록 하고,
			# 각각의 응답을 순서 그대로 담도록 해야 한다.

		# pb_utils.InferenceRenponse의 리스트를 반환
		# len(pb_utils.InferenceResponse) == len(pb_utils.InferenceRequest)
        return responses

    def finalize(self):
		"""
		`finalize`는 모델 언로드 시 반드시 한 번 실행된다.
		종료 이전 필요한 cleanup 작업들을 실행하면 된다.
        """
        print('Cleaning up...')

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

import triton_python_backend_utils as pb_utils

class TritonPythonModel:
	def execute(self, requests):
		responses = []
		for request in requests:
			if an_error_occured:
			responses.append(pb_utils.InferenceResponse(
			error = pb_utils.TritonError("An Error Occurred")))
		return responses

1

pb_utils.TritonError("The file is not found", pb_utils.TritonError.NOT_FOUND)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

import triton_python_backend_utils as pb_utils

class TritonPythonModel:
	...
	def execute(self, requests):
		responses = []
		for request in requests:
			if request.is_cancelled():
				responses.append(pb_utils.InferenceResponse(
				error=pb.utils.TritonError("Message", pb_utils.TritonError.CANCELLED)))
			else:
				...
		return responses

1
2
3
4
5

models
└── add_sub(모델명)
    ├── 1
    │   └── model.py
    └── config.pbtxt

1

request = pb_utils.InferenceRequest(parameters={"key": "value"}, ...)

1
2
3

response = pb_utils.InferenceResponse(
    output_tensors, parameters={"key": "value"}
)

1
2
3
4
5
6

git clone https://github.com/triton-inference-server/python_backend -b
<GIT_BRANCH_NAME>
cd python_backend
mkdir build && cd build
cmake -DTRITON_ENABLE_GPU=ON -DTRITON_BACKEND_REPO_TAG=<GIT_BRANCH_NAME> -DTRITON_COMMON_REPO_TAG=<GIT_BRANCH_NAME> -DTRITON_CORE_REPO_TAG=<GIT_BRANCH_NAME> -DCMAKE_INSTALL_PREFIX:PATH=`pwd`/install ..
make triton-python-backend-stub

1
2
3
4

ldd triton_python_backend_stub
...
libpython3.6m.so.1.0 => /home/ubuntu/envs/miniconda3/envs/python-3-6/lib/libpython3.6m.so.1.0(0x00007fbb69cf3000)
...

1
2
3
4
5
6

models
|-- model_a
    |-- 1
    |   |-- model.py
    |-- config.pbtxt
    `-- triton_python_backend_stub

1
2
3
4

conda-pack
Collecting packages...
Packing environment at '/home/iman/miniconda3/envs/python-3-6' to 'python-3-6.tar.gz'
[########################################] | 100% Completed |  4.5s

1

export PYTHONNOUSERSITE=True

1
2
3
4
5
6
7

name: "model_a"
backend: "python"
...
parameters: {
	key: "EXECUTION_ENV_PATH",
	value: {string_value: "home/ubuntu/miniconda3/envs/python-3-6/python3.6.tar.gz"}
}

1
2
3
4
5
6
7

name: "model_a"
backend: "python"
...
parameters: {
	key: "EXECUTION_ENV_PATH",
	value: {string_value: "$$TRITON_MODEL_DIRECTORY/python3.6.tar.gz"}
}

1
2
3
4
5
6
7

models
|-- model_a
|   |-- 1
|   |   `-- model.py
|   |-- config.pbtxt
|   |-- python3.6.tar.gz
|   `-- triton_python_backend_stub

1
2

mkdir -p $pwd/models/model_a/python3.6
tar -xvf $pwd/models/model_a/python3.6.tar.gz -C $pwd/models/model_a/python3.6

1
2
3
4

parameters: {
  key: "EXECUTION_ENV_PATH",
  value: {string_value: "$$TRITON_MODEL_DIRECTORY/python3.6"}
}

1
2
3
4
5
6
7

model_repository
└── add_one
    ├── 1
    │   └── model.py
    ├── config.pbtxt
    ├── add_one_env.tar.gz 
    └── triton_python_backend_stub

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

{
	"name": "Python Backend",

	"build": {
		"dockerfile": "Dockerfile"
	},
	"customizations": {
		"vscode": {
			"extensions": [
				"ms-python.vscode-pylance",
				"ms-python.python",
				"ms-vscode.cpptools-extension-pack",
				"ms-vscode.cmake-tools",
				"github.vscode-pull-request-github"
			]
		}
	},
	"postCreateCommand": "sudo chown -R triton-server:triton-server ~/.cache",

	// "--gpus=all"은 nvidia gpu가 없으면 제거
	"runArgs": [ "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined", "--gpus=all", "--shm-size=2g", "--ulimit", "stack=67108864" ],
	"mounts": [
		"source=${localEnv:HOME}/.ssh,target=/home/triton-server/.ssh,type=bind,consistency=cached",
		"source=${localEnv:HOME}/.cache/huggingface,target=/home/triton-server/.cache/huggingface,type=bind,consistency=cached"
	],
	"remoteUser": "triton-server"
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48

# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#  * Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
#  * Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#  * Neither the name of NVIDIA CORPORATION nor the names of its
#    contributors may be used to endorse or promote products derived
#    from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

FROM nvcr.io/nvidia/tritonserver:25.09-py3 # 원하는 버전에 맞추기

ARG USERNAME=triton-server

RUN apt-get update \
    && apt-get install -y sudo

RUN pip3 install transformers torch

# Create the user
RUN apt-get update \
    && apt-get install -y sudo \
    && echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME \
    && chmod 0440 /etc/sudoers.d/$USERNAME

RUN pip3 install pre-commit ipdb

RUN mkhomedir_helper triton-server

RUN apt-get install -y cmake rapidjson-dev

USER ${USERNAME}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43

# 컨테이너 안에서 
# 패키지 업데이트
apt update && apt install -y rapidjson-dev libarchive-dev

# Miniconda 설치
wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh
sudo bash Miniconda3-latest-Linux-x86_64.sh -b -p /opt/miniconda
export PATH="/opt/miniconda/bin:$PATH"

# TOS 승인
conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/main
conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/r

# conda 환경 초기화
conda init
source ~/.bashrc

# conda 환경 생성
conda create -n py38 python=3.8 -y
conda activate py38
pip install cmake==3.31.10

# Triton Python Backend 레포지토리 클론 및 빌드 작업 경로로 디렉토리 이동
git clone https://github.com/triton-inference-server/python_backend -b r25.09 # 자신에게 맞는 버전의 브랜치를 쓸 것.
cd python_backend
mkdir build && cd build

# 빌드
# GPU를 쓰지 않는다면, `-DTRITON_ENABLE_GPU=OFF`로 할 것.
# 여기서도 Triton 버전을 알맞게 표시할 것
cmake -DTRITON_ENABLE_GPU=ON \
 -DTRITON_BACKEND_REPO_TAG=r25.09 \
 -DTRITON_COMMON_REPO_TAG=r25.09 \
 -DTRITON_CORE_REPO_TAG=r25.09 \
 -DCMAKE_INSTALL_PREFIX:PATH=$(pwd)/install ..

make triton-python-backend-stub

# libpython3.8.so.1.0을 링크하는지 확인
ldd triton_python_backend_stub | grep python

# 빌드된 아티팩트 이동
cp triton_python_backend_stub /workspaces/triton-devcontainer/model_repository/add_one/triton_python_backend_stub

1
2
3
4
5
6

export PYTHONNOUSERSITE=True

pip install numpy
pip install conda-pack

conda pack -n py38 -o add_one_env.tar.gz

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18

~/triton-devcontainer on ☁️  (ap-northeast-2) 
# ldd에서 not found로 보임
20:55:46  ❯ ldd model_repository/add_one/triton_python_backend_stub | grep python 
        libpython3.8.so.1.0 => not found

# conda pack 압축해제
~/triton-devcontainer on ☁️  (ap-northeast-2) 
20:55:51  ❯ tar -xvf model_repository/add_one/add_one_env.tar.gz -C add_one_env                           

(...)

# libpython이 conda pack안에 있다
~/triton-devcontainer on ☁️  (ap-northeast-2) 
21:02:17  ❯ ls add_one_env/lib | grep python
libpython3.8.so -> libpython3.8.so.1.0
libpython3.8.so.1.0
libpython3.so
python3.8

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37

import triton_python_backend_utils as pb_utils
import numpy as np
import sys

class TritonPythonModel:
    """
    Triton Python Model
    """

    def initialize(self, args):
        pb_utils.Logger.log_info("Model Initializing...")
        pb_utils.Logger.log_info(sys.version) # 모델 로딩시의 버전을 확인해봅시다
        
    def execute(self, requests):
        responses = []
        for request in requests:
            input_tensor = pb_utils.get_input_tensor_by_name(request, "INPUT")
            input_array = input_tensor.as_numpy()

            output_array = input_array + 1

            output_tensor = pb_utils.Tensor(
                "OUTPUT",
                output_array.astype(np.int32)
            )

            response = pb_utils.InferenceResponse(
                output_tensors=[output_tensor]
            )

            responses.append(response)

        return responses

    def finalize(self):
        """finalizer"""
        pb_utils.Logger.log_info("Model cleanup...")

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31

name: "add_one"
backend: "python"

input [
  {
    name: "INPUT"
    data_type: TYPE_INT32
    dims: [ -1 ]
  }
]

output [
  {
    name: "OUTPUT"
    data_type: TYPE_INT32
    dims: [ -1 ]
  }
]

instance_group [
  {
    kind: KIND_CPU
  }
]

parameters: {
  key: "EXECUTION_ENV_PATH",
  value: {
    string_value: "$$TRITON_MODEL_DIRECTORY/add_one_env.tar.gz"
  }
}

1

docker run --rm -p 8000:8000 -p 8001:8001 -p 8002:8002 -v ./model_repository:/models nvcr.io/nvidia/tritonserver:25.09-py3 tritonserver --model-repository=/models # 태그 알맞게 작성!

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34

=============================
== Triton Inference Server ==
=============================

...

I0120 10:28:48.767944 1 model_lifecycle.cc:473] "loading: add_one:1"
I0120 10:28:48.772835 1 python_be.cc:1851] "Using Python execution env /models/add_one/add_one_env.tar.gz"
I0120 10:28:51.613833 1 python_be.cc:2289] "TRITONBACKEND_ModelInstanceInitialize: add_one_0_0 (CPU device 0)"
I0120 10:28:51.732199 1 model.py:11] "Model Initializing..."
I0120 10:28:51.732443 1 model.py:12] "3.8.20 (default, Oct  3 2024, 15:32:15) \n[GCC 11.2.0]" # << 여기!!
I0120 10:28:51.737114 1 model_lifecycle.cc:849] "successfully loaded 'add_one'"

...

I0120 10:28:51.737425 1 server.cc:638] 
+---------+-------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+
| Backend | Path                                                  | Config                                                                                                                                                        |
+---------+-------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+
| python  | /opt/tritonserver/backends/python/libtriton_python.so | {"cmdline":{"auto-complete-config":"true","backend-directory":"/opt/tritonserver/backends","min-compute-capability":"6.000000","default-max-batch-size":"4"}} |
+---------+-------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+

I0120 10:28:51.737493 1 server.cc:681] 
+---------+---------+--------+
| Model   | Version | Status |
+---------+---------+--------+
| add_one | 1       | READY  |
+---------+---------+--------+

...

I0120 10:28:51.743777 1 grpc_server.cc:2562] "Started GRPCInferenceService at 0.0.0.0:8001"
I0120 10:28:51.744013 1 http_server.cc:4789] "Started HTTPService at 0.0.0.0:8000"
I0120 10:28:51.786121 1 http_server.cc:358] "Started Metrics Service at 0.0.0.0:8002"