OpenVINO™ による自動デバイス選択#

# LOG_LEVEL を LOG_INFO に設定 
core.set_property("AUTO", {"LOG_LEVEL": "LOG_INFO"}) 

# モデルをターゲットデバイスにロード 
compiled_model = core.compile_model(ov_model) 

if isinstance(compiled_model, ov.CompiledModel): 
    print("Successfully compiled model without a device_name.")

# 削除されたモデルは、選択したデバイスでのコンパイルが完了するまで待機 
del compiled_model 
print("Deleted compiled_model")

# LOG_LEVEL を LOG_NONE に設定 
core.set_property("AUTO", {"LOG_LEVEL": "LOG_NONE"}) 

compiled_model = core.compile_model(model=ov_model, device_name="AUTO") 

if isinstance(compiled_model, ov.CompiledModel): 
    print("Successfully compiled model using AUTO.")

# 削除されたモデルは、選択したデバイスでのコンパイルが完了するまで待機 
del compiled_model 
print("Deleted compiled_model")

# `notebook_utils` モジュールを取得 
import requests 

r = requests.get(
url="https://raw.githubusercontent.com/openvinotoolkit/openvino_notebooks/latest/utils/notebook_utils.py") 
open("notebook_utils.py", "w").write(r.text) 

from notebook_utils import download_file

from PIL import Image 

# Download the image from the openvino_notebooks storage 
image_filename = download_file( 
    "https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/image/coco.jpg", 
    directory="data", 
) 

image = Image.open(str(image_filename)) 
input_transform = torchvision.models.ResNet50_Weights.DEFAULT.transforms() 

input_tensor = input_transform(image) 
input_tensor = input_tensor.unsqueeze(0).numpy() 
image

if not any("GPU" in device for device in core.available_devices): 
    print(f"A GPU device is not available. Available devices are: {core.available_devices}") 
else:
    # 開始時間 
    gpu_load_start_time = time.perf_counter() 
    compiled_model = core.compile_model(model=ov_model, device_name="GPU") # load to GPU 

    # 最初の推論を実行 
    results = compiled_model(input_tensor)[0] 

    # 最初の推論までの時間を測定 
    gpu_fil_end_time = time.perf_counter() 
    gpu_fil_span = gpu_fil_end_time - gpu_load_start_time 
    print(f"Time to load model on GPU device and get first inference: {gpu_fil_end_time-gpu_load_start_time:.2f} seconds.") 
    del compiled_model

# 開始時間 
auto_load_start_time = time.perf_counter() 
compiled_model = core.compile_model(model=ov_model) # device_name はデフォルトでは AUTO

# 最初の推論を実行 
results = compiled_model(input_tensor)[0] 

# 最初の推論までの時間を測定 
auto_fil_end_time = time.perf_counter() 
auto_fil_span = auto_fil_end_time - auto_load_start_time 
print(f"Time to load model using AUTO device and get first inference: {auto_fil_end_time-auto_load_start_time:.2f} seconds.")

# 削除されたモデルは、選択したデバイスでのコンパイルが完了するまで待機 
del compiled_model

class PerformanceMetrics: 
    """ 
    Record the latest performance metrics (fps and latency), update the metrics in each @interval seconds 
    :member: fps: Frames per second, indicates the average number of inferences executed each second during the last @interval seconds. 
    :member: latency: Average latency of inferences executed in the last @interval seconds. 
    :member: start_time: Record the start timestamp of onging @interval seconds duration. 
    :member: latency_list: Record the latency of each inference execution over @interval seconds duration. 
    :member: interval: The metrics will be updated every @interval seconds 
    """ 

    def __init__(self, interval): 
        """ 
        Create and initilize one instance of class PerformanceMetrics. 
        :param: interval: The metrics will be updated every @interval seconds 
        :returns:
            Instance of PerformanceMetrics 
        """ 
        self.fps = 0 
        self.latency = 0
 
        self.start_time = time.perf_counter() 
        self.latency_list = [] 
        self.interval = interval 

    def update(self, infer_request: ov.InferRequest) -> bool: 
        """ 
        Update the metrics if current ongoing @interval seconds duration is expired. Record the latency only if it is not expired. 
        :param: infer_request: InferRequest returned from inference callback, which includes the result of inference request. 
        :returns: 
            True, if metrics are updated. 
            False, if @interval seconds duration is not expired and metrics are not updated.
        """ 
        self.latency_list.append(infer_request.latency) 
        exec_time = time.perf_counter() - self.start_time 
        if exec_time >= self.interval:
            # パフォーマンス・メトリックを更新 
            self.start_time = time.perf_counter() 
            self.fps = len(self.latency_list) / exec_time 
            self.latency = sum(self.latency_list) / len(self.latency_list) 
            print(f"throughput: {self.fps: .2f}fps, latency: {self.latency: .2f}ms, time interval:{exec_time: .2f}s") 
            sys.stdout.flush() 
            self.latency_list = [] 
            return True 
        else: 
            return False 

class InferContext: 
    """ 
    Inference context.Record and update peforamnce metrics via @metrics, set @feed_inference to False once @remaining_update_num <=0 
    :member: metrics: instance of class PerformanceMetrics 
    :member: remaining_update_num: the remaining times for peforamnce metrics updating. 
    :member: feed_inference: if feed inference request is required or not.
    """ 

    def __init__(self, update_interval, num): 
        """ 
        Create and initilize one instance of class InferContext. 
        :param: update_interval: The performance metrics will be updated every @update_interval seconds. This parameter will be passed to class PerformanceMetrics directly. 
        :param: num: The number of times performance metrics are updated. 
        :returns: 
            Instance of InferContext.
        """ 
        self.metrics = PerformanceMetrics(update_interval) 
        self.remaining_update_num = num 
        self.feed_inference = True 

    def update(self, infer_request: ov.InferRequest): 
        """ 
        Update the context.Set @feed_inference to False if the number of remaining performance metric updates (@remaining_update_num) reaches 0 
        :param: infer_request: InferRequest returned from inference callback, which includes the result of inference request. 
        :returns: None 
        """ 
        if self.remaining_update_num <= 0: 
            self.feed_inference = False 

        if self.metrics.update(infer_request): 
            self.remaining_update_num = self.remaining_update_num - 1 
            if self.remaining_update_num <= 0: 
                self.feed_inference = False 

def completion_callback(infer_request: ov.InferRequest, context) -> None: 
    """ 
    callback for the inference request, pass the @infer_request to @context for updating 
    :param: infer_request: InferRequest returned for the callback, which includes the result of inference request.
    :param: context: user data which is passed as the second parameter to AsyncInferQueue:start_async() 
    :returns: None 
    """ 
    context.update(infer_request) 

# パフォーマンス・メトリックの更新間隔 (秒) と回数 
metrics_update_interval = 10 
metrics_update_num = 6

THROUGHPUT_hint_context = InferContext(metrics_update_interval, metrics_update_num) 

print("Compiling Model for AUTO device with THROUGHPUT hint") 
sys.stdout.flush() 

compiled_model = core.compile_model(model=ov_model, config={"PERFORMANCE_HINT": "THROUGHPUT"}) 

infer_queue = ov.AsyncInferQueue(compiled_model, 0) # 0 に設定すると、デフォルトで最適な数を照会 
infer_queue.set_callback(completion_callback) 

print(f"Start inference, {metrics_update_num: .0f} groups of FPS/latency will be measured over {metrics_update_interval: .0f}s intervals") 
sys.stdout.flush() 

while THROUGHPUT_hint_context.feed_inference: 
    infer_queue.start_async(input_tensor, THROUGHPUT_hint_context) 

infer_queue.wait_all() 

# 最新期間の FPS とレイテンシーを取得THROUGHPUT_hint_fps = THROUGHPUT_hint_context.metrics.fps 
THROUGHPUT_hint_latency = THROUGHPUT_hint_context.metrics.latency 

print("Done") 

del compiled_model

LATENCY_hint_context = InferContext(metrics_update_interval, metrics_update_num) 

print("Compiling Model for AUTO Device with LATENCY hint") 
sys.stdout.flush() 

compiled_model = core.compile_model(model=ov_model, config={"PERFORMANCE_HINT": "LATENCY"}) 

# 0 に設定すると、デフォルトで最適な数を照会 
infer_queue = ov.AsyncInferQueue(compiled_model, 0) 
infer_queue.set_callback(completion_callback) 

print(f"Start inference, {metrics_update_num: .0f} groups fps/latency will be out with {metrics_update_interval: .0f}s interval") 
sys.stdout.flush() 

while LATENCY_hint_context.feed_inference: 
    infer_queue.start_async(input_tensor, LATENCY_hint_context) 

infer_queue.wait_all() 

# 最新期間の FPS とレイテンシーを取得LATENCY_hint_fps = LATENCY_hint_context.metrics.fps 
LATENCY_hint_latency = LATENCY_hint_context.metrics.latency 

print("Done") 
del compiled_model

import matplotlib.pyplot as plt 

TPUT = 0 
LAT = 1 
labels = ["THROUGHPUT hint", "LATENCY hint"] 

fig1, ax1 = plt.subplots(1, 1) 
fig1.patch.set_visible(False) 
ax1.axis("tight") 
ax1.axis("off") 

cell_text = [] 
cell_text.append( 
    [ 
        "%.2f%s" % (THROUGHPUT_hint_fps, " FPS"), 
        "%.2f%s" % (THROUGHPUT_hint_latency, " ms"), 
    ] 
) 
cell_text.append(["%.2f%s" % (LATENCY_hint_fps, " FPS"), "%.2f%s" % (LATENCY_hint_latency, " ms")]) 

table = ax1.table( 
    cellText=cell_text, 
    colLabels=["FPS (Higher is better)", "Latency (Lower is better)"], 
    rowLabels=labels, 
    rowColours=["deepskyblue"] * 2, 
    colColours=["deepskyblue"] * 2, 
    cellLoc="center", 
    loc="upper left", 
) 
table.auto_set_font_size(False) 
table.set_fontsize(18) 
table.auto_set_column_width(0) 
table.auto_set_column_width(1) 
table.scale(1, 3) 

fig1.tight_layout() 
plt.show()

# 差を出力 
width = 0.4 
fontsize = 14 

plt.rc("font", size=fontsize) 
fig, ax = plt.subplots(1, 2, figsize=(10, 8)) 

rects1 = ax[0].bar([0], THROUGHPUT_hint_fps, width, label=labels[TPUT], color="#557f2d") 
rects2 = ax[0].bar([width], LATENCY_hint_fps, width, label=labels[LAT]) 
ax[0].set_ylabel("frames per second") 
ax[0].set_xticks([width / 2]) 
ax[0].set_xticklabels(["FPS"]) 
ax[0].set_xlabel("Higher is better") 

rects1 = ax[1].bar([0], THROUGHPUT_hint_latency, width, label=labels[TPUT], color="#557f2d") 
rects2 = ax[1].bar([width], LATENCY_hint_latency, width, label=labels[LAT]) 
ax[1].set_ylabel("milliseconds") 
ax[1].set_xticks([width / 2]) 
ax[1].set_xticklabels(["Latency (ms)"]) 
ax[1].set_xlabel("Lower is better") 

fig.suptitle("Performance Hints") 
fig.legend(labels, fontsize=fontsize) 
fig.tight_layout() plt.show()