OpenVINO™ による非同期推論#

# モデルがダウンロードされるディレクトリー 
base_model_dir = "model" 

# Open Model Zoo で命名されたモデル名 
model_name = "person-detection-0202" 
precision = "FP16" 
model_path = f"model/intel/{model_name}/{precision}/{model_name}.xml" 
download_command = f"omz_downloader " f"--name {model_name} " f"--precision {precision} " f"--output_dir {base_model_dir} " f"--cache_dir {base_model_dir}" 
! $download_command

import ipywidgets as widgets 

core = ov.Core()
device = widgets.Dropdown( 
    options=core.available_devices + ["AUTO"], 
    value="CPU", 
    description="Device:", 
    disabled=False, 
) 

device

# OpenVINO ランタイムを初期化 
core = ov.Core() 

# ファイルからネットワークと対応する重みを読み取り 
model = core.read_model(model=model_path) 

# CPU 用にモデルをコンパイルする (CPU、GPU などを手動で選択) か、 
# エンジンに利用可能な最適なデバイスを選択させる (AUTO) 
compiled_model = core.compile_model(model=model, device_name=device.value) 

# 入力ノードを取得 
input_layer_ir = model.input(0) 
N, C, H, W = input_layer_ir.shape 
shape = (H, W)

def preprocess(image):     """ 
    Define the preprocess function for input data 

    :param: image: the orignal input frame 
    :returns: 
        resized_image: the image processed 
    """ 
    resized_image = cv2.resize(image, shape) 
    resized_image = cv2.cvtColor(np.array(resized_image), cv2.COLOR_BGR2RGB) 
    resized_image = resized_image.transpose((2, 0, 1)) 
    resized_image = np.expand_dims(resized_image, axis=0).astype(np.float32) 
    return resized_image 

def postprocess(result, image, fps): 
    """ 
    Define the postprocess function for output data 

    :param: result: the inference results 
        image: the orignal input frame 
        fps: average throughput calculated for each frame 
    :returns: 
    image: the image with bounding box and fps message 
    """ 
    detections = result.reshape(-1, 7) 
    for i, detection in enumerate(detections):
        _, image_id, confidence, xmin, ymin, xmax, ymax = detection 
        if confidence > 0.5: 
            xmin = int(max((xmin * image.shape[1]), 10)) 
            ymin = int(max((ymin * image.shape[0]), 10)) 
            xmax = int(min((xmax * image.shape[1]), image.shape[1] - 10)) 
            ymax = int(min((ymax * image.shape[0]), image.shape[0] - 10)) 
            cv2.rectangle(image, (xmin, ymin), (xmax, ymax), (0, 255, 0), 2) 
            cv2.putText( 
                image, 
                str(round(fps, 2)) + " fps", 
                (5, 20), 
                cv2.FONT_HERSHEY_SIMPLEX, 
                0.7, 
                (0, 255, 0), 
                3, ) 
    return image

video_path = 
"https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/video/CEO%20Pat%20Gelsinger%20on%20Leading%20Intel.mp4"

def sync_api(source, flip, fps, use_popup, skip_first_frames): 
    """ 
    Define the main function for video processing in sync mode 

    :param: source: the video path or the ID of your webcam 
    :returns: 
        sync_fps: the inference throughput in sync mode 
    """ 
    frame_number = 0 
    infer_request = compiled_model.create_infer_request() 
    player = None 
    try:
        # ビデオプレーヤーを作成 
        player = utils.VideoPlayer(source, flip=flip, fps=fps, skip_first_frames=skip_first_frames) 
        # キャプチャーを開始 
        start_time = time.time() 
        player.start() 
        if use_popup: 
            title = "Press ESC to Exit" 
            cv2.namedWindow(title, cv2.WINDOW_GUI_NORMAL | cv2.WINDOW_AUTOSIZE) 
        while True: 
            frame = player.next() 
            if frame is None: 
                print("Source ended") 
                break 
            resized_frame = preprocess(frame) 
            infer_request.set_tensor(input_layer_ir, ov.Tensor(resized_frame)) 
            # 同期モードで推論要求を開始 
            infer_request.infer() 
            res = infer_request.get_output_tensor(0).data 
            stop_time = time.time() 
            total_time = stop_time - start_time 
            frame_number = frame_number + 1 
            sync_fps = frame_number / total_time 
            frame = postprocess(res, frame, sync_fps) 
            # 結果を表示 
            if use_popup: 
                cv2.imshow(title, frame) 
                key = cv2.waitKey(1) 
                # escape = 27 
                if key == 27: 
                    break 
            else:
                # numpy 配列を jpg にエンコード 
                _, encoded_img = cv2.imencode(".jpg", frame, params=[cv2.IMWRITE_JPEG_QUALITY, 90]) 
                # Ipython イメージを作成 
                i = display.Image(data=encoded_img) 
                # このノートブックに画像を表示 
                display.clear_output(wait=True) 
                display.display(i) 
# ctrl-c 
except KeyboardInterrupt: 
    print("Interrupted") 
# 異なるエラー 
except RuntimeError as e: 
    print(e) 
finally: 
    if use_popup: 
        cv2.destroyAllWindows() 
    if player is not None:
        # キャプチャーを停止 
        player.stop() 
return sync_fps

sync_fps = sync_api(source=video_path, flip=False, fps=30, use_popup=False, skip_first_frames=800) 
print(f"average throuput in sync mode: {sync_fps:.2f} fps")

def async_api(source, flip, fps, use_popup, skip_first_frames): 
    """ 
    Define the main function for video processing in async mode 

    :param: source: the video path or the ID of your webcam 
    :returns: 
        async_fps: the inference throughput in async mode 
    """ 
    frame_number = 0 
    # 2 つの推論要求を作成 
    curr_request = compiled_model.create_infer_request() 
    next_request = compiled_model.create_infer_request() 
    player = None 
    async_fps = 0 
    try:
        # ビデオプレーヤーを作成 
        player = utils.VideoPlayer(source, flip=flip, fps=fps, skip_first_frames=skip_first_frames) 
        # キャプチャーを開始 
        start_time = time.time() 
        player.start() 
        if use_popup: 
            title = "Press ESC to Exit" 
            cv2.namedWindow(title, cv2.WINDOW_GUI_NORMAL | cv2.WINDOW_AUTOSIZE)
        # 現在のフレームをキャプチャー 
        frame = player.next() 
        resized_frame = preprocess(frame) 
        curr_request.set_tensor(input_layer_ir, ov.Tensor(resized_frame)) 
        # 現在の推論要求を開始 
        curr_request.start_async() 
        while True:
            # 次のフレームをキャプチャー 
            next_frame = player.next() 
            if next_frame is None: 
                print("Source ended") 
                break 
            resized_frame = preprocess(next_frame) 
            next_request.set_tensor(input_layer_ir, ov.Tensor(resized_frame)) 
            # 次の推論要求を開始 
            next_request.start_async() 
            # 現在の推論結果を待機 
            curr_request.wait() 
            res = curr_request.get_output_tensor(0).data 
            stop_time = time.time() 
            total_time = stop_time - start_time 
            frame_number = frame_number + 1 
            async_fps = frame_number / total_time 
            frame = postprocess(res, frame, async_fps) 
            # 結果を表示 
            if use_popup: 
                cv2.imshow(title, frame) 
                key = cv2.waitKey(1) 
                # escape = 27 
                if key == 27: 
                    break 
            else:
                # numpy 配列を jpg にエンコード 
                _, encoded_img = cv2.imencode(".jpg", frame, params=[cv2.IMWRITE_JPEG_QUALITY, 90]) 
                # Ipython イメージを作成 
                i = display.Image(data=encoded_img) 
                # このノートブックに画像を表示 
                display.clear_output(wait=True) 
                display.display(i) 
            # 現在のフレームと次のフレームを入れ替え 
            frame = next_frame 
            # 現在の推論要求と次の推論要求を入れ替え 
            curr_request, next_request = next_request, curr_request 
# ctrl-c 
except KeyboardInterrupt: 
    print("Interrupted") 
# 異なるエラー 
except RuntimeError as e: 
    print(e) 
finally: 
    if use_popup: 
        cv2.destroyAllWindows() 
    if player is not None:
        # キャプチャーを停止 
        player.stop() 
    return async_fps

async_fps = async_api(source=video_path, flip=False, fps=30, use_popup=False, skip_first_frames=800) 
print(f"average throuput in async mode: {async_fps:.2f} fps")

width = 0.4 
fontsize = 14 

plt.rc("font", size=fontsize) 
fig, ax = plt.subplots(1, 1, figsize=(10, 8)) 

rects1 = ax.bar([0], sync_fps, width, color="#557f2d") 
rects2 = ax.bar([width], async_fps, width) 
ax.set_ylabel("frames per second") 
ax.set_xticks([0, width]) 
ax.set_xticklabels(["Sync mode", "Async mode"]) 
ax.set_xlabel("Higher is better") 

fig.suptitle("Sync mode VS Async mode") 
fig.tight_layout() 

plt.show()

def callback(infer_request, info) -> None: 
    """ 
    Define the callback function for postprocessing 

    :param: infer_request: the infer_request object 
        info: a tuple includes original frame and starts time 
    :returns:
        None 
    """ 
    global frame_number 
    global total_time 
    global inferqueue_fps 
    stop_time = time.time() 
    frame, start_time = info 
    total_time = stop_time - start_time 
    frame_number = frame_number + 1 
    inferqueue_fps = frame_number / total_time 

    res = infer_request.get_output_tensor(0).data[0] 
    frame = postprocess(res, frame, inferqueue_fps) 
    # numpy 配列を jpg にエンコード 
    _, encoded_img = cv2.imencode(".jpg", frame, params=[cv2.IMWRITE_JPEG_QUALITY, 90]) 
    # Ipython イメージを作成 
    i = display.Image(data=encoded_img) 
    # このノートブックに画像を表示 
    display.clear_output(wait=True) 
    display.display(i)

def inferqueue(source, flip, fps, skip_first_frames) -> None: 
    """ 
    Define the main function for video processing with async infer queue 

    :param: source: the video path or the ID of your webcam 
    :retuns:
         None 
    """ 
    # Create infer requests queue 
    infer_queue = ov.AsyncInferQueue(compiled_model, 2) 
    infer_queue.set_callback(callback) 
    player = None 
    try:
        # ビデオプレーヤーを作成 
        player = utils.VideoPlayer(source, flip=flip, fps=fps, skip_first_frames=skip_first_frames) 
        # キャプチャーを開始 
        start_time = time.time() 
        player.start() 
        while True:
            # フレームをキャプチャー 
            frame = player.next() 
            if frame is None: 
                print("Source ended") 
                break 
            resized_frame = preprocess(frame) 
            # 非同期推論キューで推論リクエストを開始 
            infer_queue.start_async({input_layer_ir.any_name: resized_frame}, (frame, start_time)) 
    except KeyboardInterrupt: 
        print("Interrupted") 
    # 異なるエラー 
    except RuntimeError as e: 
        print(e) 
    finally: 
        infer_queue.wait_all() 
        player.stop()

frame_number = 0 
total_time = 0 
inferqueue(source=video_path, flip=False, fps=30, skip_first_frames=800) 
print(f"average throughput in async mode with async infer queue: {inferqueue_fps:.2f} fps")