TensorFlow インスタンス・セグメント化モデルを OpenVINO™ に変換#

openvino_ir_model = core.read_model(openvino_ir_path) 
compiled_model = core.compile_model(model=openvino_ir_model, device_name=device.value)

model_inputs = compiled_model.inputs 
model_outputs = compiled_model.outputs 

print("Model inputs count:", len(model_inputs)) 
print("Model inputs:") 
for _input in model_inputs: 
    print(" ", _input) 

print("Model outputs count:", len(model_outputs)) 
print("Model outputs:") 
for output in model_outputs: 
    print(" ", output)

image_path = Path("./data/coco_bike.jpg") 

download_file( 
    url="https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/image/coco_bike.jpg", 
    filename=image_path.name, 
    directory=image_path.parent, 
);

# 画像を読み込み 
image = cv2.imread(filename=str(image_path)) 

# ネットワークは RGB 形式の画像を想定 
image = cv2.cvtColor(image, code=cv2.COLOR_BGR2RGB) 

# ネットワーク入力の形状に合わせて画像のサイズを変更
resized_image = cv2.resize(src=image, dsize=(255, 255)) 

# 画像にバッチ次元を追加 
network_input_image = np.expand_dims(resized_image, 0) 

# 画像を表示 
plt.imshow(image)

inference_result = compiled_model(network_input_image)

detection_boxes = compiled_model.output("detection_boxes") 
image_detection_boxes = inference_result[detection_boxes] 
print("image_detection_boxes:", image_detection_boxes.shape) 

detection_masks = compiled_model.output("detection_masks") 
image_detection_masks = inference_result[detection_masks] 
print("image_detection_masks:", image_detection_masks.shape) 

detection_classes = compiled_model.output("detection_classes") 
image_detection_classes = inference_result[detection_classes] 
print("image_detection_classes:", image_detection_classes.shape) 

detection_scores = compiled_model.output("detection_scores") 
image_detection_scores = inference_result[detection_scores] 
print("image_detection_scores:", image_detection_scores.shape) 

num_detections = compiled_model.output("num_detections") 
image_num_detections = inference_result[num_detections] 
print("image_detections_num:", image_num_detections) 

# あるいは、`.get()` メソッドを使用してモデル出力名で推論結果データを抽出することもできます。 
assert (inference_result[detection_boxes] == inference_result.get("detection_boxes")).all(), "extracted inference result data should be equal"

import random 
from typing import Optional 

def add_detection_box(box: np.ndarray, image: np.ndarray, mask: np.ndarray, label: Optional[str] = None) -> np.ndarray: 
    """ 
    Helper function for adding single bounding box to the image 

    Parameters 
    ---------- 
    box : np.ndarray 
        Bounding box coordinates in format [ymin, xmin, ymax, xmax] 
    image : np.ndarray 
        The image to which detection box is added 
    mask: np.ndarray 
        Segmentation mask in format (H, W) 
    label : str, optional 
        Detection box label string, if not provided will not be added to result image (default is None) 

    Returns 
    ------- 
    np.ndarray 
        NumPy array including image, detection box, and segmentation mask 
    """ 
    ymin, xmin, ymax, xmax = box 
    point1, point2 = (int(xmin), int(ymin)), (int(xmax), int(ymax)) 
    box_color = [random.randint(0, 255) for _ in range(3)] 
    line_thickness = round(0.002 * (image.shape[0] + image.shape[1]) / 2) + 1 

    result = cv2.rectangle( 
        img=image, 
        pt1=point1, 
        pt2=point2, 
        color=box_color, 
        thickness=line_thickness, 
        lineType=cv2.LINE_AA, 
    ) 

    if label: 
        font_thickness = max(line_thickness - 1, 1) 
        font_face = 0 
        font_scale = line_thickness / 3 
        font_color = (255, 255, 255) 
        text_size = cv2.getTextSize( 
            text=label, 
            fontFace=font_face, 
            fontScale=font_scale, 
            thickness=font_thickness, 
        )[0] 
        # 長方形の座標を計算 
        rectangle_point1 = point1 
        rectangle_point2 = (point1[0] + text_size[0], point1[1] - text_size[1] - 3) 
        # 塗りつぶされた四角形を追加 
        result = cv2.rectangle( 
            img=result, 
            pt1=rectangle_point1, 
            pt2=rectangle_point2, 
            color=box_color, 
            thickness=-1, 
            lineType=cv2.LINE_AA, 
        ) 
        # テキストの位置を計算 
        text_position = point1[0], point1[1] - 3 
        # 塗りつぶされた四角形にラベル付きのテキストを追加 
        result = cv2.putText( 
            img=result, 
            text=label, 
            org=text_position, 
            fontFace=font_face, 
            fontScale=font_scale, 
            color=font_color, 
            thickness=font_thickness, 
            lineType=cv2.LINE_AA, 
        ) 
    mask_img = mask[:, :, np.newaxis] * box_color 
    result = cv2.addWeighted(result, 1, mask_img.astype(np.uint8), 0.6, 0) 
    return result

def get_mask_frame(box, frame, mask): 
    """ 
    Transform a binary mask to fit within a specified bounding box in a frame using perspective transformation.

    Args: 
        box (tuple): A bounding box represented as a tuple (y_min, x_min, y_max, x_max). 
        frame (numpy.ndarray): The larger frame or image where the mask will be placed. 
    mask (numpy.ndarray): A binary mask image to be transformed.
 
    Returns: 
        numpy.ndarray: A transformed mask image that fits within the specified bounding box in the frame.
    """ 
    x_min = frame.shape[1] * box[1] 
    y_min = frame.shape[0] * box[0] 
    x_max = frame.shape[1] * box[3] 
    y_max = frame.shape[0] * box[2] 
    rect_src = np.array( 
        [ 
            [0, 0], 
            [mask.shape[1], 0], 
            [mask.shape[1], mask.shape[0]], 
            [0, mask.shape[0]], 
        ], 
        dtype=np.float32, 
    ) 
    rect_dst = np.array( 
        [[x_min, y_min], [x_max, y_min], [x_max, y_max], [x_min, y_max]], 
        dtype=np.float32, 
    ) 
    M = cv2.getPerspectiveTransform(rect_src[:, :], rect_dst[:, :]) 
    mask_frame = cv2.warpPerspective(mask, M, (frame.shape[1], frame.shape[0]), flags=cv2.INTER_CUBIC) 
    return mask_frame

from typing import Dict 

from openvino.runtime.utils.data_helpers import OVDict 

def visualize_inference_result( 
    inference_result: OVDict, 
    image: np.ndarray, 
    labels_map: Dict, 
    detections_limit: Optional[int] = None, 
): 
    """ 
    Helper function for visualizing inference result on the image 

    Parameters 
    ---------- 
    inference_result : OVDict 
        Result of the compiled model inference on the test image 
    image : np.ndarray 
        Original image to use for visualization 
    labels_map : Dict 
        Dictionary with mappings of detection classes numbers and its names 
    detections_limit : int, optional 
        Number of detections to show on the image, if not provided all detections will be shown (default is None) 
    """ 
    detection_boxes = inference_result.get("detection_boxes") 
    detection_classes = inference_result.get("detection_classes") 
    detection_scores = inference_result.get("detection_scores") 
    num_detections = inference_result.get("num_detections") 
    detection_masks = inference_result.get("detection_masks")
 
    detections_limit = int(min(detections_limit, num_detections[0]) if detections_limit is not None else num_detections[0])
 
    # 検出ボックスの座標を元の画像サイズに正規化 
    original_image_height, original_image_width, _ = image.shape 
    normalized_detection_boxes = detection_boxes[0, :detections_limit] * [ 
        original_image_height, 
        original_image_width, 
        original_image_height, 
        original_image_width, 
    ] 
    result = np.copy(image) 
    for i in range(detections_limit): 
        detected_class_name = labels_map[int(detection_classes[0, i])] 
        score = detection_scores[0, i] 
        mask = detection_masks[0, i] 
        mask_reframed = get_mask_frame(detection_boxes[0, i], image, mask) 
        mask_reframed = (mask_reframed > 0.5).astype(np.uint8) 
        label = f"{detected_class_name} {score:.2f}" 
        result = add_detection_box( 
            box=normalized_detection_boxes[i], 
            image=result, 
            mask=mask_reframed, label=label, 
        ) 

    plt.imshow(result)

coco_labels_file_path = Path("./data/coco_91cl.txt") 

download_file( 

url="https://raw.githubusercontent.com/openvinotoolkit/open_model_zoo/master/data/dataset_classes/coco_91cl.txt", 
    filename=coco_labels_file_path.name, 
    directory=coco_labels_file_path.parent, 
);

with open(coco_labels_file_path, "r") as file: 
    coco_labels = file.read().strip().split("\n") 
    coco_labels_map = dict(enumerate(coco_labels, 1)) 

print(coco_labels_map)

visualize_inference_result( 
    inference_result=inference_result, 
    image=image, 
    labels_map=coco_labels_map, 
    detections_limit=5, 
)