FILM と OpenVINO を使用したフレーム補間#

import platform 
import os 

%pip install -q "tensorflow-macos>=2.5; sys_platform == 'darwin' and platform_machine == 'arm64' and python_version > '3.8'" # macOS M1 and M2 
%pip install -q "tensorflow-macos>=2.5,<=2.12.0; sys_platform == 'darwin' and platform_machine == 'arm64' and python_version <= '3.8'" # macOS M1 and M2 
%pip install -q "tensorflow>=2.5; sys_platform == 'darwin' and platform_machine != 'arm64' and python_version > '3.8'" # macOS x86 
%pip install -q "tensorflow>=2.5,<=2.12.0; sys_platform == 'darwin' and platform_machine != 'arm64' and python_version <= '3.8'" # macOS x86 
%pip install -q "tensorflow>=2.5; sys_platform != 'darwin' and python_version > '3.8'" 
%pip install -q "tensorflow>=2.5,<=2.12.0; sys_platform != 'darwin' and python_version <= '3.8'" 

%pip install -q tensorflow_hub tf_keras numpy "opencv-python" tqdm "gradio>=4.19" Pillow "openvino>=2023.2.0" 

if platform.system() != "Windows":
     %pip install -q "matplotlib>=3.4" 
else:
     %pip install -q "matplotlib>=3.4,<3.7"

from pathlib import Path 
import requests 
from typing import Optional, Generator 
from datetime import datetime 
import gc 

os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2" 
os.environ["TF_USE_LEGACY_KERAS"] = "1" 
os.environ["TFHUB_CACHE_DIR"] = str(Path("./tfhub_modules").resolve()) 

import tensorflow_hub as hub 
import tensorflow as tf 
import openvino as ov 
import numpy as np 
import cv2 
import matplotlib.pyplot as plt 
from tqdm.auto import tqdm 
import gradio as gr 
import PIL 
import IPython

MODEL_PATH = Path("models/model.xml") 
DATA_PATH = Path("data") 
IMAGES = { 
    "https://github.com/openvinotoolkit/openvino_notebooks/assets/29454499/c3ddf65f-95ec-44ca-9ed4-3ef2d8f4b47e": Path("data/one.jpg"), 
    "https://github.com/openvinotoolkit/openvino_notebooks/assets/29454499/6d21f1ce-69eb-41b5-aedd-6e3c29013b30": Path("data/two.jpg"), 
} 
OUTPUT_VIDEO_PATH = DATA_PATH / "output.webm" 
OV_OUTPUT_VIDEO_PATH = DATA_PATH / "ov_output.webm" 
TIMES_TO_INTERPOLATE = 5 
DATA_PATH.mkdir(parents=True, exist_ok=True) 

PIL.ImageFile.LOAD_TRUNCATED_IMAGES = True 
# Gradio は大きなメタデータを持つ PNG 画像を読み取ることができます

def preprocess_np_frame(frame): 
    result = frame.astype(np.float32) / 255 # [0, 1] に正規化 
    result = result[np.newaxis, ...] # バッチ次元を追加 
    return result 

def prepare_input(img_url: str): 
    if not IMAGES[img_url].exists(): 
        r = requests.get(img_url) 
        with IMAGES[img_url].open("wb") as f: 
            f.write(r.content) 
    filename = str(IMAGES[img_url]) 
    img = cv2.imread(filename) 
    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) 
    img = np.array(img) 
    img = preprocess_np_frame(img) 

    return img 

input_images = [prepare_input(url) for url in IMAGES] 

input = { 
    "x0": input_images[0], 
    "x1": input_images[1], 
    "time": np.array([[0.5]], dtype=np.float32), 
}

plt.figure(figsize=(16, 8), layout="tight") 
plt.subplot(1, 2, 1) 
plt.imshow(input_images[0][0]) 
plt.axis("off") 
plt.subplot(1, 2, 2) 
plt.imshow(input_images[1][0]) 
plt.axis("off");

inputs = dict( 
    x0=tf.keras.layers.Input(shape=(None, None, 3)), 
    x1=tf.keras.layers.Input(shape=(None, None, 3)), 
    time=tf.keras.layers.Input(shape=(1)), 
) 
model_url = 
"https://www.kaggle.com/models/google/film/frameworks/tensorFlow2/variations/film/versions/1" 
film_layer = hub.KerasLayer(model_url)(inputs) 
film_model = tf.keras.Model(inputs=inputs, outputs=film_layer)

if not MODEL_PATH.exists(): 
    converted_model = ov.convert_model(film_model) 
    ov.save_model(converted_model, MODEL_PATH) 
    del converted_model 
del film_model 
gc.collect()

def generate(frame1, frame2, times_to_interpolate, _=gr.Progress(track_tqdm=True)): 
    x0, x1 = [preprocess_np_frame(frame) for frame in [frame1, frame2]] 
    frames = ov_interpolator.interpolate_recursively(x0, x1, times_to_interpolate) 
    height, width = frame1.shape[:2] 
    filename = DATA_PATH / f"output_{datetime.now().isoformat()}.webm" 
    save_as_video(frames, width, height, filename) 
    return filename 

demo = gr.Interface( 
    generate, 
    [ 
        gr.Image(label="First image"), 
        gr.Image(label="Last image"), 
        gr.Slider( 
            1, 
            8, 
            step=1, 
            label="Times to interpolate", 
            info="""Controls the number of times the frame interpolator is invoked.
        The output will be the interpolation video with (2^value + 1) frames, fps of 30.""", 
        ), 
    ], 
    gr.Video(), 
    examples=[[*IMAGES.values(), 5]], 
    allow_flagging="never", 
) 
try: 
    demo.queue().launch(debug=False) 
except Exception: 
    demo.queue().launch(share=True, debug=False) 
# リモートで起動する場合は、server_name と server_port を指定 
# demo.launch(server_name='your server name', server_port='server port in int') 
# 詳細については、ドキュメントをご覧ください: https://gradio.app/docs/