nanoLLaVA と OpenVINO による視覚言語アシスタント#

%pip install -q "torch>=2.1" "transformers>=4.40" "accelerate" "pillow" "gradio>=4.26" "openvino>=2024.1.0" "tqdm" "nncf>=2.10" --extra-index-url https://download.pytorch.org/whl/cpu

from huggingface_hub import snapshot_download 
from pathlib import Path 

model_local_dir = Path("nanoLLaVA") 

if not model_local_dir.exists(): 
    snapshot_download(repo_id="qnguyen3/nanoLLaVA", local_dir=model_local_dir) 

modeling_file = model_local_dir / "modeling_llava_qwen2.py" 
orig_modeling_file = model_local_dir / f"orig_{modeling_file.name}" 

# モデルコードは flash_attn パッケージに依存しており、読み込みに問題が発生する可能性があります。このパッケージのインポートを回避するためのパッチ・モデル・コード 
if not orig_modeling_file.exists(): 
    modeling_file.rename(orig_modeling_file) 
with orig_modeling_file.open("r") as f: 
    content = f.read() 
replacement_lines = [ 
    ("from flash_attn import flash_attn_func, flash_attn_varlen_func", ""), 
    ("from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input", ""), 
    (' _flash_supports_window_size = "window_size" in list(inspect.signature(flash_attn_func).parameters)', "pass"), 
] 

for replace_pair in replacement_lines: 
    content = content.replace(*replace_pair) 

with modeling_file.open("w") as f: 
    f.write(content)

import transformers 
from transformers import AutoModelForCausalLM, AutoTokenizer 
from PIL import Image 
import warnings 

transformers.logging.set_verbosity_error() 
warnings.filterwarnings("ignore") 

model = AutoModelForCausalLM.from_pretrained(model_local_dir, trust_remote_code=True) 
tokenizer = AutoTokenizer.from_pretrained(model_local_dir, trust_remote_code=True)

import torch 
import requests 

prompt = "Describe this image in detail" 

messages = [{"role": "user", "content": f"<image>\n{prompt}"}] 
text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) 

text_chunks = [tokenizer(chunk).input_ids for chunk in text.split("<image>")] 
input_ids = torch.tensor(text_chunks[0] + [-200] + text_chunks[1], dtype=torch.long).unsqueeze(0) 
url = "https://github.com/openvinotoolkit/openvino_notebooks/assets/29454499/8bf7d9f2-018a-4498-bec4-55f17c273ecc" 
image = Image.open(requests.get(url, stream=True).raw) 
image_tensor = model.process_images([image], model.config) 
print(prompt) 
image

from transformers import TextStreamer 

streamer = TextStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True) 

output_ids = model.generate(input_ids, images=image_tensor, max_new_tokens=128, use_cache=True, streamer=streamer)

from transformers.generation import GenerationConfig, GenerationMixin 
from transformers.modeling_outputs import CausalLMOutputWithPast 
from transformers import AutoConfig 
from transformers.image_processing_utils import BatchFeature, get_size_dict 
from transformers.image_transforms import ( 
    convert_to_rgb, 
    normalize, 
    rescale, 
    resize, 
    to_channel_dimension_format, 
) 
from transformers.image_utils import ( 
    ChannelDimension, 
    PILImageResampling, 
    to_numpy_array, 
) 
import numpy as np 
import torch 
from typing import Dict 
from functools import partial, reduce 

IGNORE_INDEX = -100 
IMAGE_TOKEN_INDEX = -200 

class ImageProcessor: 
    def __init__( 
        self, 
        image_mean=(0.5, 0.5, 0.5), 
        image_std=(0.5, 0.5, 0.5), 
        size=(384, 384), 
        crop_size: Dict[str, int] = None, 
        resample=PILImageResampling.BICUBIC, 
        rescale_factor=1 / 255, 
        data_format=ChannelDimension.FIRST, 
    ): 
        crop_size = crop_size if crop_size is not None else {"height": 384, "width": 384} 
        crop_size = get_size_dict(crop_size, default_to_square=True, param_name="crop_size") 

        self.image_mean = image_mean 
        self.image_std = image_std 
        self.size = size 
        self.resample = resample 
        self.rescale_factor = rescale_factor 
        self.data_format = data_format 
        self.crop_size = crop_size 

    def preprocess(self, images, return_tensors): 
        if isinstance(images, Image.Image): 
            images = [images] 
        else: 
            assert isinstance(images, list) 

        transforms = [ 
            convert_to_rgb, 
            to_numpy_array, 
            partial(resize, size=self.size, resample=self.resample, data_format=self.data_format), 
            partial(rescale, scale=self.rescale_factor, data_format=self.data_format), 
            partial(normalize, mean=self.image_mean, std=self.image_std, data_format=self.data_format), 
            partial(to_channel_dimension_format, channel_dim=self.data_format, input_channel_dim=self.data_format), 
        ] 

        images = reduce(lambda x, f: [*map(f, x)], transforms, images) 
        data = {"pixel_values": images} 

        return BatchFeature(data=data, tensor_type=return_tensors) 

class OVLlavaQwen2ForCausalLM(GenerationMixin): 
    def __init__(self, core, model_dir, device): 
        self.image_encoder = core.compile_model(model_dir / "image_encoder.xml", device) 
        self.embed_tokens = core.compile_model(model_dir / "token_embed.xml", device) 
        self.model = core.read_model(model_dir / "llava_with_past.xml") 
        self.input_names = {key.get_any_name(): idx for idx, key in enumerate(self.model.inputs)} 
        self.output_names = {key.get_any_name(): idx for idx, key in enumerate(self.model.outputs)} 
        self.key_value_input_names = [key for key in self.input_names if "key_values" in key] 
        self.key_value_output_names = [key for key in self.output_names if "present" in key] 
        compiled_model = core.compile_model(self.model, device) 
        self.request = compiled_model.create_infer_request() 
        self.config = AutoConfig.from_pretrained(model_dir) 
        self.generation_config = GenerationConfig.from_model_config(self.config) 
        self.main_input_name = "input_ids" 
        self.device = torch.device("cpu") 
        self.num_pkv = 2 
        self.image_processor = ImageProcessor() 
        self._supports_cache_class = False 

    def can_generate(self): 
        """Returns True to validate the check that the model using `GenerationMixin.generate()` can indeed generate.""" 
        return True 

    def __call__( 
        self, 
        input_ids: torch.LongTensor, 
        images: torch.Tensor, 
        attention_mask: Optional[torch.LongTensor] = None, 
        position_ids: Optional[torch.LongTensor] = None, 
        past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, 
        **kwargs, 
    ) -> CausalLMOutputWithPast: 
        return self.forward(input_ids, images, attention_mask, position_ids, past_key_values) 

    def forward( 
        self, 
        input_ids: torch.LongTensor, 
        images: torch.Tensor, 
        attention_mask: Optional[torch.LongTensor] = None, 
        position_ids: Optional[torch.LongTensor] = None, 
        past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, 
        **kwargs, 
    ) -> CausalLMOutputWithPast: 
        """General inference method""" 
        inputs = self.prepare_inputs_for_multimodal(input_ids, position_ids, attention_mask, past_key_values, images) 

        # Run inference 
        self.request.start_async(inputs, share_inputs=True) 
        self.request.wait() 

        logits = torch.from_numpy(self.request.get_tensor("logits").data) 

        # 長さが等しいタプル: レイヤー数 * デコーダーレイヤーごとの past_key_value の数 (2 は自己注意レイヤーに対応) 
        past_key_values = tuple(self.request.get_tensor(key).data for key in self.key_value_output_names) 
        # 長さが `n_layers` のタプルのタプル。各タプルの長さは 2 (自己注意の k/v) に等しい 

        past_key_values = tuple(past_key_values[i : i + self.num_pkv] for i in range(0, len(past_key_values), self.num_pkv)) 
        return CausalLMOutputWithPast(logits=logits, past_key_values=past_key_values) 

    def prepare_inputs_for_multimodal(self, input_ids, position_ids, attention_mask, past_key_values, images): 
        inputs = {} 
        if past_key_values is None: 
            past_key_values = self._dummy_past_key_values(input_ids.shape[0]) 
        else: 
            past_key_values = tuple(past_key_value for pkv_per_layer in past_key_values for past_key_value in pkv_per_layer) 
        inputs.update(zip(self.key_value_input_names, past_key_values)) 

        if images is None or input_ids.shape[1] == 1: 
            target_shape = past_key_values[-1][-1].shape[-2] + 1 if past_key_values is not None else input_ids.shape[1] 
            attention_mask = torch.cat( 
                ( 
                    attention_mask, 
                    torch.ones((attention_mask.shape[0], target_shape - attention_mask.shape[1]), dtype=attention_mask.dtype, device=attention_mask.device), 
                ), 
                dim=1, 
            ) 
            position_ids = torch.sum(attention_mask, dim=1).unsqueeze(-1) - 1 
            inputs_embeds = self.embed_tokens(input_ids)[0] 
            inputs["attention_mask"] = attention_mask.numpy() 
            inputs["position_ids"] = position_ids.numpy() 
            inputs["inputs_embeds"] = inputs_embeds 

            return inputs 

        if type(images) is list or images.ndim == 5: 
            concat_images = torch.cat([image for image in images], dim=0) 
            image_features = self.encode_images(concat_images) 
            split_sizes = [image.shape[0] for image in images] 
            image_features = torch.split(image_features, split_sizes, dim=0) 
            image_features = [x.flatten(0, 1).to(self.device) for x in image_features] 
        else: 
            image_features = self.encode_images(images).to(self.device) 

        # ダミーテンソルが存在しない場合は追加するだけにします 
        # 常に None を処理するのは面倒です
        # しかし、これは理想的ではありません。良いアイデアがある場合は、 
        # 問題を提起するか PR を送信してください。お願いします。 
        labels = None 
        _attention_mask = attention_mask 
        if attention_mask is None: 
            attention_mask = torch.ones_like(input_ids, dtype=torch.bool) 
        else: 
            attention_mask = attention_mask.bool() 
        if position_ids is None: 
            position_ids = torch.arange(0, input_ids.shape[1], dtype=torch.long, device=input_ids.device) 
        if labels is None: 
            labels = torch.full_like(input_ids, IGNORE_INDEX) 

        # attention_mask を使用してパディングを削除 -- TODO: 再確認 
        input_ids = [cur_input_ids[cur_attention_mask] for cur_input_ids, cur_attention_mask in zip(input_ids, attention_mask)] 
        labels = [cur_labels[cur_attention_mask] for cur_labels, cur_attention_mask in zip(labels, attention_mask)] 

        new_input_embeds = [] 
        new_labels = [] 
        cur_image_idx = 0 
        for batch_idx, cur_input_ids in enumerate(input_ids): 
            num_images = (cur_input_ids == IMAGE_TOKEN_INDEX).sum() 
            if num_images == 0: 
                cur_image_features = image_features[cur_image_idx] 
                cur_input_embeds_1 = self.embed_tokens(cur_input_ids) 
                cur_input_embeds = torch.cat([cur_input_embeds_1, cur_image_features[0:0]], dim=0) 
                new_input_embeds.append(cur_input_embeds) 
                new_labels.append(labels[batch_idx]) 
                cur_image_idx += 1 
                continue 

            image_token_indices = [-1] + torch.where(cur_input_ids == IMAGE_TOKEN_INDEX)[0].tolist() + [cur_input_ids.shape[0]] 
            cur_input_ids_noim = [] 
            cur_labels = labels[batch_idx] 
            cur_labels_noim = [] 
            for i in range(len(image_token_indices) - 1): 
                cur_input_ids_noim.append(cur_input_ids[image_token_indices[i] + 1 : image_token_indices[i + 1]]) 
                cur_labels_noim.append(cur_labels[image_token_indices[i] + 1 : image_token_indices[i + 1]]) 
            split_sizes = [x.shape[0] for x in cur_labels_noim] 
            cur_input_embeds = torch.from_numpy(self.embed_tokens(torch.cat(cur_input_ids_noim).unsqueeze(0))[0])[0] 
            cur_input_embeds_no_im = torch.split(cur_input_embeds, split_sizes, dim=0) 
            cur_new_input_embeds = [] 
            cur_new_labels = [] 

            for i in range(num_images + 1): 
                cur_new_input_embeds.append(cur_input_embeds_no_im[i]) 
                cur_new_labels.append(cur_labels_noim[i]) 
                if i < num_images: 
                    cur_image_features = image_features[cur_image_idx] 
                    cur_image_idx += 1 
                    cur_new_input_embeds.append(cur_image_features) 
                    cur_new_labels.append(torch.full((cur_image_features.shape[0],), IGNORE_INDEX, device=cur_labels.device, dtype=cur_labels.dtype)) 

            cur_new_input_embeds = torch.cat(cur_new_input_embeds) 
            cur_new_labels = torch.cat(cur_new_labels) 

            new_input_embeds.append(cur_new_input_embeds) 
            new_labels.append(cur_new_labels) 

        # 画像の埋め込みによりシーケンスが長くなる可能性があるため、シーケンスを最大長に切り捨て 
        tokenizer_model_max_length = getattr(self.config, "tokenizer_model_max_length", None) 
        if tokenizer_model_max_length is not None: 
            new_input_embeds = [x[:tokenizer_model_max_length] for x in new_input_embeds] 
            new_labels = [x[:tokenizer_model_max_length] for x in new_labels] 

        # 組み合わせる 
        max_len = max(x.shape[0] for x in new_input_embeds) 
        batch_size = len(new_input_embeds) 

        new_input_embeds_padded = [] 
        new_labels_padded = torch.full((batch_size, max_len), IGNORE_INDEX, dtype=new_labels[0].dtype, device=new_labels[0].device) 
        attention_mask = torch.zeros((batch_size, max_len), dtype=attention_mask.dtype, device=attention_mask.device) 
        position_ids = torch.zeros((batch_size, max_len), dtype=position_ids.dtype, device=position_ids.device) 

        for i, (cur_new_embed, cur_new_labels) in enumerate(zip(new_input_embeds, new_labels)): 
            cur_len = cur_new_embed.shape[0] 
            if getattr(self.config, "tokenizer_padding_side", "right") == "left": 
                new_input_embeds_padded.append( 
                    torch.cat( 
                        (torch.zeros((max_len - cur_len, cur_new_embed.shape[1]), dtype=cur_new_embed.dtype, device=cur_new_embed.device), cur_new_embed), dim=0 
                    ) 
                ) 
                if cur_len > 0: 
                    new_labels_padded[i, -cur_len:]= cur_new_labels 
                    attention_mask[i, -cur_len:]= True 
                    position_ids[i, -cur_len:]= torch.arange(0, cur_len, dtype=position_ids.dtype, device=position_ids.device) 
                else: 
                    new_input_embeds_padded.append( 
                        torch.cat( 
                            (cur_new_embed, torch.zeros((max_len - cur_len, cur_new_embed.shape[1]), dtype=cur_new_embed.dtype, device=cur_new_embed.device)), dim=0 
                        ) 
                    ) 
                    if cur_len > 0: 
                        new_labels_padded[i, :cur_len] = cur_new_labels 
                        attention_mask[i, :cur_len] = True 
                        position_ids[i, :cur_len] = torch.arange(0, cur_len, dtype=position_ids.dtype, device=position_ids.device) 

        new_input_embeds = torch.stack(new_input_embeds_padded, dim=0) 
        attention_mask = attention_mask.to(dtype=_attention_mask.dtype) 
        inputs["inputs_embeds"] = new_input_embeds.numpy() 
        inputs["attention_mask"] = attention_mask.numpy() 
        inputs["position_ids"] = position_ids.numpy() 

        return inputs 

    def prepare_inputs_for_generation(self, input_ids, past_key_values=None, **kwargs): 
        """ 
        This function is used during running GenerationMixin.generate for preparing model specific inputs for 
        each generation step 
        """ 
        past_len = 0 
        if past_key_values is not None: 
            input_ids = input_ids[:, -1].unsqueeze(-1) 
            past_len = past_key_values[-1][-1].shape[-2] 
        attention_mask = kwargs.get( 
            "attention_mask", 
            torch.ones(input_ids.shape[0], input_ids.shape[1] + past_len), 
        ) 
        return { 
            "input_ids": input_ids, 
            "attention_mask": attention_mask, 
            "position_ids": kwargs.get("position_ids", None), 
            "past_key_values": past_key_values, 
            "images": kwargs.get("images", None), 
        } 

    def _reorder_cache(self, past_key_values: Tuple[Tuple[torch.Tensor]], beam_idx: torch.Tensor) -> Tuple[Tuple[torch.Tensor]]: 
        """ 
        This function is used to re-order the `past_key_values` cache if [`~PreTrainedModel.beam_search`] or 
        [`~PreTrainedModel.beam_sample`] is called. 
        This is required to match `past_key_values` with the correct beam_idx at every generation step.

        """ 

        # from transformers.models.gpt2.modeling_gpt2.GPT2LMHeadModel._reorder_cache 
        return tuple(tuple(np.take(past_state, beam_idx, 0) for past_state in layer_past) for layer_past in past_key_values) 

    def _dummy_past_key_values(self, batch_size): 
        pkv = [] 
        for input_name in self.key_value_input_names: 
            input_t = self.model.input(input_name) 
            input_shape = self.model.input(input_name).get_partial_shape() 
            input_shape[0] = batch_size 
            input_shape[2] = 0 
            pkv.append(ov.Tensor(input_t.get_element_type(), input_shape.get_shape())) 

        return pkv 

    def encode_images(self, images): 
        return torch.from_numpy(self.image_encoder(images)[0]) 

    def expand2square(self, pil_img, background_color): 
        width, height = pil_img.size 
        if width == height: 
            return pil_img 
        elif width > height: 
            result = Image.new(pil_img.mode, (width, width), background_color) 
            result.paste(pil_img, (0, (width - height) // 2)) return result 
        else: 
            result = Image.new(pil_img.mode, (height, height), background_color) 
            result.paste(pil_img, ((height - width) // 2, 0)) 
            return result 

    def process_images(self, images, model_cfg): 
        image_aspect_ratio = getattr(model_cfg, "image_aspect_ratio", None) 
        new_images = [] 
        if image_aspect_ratio == "pad": 
            for image in images: 
                image = self.expand2square(image, tuple(int(x * 255) for x in self.image_processor.image_mean)) 
                image = self.image_processor.preprocess(image, return_tensors="pt")["pixel_values"][0] 
                new_images.append(image) 
        else: 
            return self.image_processor(images, return_tensors="pt")["pixel_values"] 
        if all(x.shape == new_images[0].shape for x in new_images): 
            new_images = torch.stack(new_images, dim=0) 
        return new_images

import ipywidgets as widgets 

core = ov.Core() 

support_devices = core.available_devices 
if "NPU" in support_devices: 
    support_devices.remove("NPU") 

device = widgets.Dropdown( 
    options=support_devices + ["AUTO"], 
    value="AUTO", 
    description="Device:", 
    disabled=False, 
) 

device

ov_model = OVLlavaQwen2ForCausalLM(core, ov_out_path, device.value)

streamer = TextStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True) 

output_ids = ov_model.generate(input_ids, images=image_tensor, max_new_tokens=128, use_cache=True, streamer=streamer)

import gradio as gr 
import time 
from transformers import TextIteratorStreamer, StoppingCriteria 
from threading import Thread 
import requests 

example_image_urls = [ 
    ( 
        "https://github.com/openvinotoolkit/openvino_notebooks/assets/29454499/1d6a0188-5613-418d-a1fd-4560aae1d907", 
        "bee.jpg", 
    ), 
    ( 
        "https://github.com/openvinotoolkit/openvino_notebooks/assets/29454499/6cc7feeb-0721-4b5d-8791-2576ed9d2863", 
        "baklava.png", 
    ), 
    (
        "https://github.com/openvinotoolkit/openvino_notebooks/assets/29454499/dd5105d6-6a64-4935-8a34-3058a82c8d5d", 
        "small.png"
    ), 
    (
        "https://github.com/openvinotoolkit/openvino_notebooks/assets/29454499/1221e2a8-a6da-413a-9af6-f04d56af3754", 
        "chart.png"
    ), 
] 
for url, file_name in example_image_urls: 
    if not Path(file_name).exists():         Image.open(requests.get(url, stream=True).raw).save(file_name) 

class KeywordsStoppingCriteria(StoppingCriteria): 
    def __init__(self, keywords, tokenizer, input_ids): 
        self.keywords = keywords 
        self.keyword_ids = [] 
        self.max_keyword_len = 0 
        for keyword in keywords: 
            cur_keyword_ids = tokenizer(keyword).input_ids 
            if len(cur_keyword_ids) > 1 and cur_keyword_ids[0] == tokenizer.bos_token_id: 
                cur_keyword_ids = cur_keyword_ids[1:] 
            if len(cur_keyword_ids) > self.max_keyword_len: 
                self.max_keyword_len = len(cur_keyword_ids) 
            self.keyword_ids.append(torch.tensor(cur_keyword_ids)) 
            self.tokenizer = tokenizer self.start_len = input_ids.shape[1] 

    def call_for_batch(self, output_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> bool: 
        offset = min(output_ids.shape[1] - self.start_len, self.max_keyword_len) 
        self.keyword_ids = [keyword_id.to(output_ids.device) 
        for keyword_id in self.keyword_ids] for keyword_id in self.keyword_ids: 
            truncated_output_ids = output_ids[0, -keyword_id.shape[0] :] 
            if torch.equal(truncated_output_ids, keyword_id): 
                return True 
        outputs = self.tokenizer.batch_decode(output_ids[:, -offset:], skip_special_tokens=True)[0] 
        for keyword in self.keywords: 
            if keyword in outputs: 
                return True 
        return False 

    def __call__(self, output_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> bool: 
        outputs = [] 
        for i in range(output_ids.shape[0]): 
            outputs.append(self.call_for_batch(output_ids[i].unsqueeze(0), scores)) 
        return all(outputs) 

def bot_streaming(message, history): 
    messages = [] 
    if message["files"]: 
        image = message["files"][-1]["path"] if isinstance(message["files"][-1], dict) else message["files"][-1] 
    else: 
        for _, hist in enumerate(history): 
            if isinstance(hist[0], tuple): 
                image = hist[0][0] 

    if len(history) > 0 and image is not None: 
        messages.append({"role": "user", "content": f"<image>\n{history[1][0]}"}) 
        messages.append({"role": "assistant", "content": history[1][1]}) 
        for human, assistant in history[2:]: 
            if assistant is None: 
                continue 
            messages.append({"role": "user", "content": human}) 
            messages.append({"role": "assistant", "content": assistant}) 
        messages.append({"role": "user", "content": message["text"]}) 
    elif len(history) > 0 and image is None: 
        for human, assistant in history: 
            if assistant is None: 
                continue 
            messages.append({"role": "user", "content": human}) 
            messages.append({"role": "assistant", "content": assistant}) 
        messages.append({"role": "user", "content": message["text"]}) 
    elif len(history) == 0 and image is not None: 
        messages.append({"role": "user", "content": f"<image>\n{message['text']}"}) 
    elif len(history) == 0 and image is None: 
        messages.append({"role": "user", "content": message["text"]}) 

    print(messages) 
    image = Image.open(image).convert("RGB") 
    text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) 
    text_chunks = [tokenizer(chunk).input_ids for chunk in text.split("<image>")] 
    input_ids = torch.tensor(text_chunks[0] + [-200] + text_chunks[1], dtype=torch.long).unsqueeze(0) 
    stop_str = "<|im_end|>" 
    keywords = [stop_str] 
    stopping_criteria = KeywordsStoppingCriteria(keywords, tokenizer, input_ids) 
    streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True) 

    image_tensor = ov_model.process_images([image], ov_model.config) 
    generation_kwargs = dict( 
        input_ids=input_ids, images=image_tensor, streamer=streamer, max_new_tokens=128, stopping_criteria=[stopping_criteria], temperature=0.01 
    ) 
    thread = Thread(target=ov_model.generate, kwargs=generation_kwargs) 
    thread.start() 

    buffer = "" 
    for new_text in streamer: 
        buffer += new_text 
        generated_text_without_prompt = buffer[:] 
        time.sleep(0.04) 
        yield generated_text_without_prompt 

demo = gr.ChatInterface( 
    fn=bot_streaming, 
    title="🚀nanoLLaVA", 
    examples=[ 
        {"text": "What is on the flower?", "files": ["./bee.jpg"]}, 
        {"text": "How to make this pastry?", "files": ["./baklava.png"]}, 
        {"text": "What is the text saying?", "files": ["./small.png"]}, 
        {"text": "What does the chart display?", "files": ["./chart.png"]}, 
    ], 
    description="Try [nanoLLaVA](https://huggingface.co/qnguyen3/nanoLLaVA) using OpenVINO in this demo.Upload an image and start chatting about it, or simply try one of the examples below.If you don't upload an image, you will receive an error.", 
    stop_btn="Stop Generation", 
    multimodal=True, 
) 

# リモートで起動する場合は、server_name と server_port を指定 
# demo.launch(server_name='your server name', server_port='server port in int') 
# 詳細については、ドキュメントをご覧ください: https://gradio.app/docs/ 
try: 
    demo.launch(debug=False) 
except Exception: 
    demo.launch(share=True, debug=False)