AI多模态实战指南：从CLIP到视觉语言大模型

def contrastive_loss(image_features, text_features, temperature=0.07):
    # 归一化
    image_features = image_features / image_features.norm(dim=1, keepdim=True)
    text_features = text_features / text_features.norm(dim=1, keepdim=True)

    # 相似度矩阵
    logits = (image_features @ text_features.T) / temperature

    # 对角线是正样本
    batch_size = image_features.shape[0]
    labels = torch.arange(batch_size)

    # 双向交叉熵
    loss_i = F.cross_entropy(logits, labels)
    loss_t = F.cross_entropy(logits.T, labels)

    return (loss_i + loss_t) / 2

import clip
import torch
from PIL import Image

device = "cuda" if torch.cuda.is_available() else "cpu"
model, preprocess = clip.load("ViT-B/32", device=device)

image = preprocess(Image.open("product.jpg")).unsqueeze(0).to(device)
text = clip.tokenize([
    "a photo of a red dress",
    "a photo of a blue shirt",
    "a photo of black pants"
]).to(device)

with torch.no_grad():
    image_features = model.encode_image(image)
    text_features = model.encode_text(text)
    logits_per_image, _ = model(image, text)
    probs = logits_per_image.softmax(dim=-1).cpu().numpy()

print("Label probs:", probs)

# 不好的提示词（准确率 ~60%）
prompts = ["dress", "shirt", "pants", "shoes"]

# 好的提示词（准确率 85%+）
prompts = [
    "A photo of a dress",
    "A photo of a shirt",
    "A photo of pants",
    "A photo of shoes"
]

# 更精细的提示词
prompts = [
    "A photo of a red dress, fashion, clothing",
    "A photo of a beach, ocean, sand, summer"
]

class CLIPClassifier(nn.Module):
    def __init__(self, clip_model, num_classes):
        super().__init__()
        self.clip_model = clip_model
        self.classifier = nn.Linear(512, num_classes)  # CLIP 特征维度 512

    def forward(self, images):
        with torch.no_grad():
            features = self.clip_model.encode_image(images)
        return self.classifier(features)

with torch.no_grad():
    image_features = []
    for i in range(0, len(images), 8):
        batch = images[i:i+8]
        image_features.append(model.encode_image(batch))
    image_features = torch.cat(image_features)

image = cv2.imread("product.jpg")
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
image = Image.fromarray(image)
image = preprocess(image)

def image_to_text_search(query_image, text_database, model):
    image_feature = model.encode_image(preprocess(query_image).unsqueeze(0))
    text_features = model.encode_text(clip.tokenize(text_database))

    similarities = (image_feature @ text_features.T).squeeze(0)
    best_idx = similarities.argmax().item()
    return text_database[best_idx]

def anomaly_detection(image, normal_descriptions, threshold=0.25):
    image_feature = model.encode_image(preprocess(image).unsqueeze(0))
    text_features = model.encode_text(clip.tokenize(normal_descriptions))

    similarities = (image_feature @ text_features.T).squeeze(0)
    max_sim = similarities.max().item()

    return max_sim < threshold  # 最匹配的都不够匹配 → 异常

# 第一级：ResNet 粗排（top 50）
def coarse_search(query_image, top_k=50):
    pass

# 第二级：CLIP 精排
def fine_search(query_text, candidate_indices):
    candidate_features = image_db[candidate_indices]
    query_features = model.get_text_features(...)
    similarities = (candidate_features @ query_features.T).squeeze()
    return candidate_indices[similarities.argsort()[::-1]]

def clean_coco_annotations(annotations_path, output_path):
    with open(annotations_path) as f:
        data = json.load(f)

    filtered = []
    for ann in data['annotations']:
        caption = ann['caption'].lower().strip()
        words = caption.split()

        if len(words) < 10:  # 太短
            continue

        word_counts = Counter(words)
        max_repeat = max(word_counts.values())
        if max_repeat / len(words) > 0.3:  # 重复词过多
            continue

        filtered.append(ann)

    data['annotations'] = filtered
    with open(output_path, 'w') as f:
        json.dump(data, f)

def forward(self, images, captions, teacher_forcing_ratio=0.5):
    # ... 省略部分代码 ...
    for t in range(max_length):
        output, hidden = self.rnn(input, hidden)
        outputs[:, t, :] = self.linear(output.squeeze(1))

        # 一半概率用真实标签，一半用模型预测
        teacher_force = random.random() < teacher_forcing_ratio
        top1 = output.argmax(1)
        input = captions[:, t] if teacher_force else top1
        input = self.embedding(input).unsqueeze(1)
    return outputs

from transformers import BlipProcessor, BlipForConditionalGeneration

processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-base")
model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-base")

image = Image.open("photo.jpg").convert('RGB')
inputs = processor(image, return_tensors="pt")
out = model.generate(**inputs)
caption = processor.decode(out[0], skip_special_tokens=True)

def vqa_clip(image_path, question, answer_candidates):
    image_features = model.get_image_features(...)
    question_features = model.get_text_features(...)
    answer_features = model.get_text_features(answer_candidates)

    # 组合特征（简化为加法）
    combined_features = (image_features + question_features).cpu().numpy().squeeze()
    similarities = (answer_features @ combined_features)
    return answer_candidates[similarities.argmax()]

from transformers import LlavaProcessor, LlavaForConditionalGeneration

processor = LlavaProcessor.from_pretrained("llava-hf/llava-1.5-7b-hf")
model = LlavaForConditionalGeneration.from_pretrained("llava-hf/llava-1.5-7b-hf")

prompt = "<image>\nUSER: 图里左边那个人穿什么颜色？\nASSISTANT:"
image = Image.open("group_photo.jpg").convert('RGB')
inputs = processor(text=prompt, images=image, return_tensors="pt")
output = model.generate(**inputs, max_new_tokens=50)
answer = processor.decode(output[0], skip_special_tokens=True)

def vqa_hybrid(image_path, question):
    simple_keywords = ['是什么', '有多少', '什么颜色', '几只']
    if any(kw in question for kw in simple_keywords):
        return vqa_clip(image_path, question, simple_answer_candidates)
    else:
        return vqa_llava(image_path, question)

# 错误做法
combined_features = torch.cat([
    image_features.squeeze(0),  # CLIP: 512 维
    text_inputs['input_ids'].squeeze(0).float()  # Llama: 4096 维
], dim=0)

from transformers import BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_compute_dtype=torch.float16,
    bnb_4bit_use_double_quant=True,
    bnb_4bit_quant_type="nf4"
)

model = LlavaForConditionalGeneration.from_pretrained(
    "llava-hf/llava-1.5-7b-hf",
    quantization_config=quantization_config,
    device_map="auto"
)

from vllm import LLM, SamplingParams

llm = LLM(
    model="llava-hf/llava-1.5-7b-hf",
    quantization="awq",
    tensor_parallel_size=1
)

sampling_params = SamplingParams(temperature=0.0, top_p=1.0, max_tokens=500)
outputs = llm.generate([prompt], sampling_params)

def adaptive_image_resize(image, max_pixels=512*512):
    width, height = image.size
    current_pixels = width * height

    if current_pixels > max_pixels:
        scale_factor = (max_pixels / current_pixels) ** 0.5
        new_width = int(width * scale_factor)
        new_height = int(height * scale_factor)
        image = image.resize((new_width, new_height), Image.Resampling.LANCZOS)

    return image

def robust_inference(image, question, confidence_threshold=0.6):
    classification = image_classifier(image)
    confidence = classification['confidence']

    if confidence < confidence_threshold:
        return "抱歉，这张图片不够清晰，我无法准确识别。"

    answer = llm.generate(image, question)

    if "不确定" in answer or "可能" in answer:
        answer += "\n\n（注：建议您提供更多细节以获得更准确的判断。）"

    return answer

class SmartRouter:
    def __init__(self):
        self.small_model = load_small_model()  # 1B
        self.large_model = load_large_model()  # 7B
        self.cache = LRUCache(maxsize=10000)

    def route(self, image, question):
        cache_key = generate_cache_key(image, question)
        if cache_key in self.cache:
            return self.cache[cache_key]

        complexity = estimate_complexity(question)
        model = self.small_model if complexity == "low" else self.large_model

        answer = model.generate(image, question)
        self.cache[cache_key] = answer
        return answer