AI无监督学习折腾手记

import pandas as pd
import numpy as np
from sklearn.preprocessing import StandardScaler
from sklearn.impute import SimpleImputer

# 读取数据
df = pd.read_parquet('user_behavior.parquet')

# 先看数据概况
print(f"原始数据形状: {df.shape}")
print(f"缺失值占比:\n{df.isnull().sum() / len(df)}")

# 处理缺失值
# 对于数值型特征，用中位数填充
numeric_features = df.select_dtypes(include=[np.number]).columns
imputer = SimpleImputer(strategy='median')
df[numeric_features] = imputer.fit_transform(df[numeric_features])

# 处理异常值
def cap_outliers(df, column, lower=0.01, upper=0.99):
    lower_bound = df[column].quantile(lower)
    upper_bound = df[column].quantile(upper)
    df[column] = df[column].clip(lower_bound, upper_bound)
    return df

for col in ['daily_clicks', 'daily_views', 'total_spend']:
    df = cap_outliers(df, col)

# 特征标准化（重要！）
scaler = StandardScaler()
scaled_features = scaler.fit_transform(df[numeric_features])

from sklearn.decomposition import PCA
import matplotlib.pyplot as plt

# PCA降维
pca = PCA()
pca.fit(scaled_features)

# 看看降维效果
plt.figure(figsize=(10, 6))
plt.plot(np.cumsum(pca.explained_variance_ratio_))
plt.xlabel('Number of Components')
plt.ylabel('Cumulative Explained Variance')
plt.axhline(y=0.9, color='r', linestyle='--')
plt.show()

# 选择保留90%方差的主成分数量
n_components_90 = np.argmax(np.cumsum(pca.explained_variance_ratio_) >= 0.9) + 1
print(f"保留90%方差需要 {n_components_90} 个主成分")

# 实际降维
pca_final = PCA(n_components=n_components_90)
reduced_features = pca_final.fit_transform(scaled_features)

from sklearn.cluster import KMeans
from sklearn.metrics import silhouette_score

# 用肘部法则和轮廓系数找最佳K
inertias = []
silhouette_scores = []
K_range = range(2, 15)

for k in K_range:
    kmeans = KMeans(n_clusters=k, random_state=42, n_init=10)
    kmeans.fit(reduced_features)
    inertias.append(kmeans.inertia_)
    silhouette_scores.append(silhouette_score(reduced_features, kmeans.labels_))

# 画肘部图
plt.figure(figsize=(12, 5))
plt.subplot(1, 2, 1)
plt.plot(K_range, inertias, 'bo-')
plt.xlabel('K')
plt.ylabel('Inertia')
plt.title('Elbow Method')

# 画轮廓系数
plt.subplot(1, 2, 2)
plt.plot(K_range, silhouette_scores, 'go-')
plt.xlabel('K')
plt.ylabel('Silhouette Score')
plt.title('Silhouette Score')
plt.tight_layout()
plt.show()

# 最终聚类
kmeans = KMeans(n_clusters=5, random_state=42, n_init=10)
clusters = kmeans.fit_predict(reduced_features)

# 添加聚类标签
df['cluster'] = clusters

# 分析各簇的特征
cluster_stats = df.groupby('cluster')[numeric_features].agg(['mean', 'std'])
print(cluster_stats)

# 可视化（用前两个主成分）
plt.figure(figsize=(10, 8))
scatter = plt.scatter(reduced_features[:, 0], reduced_features[:, 1],
                     c=clusters, cmap='viridis', alpha=0.6)
plt.colorbar(scatter)
plt.xlabel('PC1')
plt.ylabel('PC2')
plt.title('User Clusters (PCA Reduced)')
plt.show()

from sklearn.cluster import MiniBatchKMeans

mbk = MiniBatchKMeans(n_clusters=5, batch_size=50000, random_state=42)
clusters = mbk.fit_predict(reduced_features)

# 簇的"画像"
for cluster_id in range(5):
    cluster_users = df[df['cluster'] == cluster_id]
    print(f"\n=== Cluster {cluster_id} ===")
    print(f"用户数量: {len(cluster_users)}")
    print(f"平均购买金额: {cluster_users['total_spend'].mean():.2f}")
    print(f"平均点击次数: {cluster_users['daily_clicks'].mean():.2f}")