Machine Learning Toolbox

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier
from sklearn.inspection import permutation_importance
from sklearn.metrics import mean_squared_error, r2_score
from sklearn.model_selection import train_test_split

# ==========================================
# 1. Configuration & Data Loading
# ==========================================
# Set plot style
sns.set_style("whitegrid")
plt.rcParams['font.sans-serif'] = ['SimHei'] # For Chinese characters
plt.rcParams['axes.unicode_minus'] = False

# Model Parameters
RF_PARAMS = {
    'n_estimators': 100,
    'max_depth': 10,
    'min_samples_split': 2,
    'n_jobs': -1,
    'random_state': 42
}

# Load Data (Replace with your data source)
# df = pd.read_csv('your_data.csv')
# X = df.drop(['target', 'date'], axis=1)
# y = df['target']

# For demonstration, generate random data
from sklearn.datasets import make_regression
X, y = make_regression(n_samples=1000, n_features=20, n_informative=5, random_state=42)
feature_names = [f'Feature_{i}' for i in range(X.shape[1])]

# Split Data
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# ==========================================
# 2. Model Training
# ==========================================
print("Training Random Forest...")
model = RandomForestRegressor(**RF_PARAMS) # Use RandomForestClassifier for classification
model.fit(X_train, y_train)

# Basic Evaluation
y_pred = model.predict(X_test)
print(f"Test R2 Score: {r2_score(y_test, y_pred):.4f}")

# ==========================================
# 3. Feature Importance (Gini & Permutation)
# ==========================================

# A. Gini Importance (MDI) - Fast but biased towards high cardinality features
gini_importance = model.feature_importances_
gini_df = pd.DataFrame({
    'Feature': feature_names,
    'Gini_Importance': gini_importance
}).sort_values('Gini_Importance', ascending=False)

# B. Permutation Importance - Slower but more reliable
print("Calculating Permutation Importance...")
perm_result = permutation_importance(
    model, X_test, y_test,
    n_repeats=10,
    random_state=42,
    n_jobs=-1
)

perm_df = pd.DataFrame({
    'Feature': feature_names,
    'Permutation_Importance': perm_result.importances_mean,
    'Permutation_Std': perm_result.importances_std
}).sort_values('Permutation_Importance', ascending=False)

# ==========================================
# 4. Visualization
# ==========================================
def plot_importance(df, value_col, title):
    plt.figure(figsize=(10, 6))
    # Plot top 20 features
    plot_df = df.head(20).sort_values(value_col, ascending=True)

    plt.barh(range(len(plot_df)), plot_df[value_col], color='#1f77b4', alpha=0.8)
    plt.yticks(range(len(plot_df)), plot_df['Feature'])
    plt.xlabel('Importance')
    plt.title(title)
    plt.tight_layout()
    plt.show()

# Plot Gini Importance
plot_importance(gini_df, 'Gini_Importance', 'Random Forest Feature Importance (Gini)')

# Plot Permutation Importance
plot_importance(perm_df, 'Permutation_Importance', 'Random Forest Feature Importance (Permutation)')

# Cumulative Importance (to decide how many features to keep)
gini_sorted = gini_df.sort_values('Gini_Importance', ascending=False)
cumulative_importance = np.cumsum(gini_sorted['Gini_Importance'])

plt.figure(figsize=(10, 5))
plt.plot(range(1, len(cumulative_importance) + 1), cumulative_importance, 'b-o')
plt.axhline(y=0.9, color='r', linestyle='--')
plt.text(0, 0.91, '90% Threshold', color='r')
plt.xlabel('Number of Features')
plt.ylabel('Cumulative Importance')
plt.title('Cumulative Feature Importance')
plt.grid(True)
plt.show()

# ==========================================
# 5. Output Report
# ==========================================
print("\nTop 10 Features (Permutation):")
print(perm_df.head(10))