# xgboost修改版本 import os import pickle import pandas as pd import numpy as np import tkinter as tk import tkinter.font as tkfont from tkinter import ttk from datetime import timedelta from time import time import matplotlib.pyplot as plt from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg, NavigationToolbar2Tk from xgboost import XGBRegressor from lunardate import LunarDate from sklearn.model_selection import train_test_split from sklearn.metrics import mean_squared_error, mean_absolute_error import matplotlib # 配置 matplotlib 中文显示 matplotlib.rcParams['font.sans-serif'] = ['SimHei', 'Microsoft YaHei', 'SimSun', 'Arial Unicode MS'] matplotlib.rcParams['axes.unicode_minus'] = False matplotlib.rcParams['font.family'] = 'sans-serif' # 全局缓存变量及特征名称（此处 feature_columns 仅为占位） cached_model = None last_training_time = None feature_columns = None # 数据加载与预处理函数 # ------------------------------- def load_data(upstream_file, downstream_file): """ 加载所有相关数据 """ try: upstream_df = pd.read_csv(upstream_file) downstream_df = pd.read_csv(downstream_file) except FileNotFoundError: print("文件未找到，请检查路径") return None # 假设原始数据列依次为 ['DateTime', 'TagName', 'Value'] upstream_df.columns = ['DateTime', 'TagName', 'Value'] downstream_df.columns = ['DateTime', 'TagName', 'Value'] # 转换时间格式及数值处理 upstream_df['DateTime'] = pd.to_datetime(upstream_df['DateTime']) downstream_df['DateTime'] = pd.to_datetime(downstream_df['DateTime']) # 数值处理 upstream_df['Value'] = pd.to_numeric(upstream_df['Value'], errors='coerce') downstream_df['Value'] = pd.to_numeric(downstream_df['Value'], errors='coerce') # 过滤盐度小于5的数据 这里数据可以更改 upstream_df = upstream_df[upstream_df['Value'] >= 5] downstream_df = downstream_df[downstream_df['Value'] >= 5] # 将0替换为NaN，并利用3倍标准差法处理异常值 数据处理平滑 for df in [upstream_df, downstream_df]: df.loc[df['Value'] == 0, 'Value'] = np.nan mean_val, std_val = df['Value'].mean(), df['Value'].std() lower_bound, upper_bound = mean_val - 3 * std_val, mean_val + 3 * std_val df.loc[(df['Value'] < lower_bound) | (df['Value'] > upper_bound), 'Value'] = np.nan # 重命名 Value 列并保留需要的列 upstream_df = upstream_df.rename(columns={'Value': 'upstream'})[['DateTime', 'upstream']] downstream_df = downstream_df.rename(columns={'Value': 'downstream'})[['DateTime', 'downstream']] # 合并数据 merged_df = pd.merge(upstream_df, downstream_df, on='DateTime', how='inner') print(f"合并前数据行数: {len(merged_df)}") merged_df = merged_df.set_index('DateTime') # 插值：先用线性，再用时间插值，最后用前向后向填充 merged_df['upstream'] = merged_df['upstream'].interpolate(method='linear', limit=4) merged_df['downstream'] = merged_df['downstream'].interpolate(method='linear', limit=4) merged_df['upstream'] = merged_df['upstream'].interpolate(method='time', limit=24) merged_df['downstream'] = merged_df['downstream'].interpolate(method='time', limit=24) merged_df['upstream'] = merged_df['upstream'].fillna(method='ffill').fillna(method='bfill') merged_df['downstream'] = merged_df['downstream'].fillna(method='ffill').fillna(method='bfill') # 平滑处理：使用滑动窗口移动平均 merged_df['upstream_smooth'] = merged_df['upstream'].rolling(window=24, min_periods=1, center=True).mean() merged_df['downstream_smooth'] = merged_df['downstream'].rolling(window=24, min_periods=1, center=True).mean() # 对低盐度部分用更大窗口平滑 low_sal_mask = merged_df['upstream'] < 50 if low_sal_mask.any(): merged_df.loc[low_sal_mask, 'upstream_smooth'] = merged_df.loc[low_sal_mask, 'upstream']\ .rolling(window=48, min_periods=1, center=True).mean() merged_df = merged_df.dropna() merged_df = merged_df[merged_df['upstream'].apply(np.isfinite)] merged_df = merged_df[merged_df['downstream'].apply(np.isfinite)] merged_df = merged_df.reset_index() print(f"清洗后数据行数: {len(merged_df)}") print(f"上游盐度范围: {merged_df['upstream'].min()} - {merged_df['upstream'].max()}") print(f"下游盐度范围: {merged_df['downstream'].min()} - {merged_df['downstream'].max()}") merged_df = merged_df.sort_values('DateTime') return merged_df # 测试 # df = load_data('青龙港1.csv', '一取水.csv') # df.to_csv('merged_data.csv', index=False) # print(f"Merged data saved to 'merged_data.csv' successfully") # # 绘制盐度随时间变化图 # plt.figure(figsize=(12, 6)) # plt.plot(df['DateTime'], df['upstream_smooth'], label='上游盐度', color='blue') # plt.plot(df['DateTime'], df['downstream_smooth'], label='下游盐度', color='red') # plt.xlabel('时间') # plt.ylabel('盐度') # plt.title('盐度随时间变化图') # plt.legend() # plt.grid(True) # plt.tight_layout() # plt.savefig('salinity_time_series.png', dpi=300) # plt.show() # ----------------------特征工程部分 # ------------------------------- # 添加农历（潮汐）特征 # ------------------------------- def add_lunar_features(df): lunar_day, lunar_phase_sin, lunar_phase_cos, is_high_tide = [], [], [], [] for dt in df['DateTime']: ld = LunarDate.fromSolarDate(dt.year, dt.month, dt.day) lunar_day.append(ld.day) lunar_phase_sin.append(np.sin(2 * np.pi * ld.day / 15)) lunar_phase_cos.append(np.cos(2 * np.pi * ld.day / 15)) is_high_tide.append(1 if (ld.day <= 5 or (ld.day >= 16 and ld.day <= 20)) else 0) df['lunar_day'] = lunar_day df['lunar_phase_sin'] = lunar_phase_sin df['lunar_phase_cos'] = lunar_phase_cos df['is_high_tide'] = is_high_tide return df # ------------------------------- # 生成延迟特征（向量化，利用 shift） # ------------------------------- def batch_create_delay_features(df, delay_hours): for delay in delay_hours: df[f'upstream_delay_{delay}h'] = df['upstream_smooth'].shift(delay) df[f'downstream_delay_{delay}h'] = df['downstream_smooth'].shift(delay) return df # ------------------------------- # 添加时间特征 # ------------------------------- def add_time_features(df): df['hour'] = df['DateTime'].dt.hour df['weekday'] = df['DateTime'].dt.dayofweek df['month'] = df['DateTime'].dt.month return df # ------------------------------- # 添加统计特征 # ------------------------------- def add_statistical_features(df): # 1天统计特征 df['mean_1d_up'] = df['upstream_smooth'].rolling(window=24).mean() df['std_1d_up'] = df['upstream_smooth'].rolling(window=24).std() df['max_1d_up'] = df['upstream_smooth'].rolling(window=24).max() df['min_1d_up'] = df['upstream_smooth'].rolling(window=24).min() df['mean_1d_down'] = df['downstream_smooth'].rolling(window=24).mean() df['std_1d_down'] = df['downstream_smooth'].rolling(window=24).std() df['max_1d_down'] = df['downstream_smooth'].rolling(window=24).max() df['min_1d_down'] = df['downstream_smooth'].rolling(window=24).min() # 3天统计特征 df['mean_3d_up'] = df['upstream_smooth'].rolling(window=72).mean() df['mean_3d_down'] = df['downstream_smooth'].rolling(window=72).mean() return df # 应用特征工程并保存数据 if __name__ == "__main__": df = load_data('青龙港1.csv', '一取水.csv') # 添加时间特征 df = add_time_features(df) # 添加农历特征 df = add_lunar_features(df) # 添加统计特征 df = add_statistical_features(df) # 添加延迟特征 - 设置延迟小时数为1,2,3,6,12,24,48,72 delay_hours = [1, 2, 3, 6, 12, 24, 48, 72] df = batch_create_delay_features(df, delay_hours) # # 保存带有全部特征的数据 # df.to_csv('feature_engineered_data.csv', index=False) # print(f"特征工程后的数据已保存到 'feature_engineered_data.csv'，共{len(df)}行，{len(df.columns)}列") # 清除NaN值 df_clean = df.dropna() print(f"删除NaN后的数据行数: {len(df_clean)}") # 进行特征相关性分析 print("\n进行特征相关性分析...") # 选择数值型列进行相关性分析 numeric_cols = df_clean.select_dtypes(include=['float64', 'int64']).columns.tolist() # 排除DateTime列 if 'DateTime' in numeric_cols: numeric_cols.remove('DateTime') # 计算相关矩阵 corr_matrix = df_clean[numeric_cols].corr() # 保存相关矩阵到CSV corr_matrix.to_csv('feature_correlation_matrix.csv') print("相关矩阵已保存到 'feature_correlation_matrix.csv'") # 1. 计算与下游盐度(目标变量)的相关性 target_corrs = corr_matrix['downstream_smooth'].sort_values(ascending=False) target_corrs.to_csv('target_correlation.csv') print("\n与下游盐度最相关的前10个特征:") print(target_corrs.head(10)) # 2. 绘制相关性热图 plt.figure(figsize=(16, 14)) import seaborn as sns sns.heatmap(corr_matrix, annot=False, cmap='coolwarm', center=0, linewidths=0.5) plt.title('特征相关性热图', fontsize=16) plt.tight_layout() plt.savefig('correlation_heatmap.png', dpi=300) plt.close() print("相关性热图已保存到 'correlation_heatmap.png'") # 3. 绘制与目标变量相关性最高的前15个特征的条形图 plt.figure(figsize=(12, 8)) target_corrs.iloc[1:16].plot(kind='barh', color='skyblue') # 排除自身相关性(=1) plt.title('与下游盐度相关性最高的15个特征', fontsize=14) plt.xlabel('相关系数', fontsize=12) plt.tight_layout() plt.savefig('top_correlations.png', dpi=300) plt.close() print("目标相关性条形图已保存到 'top_correlations.png'") # 4. 检测高度相关的特征对 (相关系数>0.9) high_corr_pairs = [] for i in range(len(corr_matrix.columns)): for j in range(i): if abs(corr_matrix.iloc[i, j]) > 0.9: high_corr_pairs.append( (corr_matrix.columns[i], corr_matrix.columns[j], corr_matrix.iloc[i, j]) ) high_corr_df = pd.DataFrame(high_corr_pairs, columns=['Feature1', 'Feature2', 'Correlation']) high_corr_df = high_corr_df.sort_values('Correlation', ascending=False) high_corr_df.to_csv('high_correlation_pairs.csv', index=False) print(f"\n发现{len(high_corr_pairs)}对高度相关的特征对(|相关系数|>0.9)，已保存到'high_correlation_pairs.csv'") if len(high_corr_pairs) > 0: print("\n高度相关的特征对示例:") print(high_corr_df.head(5)) print("\n相关性分析完成，可以基于结果进行特征选择或降维。") # 保存带有全部特征的清洗后数据 df_clean.to_csv('cleaned_feature_data.csv', index=False) print(f"\n清洗后的特征数据已保存到 'cleaned_feature_data.csv'，共{len(df_clean)}行，{len(df_clean.columns)}列") # 生成好的数据送入模型训练 # ------------------------------- # 模型训练与预测，展示验证准确度（RMSE, MAE） # -------------------------------