package com.smtaiserver.smtaiserver.javaai.metrics.base; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; import org.apache.commons.text.similarity.JaccardSimilarity; import org.dom4j.Document; import org.dom4j.Element; import org.dom4j.Node; import com.smtaiserver.smtaiserver.core.SMTAIServerApp; import com.smtaiserver.smtaiserver.core.SMTAIServerRequest; import com.smtaiserver.smtaiserver.database.SMTDatabase.DBRecord; import com.smtaiserver.smtaiserver.database.SMTDatabase.DBRecords; import com.smtaiserver.smtaiserver.javaai.SMTJavaAIError; import com.smtaiserver.smtaiserver.javaai.ast.ASTDBMap; import com.smtaiserver.smtaiserver.javaai.metrics.base.SMTMetricSqlXml.SQLXMLQuery; import com.smtservlet.util.Json; import com.smtservlet.util.SMTJsonWriter; import com.smtservlet.util.SMTStatic; public abstract class SMTMetricsDefXmlDim extends SMTMetricsDefXml { public static class SMTSQLXMLDimDef { public String _simSqlId; public String _filterType; public SMTDimensionDef _dimDef; public SMTSQLXMLDimDef(Element xmlDimName) throws Exception { String dimId = SMTStatic.getXmlAttr(xmlDimName, "id"); _dimDef = SMTAIServerApp.getApp().getDimensionDef(dimId); _simSqlId = SMTStatic.getXmlAttr(xmlDimName, "sim_sql", ""); _filterType = SMTStatic.getXmlAttr(xmlDimName, "filter_type", ""); } } /////////////////////////////////////////////////////////////////////////////////// protected Map _mapId2SqlDimDef = new HashMap<>(); protected Map _mapId2SqlxmlMeticSim = new HashMap<>(); @Override protected void initInstanceByDoc(DBRecord rec, Document doc) throws Exception { // 读取维度 for(Node nodeDimName : doc.selectNodes("ROOT/DIM_NAMES/DIM_NAME")) { SMTSQLXMLDimDef sqlDimDef = new SMTSQLXMLDimDef((Element)nodeDimName); _mapId2SqlDimDef.put(sqlDimDef._dimDef.getId(), sqlDimDef); } // 读取SIM_SQL for(Node nodeSimSQL : doc.selectNodes("ROOT/SIM_SQL")) { String id = SMTStatic.getXmlAttr((Element)nodeSimSQL, "id", ""); SMTMetricSqlXml sqlXml = new SMTMetricSqlXml((Element)nodeSimSQL); _mapId2SqlxmlMeticSim.put(id, sqlXml); } } @Override public Map getDimensionMap() { Map map = new HashMap<>(); for(Entry entry : _mapId2SqlDimDef.entrySet()) { map.put(entry.getKey(), entry.getValue()._dimDef); } return map; } public SMTMetricSqlXml getSimSQLXmlByDimName(String simSqlIdPerfix, String dimName) throws Exception { SMTSQLXMLDimDef dimDef = _mapId2SqlDimDef.get(dimName); SMTMetricSqlXml sqlXml = _mapId2SqlxmlMeticSim.get(simSqlIdPerfix + dimDef._simSqlId); if(sqlXml == null) throw new Exception("cant find sim sql xml : [" + dimDef._simSqlId + "]"); return sqlXml; } protected SMTJavaAIError parseDimListFromJson(Json jsonAST, SMTAIServerRequest tranReq, Map r_map) throws Exception { Json jsonDimList = jsonAST.safeGetJson("dim_list"); if(jsonDimList != null) { for(Json jsonDim : jsonDimList.asJsonList()) { String dimName = jsonDim.asString(); // 从本指标定义的维度中寻找是否存在,如果不存在则直接忽略 SMTSQLXMLDimDef dimDef = _mapId2SqlDimDef.get(dimName); if(dimDef == null) { tranReq.sendChunkedBlock("begin", "未发现要分组的类型 : " + dimName + ",此类型将被忽略"); continue; } r_map.put(dimName, dimDef); } } return null; } protected SMTJavaAIError parseSQLFilterFromJson(ASTDBMap dbMap, Json jsonAST, String simSqlIdPerfix, SMTAIServerRequest tranReq, StringBuilder r_sbDIM_NAME_FILTERS, Map r_mapName2EqDimValue) throws Exception { // 扫描所有的维度,查看是否有过滤条件 JaccardSimilarity jaccardSimilarity = new JaccardSimilarity(); StringBuilder sbSQL = new StringBuilder(); for(SMTSQLXMLDimDef dimDef : _mapId2SqlDimDef.values()) { String dimName = dimDef._dimDef.getId(); Json jsonDimFilter = jsonAST.safeGetJson(dimName); if(jsonDimFilter == null) continue; boolean isEqualMode = jsonAST.safeGetBoolean(dimName + "__eq", false); sbSQL.setLength(0); SMTJavaAIError error = parseSQLFilterFromJsonAndType(dbMap, dimDef, jaccardSimilarity, jsonAST, jsonDimFilter, dimDef._dimDef.getId(), dimDef._dimDef.getType(), simSqlIdPerfix, isEqualMode, tranReq, sbSQL, r_mapName2EqDimValue); if(error != null) return error; if(sbSQL.length() > 0) { r_sbDIM_NAME_FILTERS.append(" AND (" + sbSQL.toString() + ")"); } } return null; } protected SMTJavaAIError parseSQLFilterFromJsonAndType( ASTDBMap dbMap, SMTSQLXMLDimDef dimDef, JaccardSimilarity jaccardSimilarity, Json jsonAST, Json jsonCondList , String fieldName, char valueType, String simSqlIdPerfix, boolean isEqualMode , SMTAIServerRequest tranReq , StringBuilder r_SQL, Map r_mapName2EqDimValue) throws Exception { SMTJavaAIError error = null; Map mapId2SqlArg = new HashMap<>(); if(jsonCondList.isPrimitive()) { String value = jsonCondList.asString(); if(SMTStatic.isNullOrEmpty(value)) { r_SQL.setLength(0); return null; } if(r_mapName2EqDimValue != null) r_mapName2EqDimValue.put(dimDef._dimDef.getId(), value); // 二次查询的等于模式 if(isEqualMode) { if(dimDef._dimDef.hasValueList()) r_SQL.append("(" + fieldName + "::varchar = ANY(string_to_array('" + value.replace("'", "''") + "', ',')))"); else r_SQL.append("(" + fieldName + "::varchar ='" + value.replace("'", "''") + "')"); } // 第一次查询的模糊匹配模式 else { value = dimDef._dimDef.matchDimension(value, jaccardSimilarity); // 获取当前维度对应的sqlxml SMTMetricSqlXml simSqlXml = getSimSQLXmlByDimName(simSqlIdPerfix, fieldName); // 生成SQL SQLXMLQuery queryONAME = new SQLXMLQuery(); mapId2SqlArg.put("__METRICS_ID__", this.getId()); mapId2SqlArg.put("__SIM_FIELD__", fieldName); mapId2SqlArg.put("__SIM_VALUE__", value); if((error = simSqlXml.parseSQL(dbMap, jsonAST, mapId2SqlArg, tranReq, queryONAME)) != null) return error; DBRecords recs = queryONAME._db.querySQL(queryONAME._sbSQLText.toString(), queryONAME._sqlParams.toArray(new Object[queryONAME._sqlParams.size()])); if(recs.getRowCount() == 0) return new SMTJavaAIError("未发现" + dimDef._dimDef.getName() + ":" + value); r_SQL.append("("); for(int recIndex = 0; recIndex < recs.getRowCount(); recIndex ++) { DBRecord rec = recs.getRecord(recIndex); value = rec.getString(fieldName); int idxOpField = recs.getColIndex("__OP__"); String filterOPStr = "$$$FIELD$$$ = $$$VALUE$$$"; if(idxOpField >= 0) { filterOPStr = rec.getString("__OP__"); } if(recIndex > 0) r_SQL.append(" OR "); r_SQL.append(filterOPStr .replace("$$$FIELD$$$", fieldName) .replace("$$$VALUE$$$", "'" + value.replace("'", "''") + "'") ); } r_SQL.append(")"); } return null; } else { if((error = parseSQLFilterTreeFromJsonAndType(dbMap, dimDef, jaccardSimilarity, jsonAST, jsonCondList, fieldName, valueType, simSqlIdPerfix, isEqualMode, tranReq, r_SQL, r_mapName2EqDimValue)) != null) return error; } return null; } protected SMTJavaAIError parseSQLFilterTreeFromJsonAndType(ASTDBMap dbMap, SMTSQLXMLDimDef dimDef, JaccardSimilarity jaccardSimilarity, Json jsonAST, Json jsonCondList, String fieldName, char valueType, String simSqlIdPerfix, boolean isEqualMode, SMTAIServerRequest tranReq , StringBuilder r_SQL, Map r_mapName2EqDimValue) throws Exception { SMTJavaAIError error = null; List listCond = jsonCondList.asJsonList(); if(listCond.size() == 0) return null; Json jsonCond = listCond.get(0); // 解析[第一个表达式] [逻辑符号] [第二个表达式] if(jsonCond.isArray()) { r_SQL.append("("); if((error = parseSQLFilterTreeFromJsonAndType(dbMap, dimDef, jaccardSimilarity, jsonAST, jsonCond, fieldName, valueType, simSqlIdPerfix, isEqualMode, tranReq, r_SQL, r_mapName2EqDimValue)) != null) return error; r_SQL.append(")"); if(listCond.size() > 1) { r_SQL.append(listCond.get(1).asString()); r_SQL.append("("); if((error = parseSQLFilterTreeFromJsonAndType(dbMap, dimDef, jaccardSimilarity, jsonAST, jsonCond, fieldName, valueType, simSqlIdPerfix, isEqualMode, tranReq, r_SQL, r_mapName2EqDimValue)) != null) return error; r_SQL.append(")"); } } // 解析[操作符号] [值] else if(jsonCond.isString()) { // 如果是GIS类型,则特殊处理 if(valueType == 'G') { // GIS类型不在这里处理 } // 如果是其他类型,则特殊处理 else { Map mapId2SqlArg = new HashMap<>(); String value = listCond.get(1).asString(); if(valueType == 'S') { if(r_mapName2EqDimValue != null) r_mapName2EqDimValue.put(fieldName, value); if(isEqualMode) { if(dimDef._dimDef.hasValueList()) r_SQL.append("(" + fieldName + "= ANY(string_to_array('" + value.replace("'", "''") + "', ',')))"); else r_SQL.append("(" + fieldName + "='" + value.replace("'", "''") + "')"); } else { // 生成SQL mapId2SqlArg.put("__METRICS_ID__", this.getId()); mapId2SqlArg.put("__SIM_FIELD__", fieldName); mapId2SqlArg.put("__SIM_VALUE__", value); value = dimDef._dimDef.matchDimension(value, jaccardSimilarity); SMTMetricSqlXml simSqlXml = getSimSQLXmlByDimName(simSqlIdPerfix, fieldName); SQLXMLQuery queryONAME = new SQLXMLQuery(); if((error = simSqlXml.parseSQL(dbMap, jsonAST, mapId2SqlArg, tranReq, queryONAME)) != null) return error; DBRecords recs = queryONAME._db.querySQL(queryONAME._sbSQLText.toString(), queryONAME._sqlParams.toArray(new Object[queryONAME._sqlParams.size()])); if(recs.getRowCount() == 0) return new SMTJavaAIError("未发现:" + value); r_SQL.append("("); for(int recIndex = 0; recIndex < recs.getRowCount(); recIndex ++) { DBRecord rec = recs.getRecord(recIndex); value = rec.getString(0); int idxOpField = recs.getColIndex("__OP__"); String filterOPStr = "$$$FIELD$$$ = $$$VALUE$$$"; if(idxOpField >= 0) { filterOPStr = rec.getString("__OP__"); } if(recIndex > 0) r_SQL.append(" OR "); r_SQL.append(filterOPStr .replace("$$$FIELD$$$", fieldName) .replace("$$$VALUE$$$", "'" + value.replace("'", "''") + "'") ); } r_SQL.append(")"); } } else { r_SQL.append(fieldName); r_SQL.append(" " + jsonCond.asString() + " "); if(valueType == 'T') { r_SQL.append("'" + SMTStatic.toString(SMTStatic.toDate(value)) + "'"); } else { r_SQL.append(SMTStatic.toString(SMTStatic.toDouble(value))); } } } } else throw new Exception("jsonCond is not array or value:" + jsonCond.toString()); return null; } protected int getLastDimPosByDimList(Json jsonAST, List listGroupNameInfo) { int lastPos = -1; Json jsonDimList = jsonAST.safeGetJson("dim_list"); if(jsonDimList == null) return -1; for(Json jsonDim : jsonDimList.asJsonList()) { SMTSQLXMLDimDef dimDef = _mapId2SqlDimDef.get(jsonDim.asString()); if(dimDef == null) continue; String dimName = dimDef._dimDef.getId(); for(int i = 0; i < listGroupNameInfo.size(); i ++) { SMTDuckTimeGroupName groupNameInfo = listGroupNameInfo.get(i); if(groupNameInfo._idCol.equalsIgnoreCase(dimName)) { if(lastPos < i) lastPos = i; } } } return lastPos; } protected SMTJavaAIError promptUserForTimeRange(String title, SMTAIServerRequest tranReq, String[] r_timeRange) throws Exception { String replyId = SMTStatic.newUUID(); SMTJsonWriter jsonWr = new SMTJsonWriter(false); jsonWr.addKeyValue("title", title); jsonWr.addKeyValue("type", "time_range"); jsonWr.addKeyValue("reply_id", replyId); Json jsonReply = tranReq.sendReplyChunkedBlock(replyId, jsonWr.getRootJson()); if(jsonReply == null) return new SMTJavaAIError("流程被用户中断"); r_timeRange[0] = jsonReply.getJson("start_time").asString(); r_timeRange[1] = jsonReply.getJson("end_time").asString(); return null; } }