package com.smtaiserver.smtaiserver.javaai.metrics.base; import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Map.Entry; import org.apache.commons.text.similarity.JaccardSimilarity; import org.dom4j.Element; import org.dom4j.Node; import org.dom4j.tree.DefaultText; import com.smtaiserver.smtaiserver.core.SMTAIServerApp; import com.smtaiserver.smtaiserver.core.SMTAIServerRequest; import com.smtaiserver.smtaiserver.database.SMTDatabase; import com.smtaiserver.smtaiserver.database.SMTDatabase.DBQueryNotify; import com.smtaiserver.smtaiserver.database.SMTDatabase.DBRecord; import com.smtaiserver.smtaiserver.database.SMTDatabase.DBRecords; import com.smtaiserver.smtaiserver.javaai.SMTJavaAIError; import com.smtaiserver.smtaiserver.javaai.ast.ASTDBMap; import com.smtservlet.util.Json; import com.smtservlet.util.SMTJsonWriter; import com.smtservlet.util.SMTStatic; public class SMTMetricSqlXml { /////////////////////////////////////////////////////////////////////////////////////// private static class SQLXMLSimRec { public double _ratio; public Map _mapParam2Value = new HashMap<>(); } /////////////////////////////////////////////////////////////////////////////////////// public static class SQLXMLQuery { public StringBuilder _sbSQLText = new StringBuilder(); public List _sqlParams = new ArrayList<>(); public DBRecords _recsResult = null; public SMTDatabase _db; public String getSqlLog() { StringBuilder sbLogSQL = new StringBuilder(); sbLogSQL.append(_sbSQLText.toString() + "\n"); for(Object param : _sqlParams) { sbLogSQL.append(" param : " + SMTStatic.toString(param) + "\n"); } return sbLogSQL.toString(); } } /////////////////////////////////////////////////////////////////////////////////////// private static class SQLXMLExecArg { public Json _jsonSqlArg = null; public Map _mapSqlArg; public SQLXMLQuery _query; public SMTAIServerRequest _tranReq; public SQLXMLExecArg(Json jsonSqlArg, Map mapSqlArg, SMTAIServerRequest tranReq, SQLXMLQuery r_query) { _mapSqlArg = mapSqlArg; _jsonSqlArg = jsonSqlArg; _query = r_query; _tranReq = tranReq; } public String getSqlArg(String key) { String value = null; if(_mapSqlArg != null) { value = (String)_mapSqlArg.get(key); if(value != null) return value; } value = _jsonSqlArg.safeGetStr(key, null); if(value == null) return null; return value; } } /////////////////////////////////////////////////////////////////////////////////////// private static abstract class SQLXMLNode { public abstract SMTJavaAIError execute(SQLXMLExecArg execArg) throws Exception; } /////////////////////////////////////////////////////////////////////////////////////// private static class SQLXMLNodeQUERY_SIM_TO_PARAM extends SQLXMLNode { private String _compKey; private String _compField; private String[] _paramFields; private List _listChildren = new ArrayList<>(); public SQLXMLNodeQUERY_SIM_TO_PARAM(Element xmlRoot) throws Exception { _compKey = SMTStatic.getXmlAttr(xmlRoot, "comp_key"); _compField = SMTStatic.getXmlAttr(xmlRoot, "comp_field"); _paramFields = SMTStatic.getXmlAttr(xmlRoot, "param_fields").split(","); for (Iterator iterInner = xmlRoot.nodeIterator(); iterInner.hasNext();) { Node nodeInner = iterInner.next(); if(nodeInner.getNodeType() == Node.TEXT_NODE) { String text = ((DefaultText)nodeInner).getText(); int lastPos = _listChildren.size() - 1; if(lastPos >= 0 && _listChildren.get(lastPos) instanceof String) { _listChildren.set(lastPos, (String)_listChildren.get(lastPos) + text); } else { _listChildren.add(text); } } else { _listChildren.add(createSQLXMLNode((Element)nodeInner)); } } } private SMTJavaAIError querySimParamValue(SQLXMLExecArg execArg, JaccardSimilarity jaccardSimilarity, String compValue, double minSimValue, SQLXMLSimRec[] lastSimRec, List r_subList) throws Exception { execArg._query._db.querySQLNotify( execArg._query._sbSQLText.toString(), execArg._query._sqlParams.toArray(new Object[execArg._query._sqlParams.size()]), new DBQueryNotify() { @Override public boolean onNextRecord(DBRecord rec) throws Exception { String value = rec.getString(_compField); double curSim = jaccardSimilarity.apply(value, compValue); // 如果匹配不合格,则记录成不合格 if(curSim < minSimValue) { if(value.indexOf(compValue) >= 0) { if(r_subList.size() < 5) r_subList.add(value); } return true; } SQLXMLSimRec simRec; // 如果完全匹配则直接退出 boolean ret; if(curSim == 1) { simRec = new SQLXMLSimRec(); lastSimRec[0] = simRec; ret = false; } else { // 获取存储的匹配值 simRec = lastSimRec[0]; // 如果未存储,则创建新的 if(simRec == null) { simRec = new SQLXMLSimRec(); lastSimRec[0] = simRec; } // 如果当前小于存储忽略 else if(curSim < simRec._ratio) { return true; } ret = true; } // 保存当前匹配值 simRec._ratio = curSim; // 保存参数值 for(String paramField : _paramFields) { simRec._mapParam2Value.put(paramField, rec.getString(paramField)); } return ret; } } ); return null; } @Override public SMTJavaAIError execute(SQLXMLExecArg execArg) throws Exception { StringBuilder orgSQLText = execArg._query._sbSQLText; List orgSqlParams = execArg._query._sqlParams; DBRecords orgRsResult = execArg._query._recsResult; try { execArg._query._sbSQLText = new StringBuilder(); execArg._query._sqlParams = new ArrayList<>(); // 生成SQL for(Object oxmlNode : _listChildren) { if(oxmlNode instanceof String) { execArg._query._sbSQLText.append(oxmlNode); } else if(oxmlNode instanceof SQLXMLNode) { SMTJavaAIError error = ((SQLXMLNode)oxmlNode).execute(execArg); if(error != null) return error; } } // 获取要对比的值 String compValue = execArg.getSqlArg(_compKey); JaccardSimilarity jaccardSimilarity = new JaccardSimilarity(); double minSimValue = SMTStatic.toDouble(SMTAIServerApp.getApp().getGlobalConfig("history.math.ratio")); SQLXMLSimRec[] lastSimRec; while(true) { lastSimRec = new SQLXMLSimRec[1]; // 0: 合格匹配,1:不合格匹配 List listSubList = new ArrayList<>(); SMTJavaAIError error = querySimParamValue(execArg, jaccardSimilarity, compValue, minSimValue, lastSimRec, listSubList); if(error != null) return error; // 如果当前对象未匹配 if(lastSimRec[0] == null) { // 上前端询问 String replyId = SMTStatic.newUUID(); SMTJsonWriter jsonWr = new SMTJsonWriter(false); jsonWr.addKeyValue("title", "未发现:" + compValue + ", 请输入其他名称"); jsonWr.addKeyValue("type", "input-select"); jsonWr.addKeyValue("reply_id", replyId); jsonWr.beginArray("options"); { for(String item : listSubList) { jsonWr.addKeyValue(null, item); } } jsonWr.endArray(); // 等待应答 Json jsonReply = execArg._tranReq.sendReplyChunkedBlock(replyId, jsonWr.getRootJson()); if(jsonReply == null) return new SMTJavaAIError("流程被用户中断"); // 将用户输入对象设置为当前对象 compValue = jsonReply.getJson("select").asString(); // 在配置中替换当前对象 String key = (String)execArg._mapSqlArg.get("__SIM_FIELD__"); if(key != null) execArg._jsonSqlArg.set(key, compValue); } else { break; } } // 将匹配到数据保存到sql参数中 for(Entry entry : lastSimRec[0]._mapParam2Value.entrySet()) { execArg._mapSqlArg.put(entry.getKey(), entry.getValue()); } } finally { execArg._query._sbSQLText = orgSQLText; execArg._query._sqlParams = orgSqlParams; execArg._query._recsResult = orgRsResult; } return null; } } /////////////////////////////////////////////////////////////////////////////////////// private static class SQLXMLNodeSQL extends SQLXMLNode { private List _listChildren = new ArrayList<>(); public SQLXMLNodeSQL(Element xmlRoot) throws Exception { for (Iterator iterInner = xmlRoot.nodeIterator(); iterInner.hasNext();) { Node nodeInner = iterInner.next(); if(nodeInner.getNodeType() == Node.TEXT_NODE) { String text = ((DefaultText)nodeInner).getText(); int lastPos = _listChildren.size() - 1; if(lastPos >= 0 && _listChildren.get(lastPos) instanceof String) { _listChildren.set(lastPos, (String)_listChildren.get(lastPos) + text); } else { _listChildren.add(text); } } else { _listChildren.add(createSQLXMLNode((Element)nodeInner)); } } } @Override public SMTJavaAIError execute(SQLXMLExecArg execArg) throws Exception { execArg._query._sbSQLText = new StringBuilder(); execArg._query._sqlParams = new ArrayList<>(); for(Object oxmlNode : _listChildren) { if(oxmlNode instanceof String) { execArg._query._sbSQLText.append(oxmlNode); } else if(oxmlNode instanceof SQLXMLNode) { SMTJavaAIError error = ((SQLXMLNode)oxmlNode).execute(execArg); if(error != null) return error; } } return null; } } /////////////////////////////////////////////////////////////////////////////////////// private static class SQLXMLNodePARAM extends SQLXMLNode { private String _key; public SQLXMLNodePARAM(Element xmlRoot) throws Exception { _key = SMTStatic.getXmlAttr(xmlRoot, "key"); } @Override public SMTJavaAIError execute(SQLXMLExecArg execArg) throws Exception { String value = execArg.getSqlArg(_key); if(value == null) return new SMTJavaAIError("请输入参数:" + _key); execArg._query._sbSQLText.append("?"); execArg._query._sqlParams.add(value); return null; } } /////////////////////////////////////////////////////////////////////////////////////// private static class SQLXMLNodeTEXT extends SQLXMLNode { private String _key; public SQLXMLNodeTEXT(Element xmlRoot) throws Exception { _key = SMTStatic.getXmlAttr(xmlRoot, "key"); } @Override public SMTJavaAIError execute(SQLXMLExecArg execArg) throws Exception { String value = execArg.getSqlArg(_key); if(value == null) return new SMTJavaAIError("请输入参数:" + _key); execArg._query._sbSQLText.append(value); return null; } } /////////////////////////////////////////////////////////////////////////////////////// private static class SQLXMLNodeMETRICS_ID extends SQLXMLNode { public SQLXMLNodeMETRICS_ID(Element xmlRoot) throws Exception { } @Override public SMTJavaAIError execute(SQLXMLExecArg execArg) throws Exception { String value = execArg.getSqlArg("__METRICS_ID__"); execArg._query._sbSQLText.append("'" + value.replace("'", "''" + "'") + "'"); return null; } } /////////////////////////////////////////////////////////////////////////////////////// private static class SQLXMLNodeDIM_NAME_FILTERS extends SQLXMLNode { public SQLXMLNodeDIM_NAME_FILTERS(Element xmlRoot) throws Exception { } @Override public SMTJavaAIError execute(SQLXMLExecArg execArg) throws Exception { String value = execArg.getSqlArg("__DIM_NAME_FILTERS__"); execArg._query._sbSQLText.append(value); return null; } } /////////////////////////////////////////////////////////////////////////////////////// private static class SQLXMLNodeDIM_NAME_MATCH extends SQLXMLNode { public SQLXMLNodeDIM_NAME_MATCH(Element xmlRoot) throws Exception { } @Override public SMTJavaAIError execute(SQLXMLExecArg execArg) throws Exception { String value = execArg.getSqlArg("__DIM_NAME_MATCH__"); execArg._query._sbSQLText.append(value); return null; } } /////////////////////////////////////////////////////////////////////////////////////// private static class SQLXMLNodeDIM_NAME_GROUP extends SQLXMLNode { public SQLXMLNodeDIM_NAME_GROUP(Element xmlRoot) throws Exception { } @Override public SMTJavaAIError execute(SQLXMLExecArg execArg) throws Exception { String value = execArg.getSqlArg("__DIM_NAME_GROUP__"); execArg._query._sbSQLText.append(value); return null; } } /////////////////////////////////////////////////////////////////////////////////////// private static class SQLXMLNodeSIM_FIELD extends SQLXMLNode { public SQLXMLNodeSIM_FIELD(Element xmlRoot) throws Exception { } @Override public SMTJavaAIError execute(SQLXMLExecArg execArg) throws Exception { String value = execArg.getSqlArg("__SIM_FIELD__"); execArg._query._sbSQLText.append(value); return null; } } /////////////////////////////////////////////////////////////////////////////////////// private static class SQLXMLNodeSIM_VALUE extends SQLXMLNode { public SQLXMLNodeSIM_VALUE(Element xmlRoot) throws Exception { } @Override public SMTJavaAIError execute(SQLXMLExecArg execArg) throws Exception { String value = execArg.getSqlArg("__SIM_VALUE__"); execArg._query._sbSQLText.append("'" + value.replace("'", "''") + "'"); return null; } } /////////////////////////////////////////////////////////////////////////////////////// private static class SQLXMLNodeSIM_MIN_VALUE extends SQLXMLNode { public SQLXMLNodeSIM_MIN_VALUE(Element xmlRoot) throws Exception { } @Override public SMTJavaAIError execute(SQLXMLExecArg execArg) throws Exception { String minSimValue = (String) SMTAIServerApp.getApp().getGlobalConfig("history.math.ratio"); execArg._query._sbSQLText.append(minSimValue); return null; } } /////////////////////////////////////////////////////////////////////////////////////// private static class SQLXMLNodeVALUE_FILTER extends SQLXMLNode { public SQLXMLNodeVALUE_FILTER(Element xmlRoot) throws Exception { } @Override public SMTJavaAIError execute(SQLXMLExecArg execArg) throws Exception { String sqlValueFilter = (String)execArg._mapSqlArg.get("__VALUE_FILTER__"); if(!SMTStatic.isNullOrEmpty(sqlValueFilter)) execArg._query._sbSQLText.append(" AND (" + sqlValueFilter + ")"); return null; } } /////////////////////////////////////////////////////////////////////////////////////// private static class SQLXMLNodeAGG_DIM_COL_NAMES extends SQLXMLNode { private String _titleColName; public SQLXMLNodeAGG_DIM_COL_NAMES(Element xmlRoot) throws Exception { _titleColName = SMTStatic.getXmlAttr(xmlRoot, "title_field"); } @SuppressWarnings("unchecked") @Override public SMTJavaAIError execute(SQLXMLExecArg execArg) throws Exception { List listAggGroupName = (List)execArg._mapSqlArg.get("__AGG_DIM_GROUP_NAMES__"); int lastGroupPos = (int)execArg._mapSqlArg.get("__AGG_DIM_LAST_POS__"); if(lastGroupPos < 0) { execArg._query._sbSQLText.append("'全部' as " + _titleColName); } else { for(int i = 0; i <= lastGroupPos; i ++) { SMTDuckTimeGroupName groupNameInfo = listAggGroupName.get(i); execArg._query._sbSQLText.append(groupNameInfo._idCol + ","); } execArg._query._sbSQLText.append(listAggGroupName.get(lastGroupPos)._titleCol + " as " + _titleColName); } return null; } } /////////////////////////////////////////////////////////////////////////////////////// private static class SQLXMLNodeAGG_DIM_COL_TITLES extends SQLXMLNode { public SQLXMLNodeAGG_DIM_COL_TITLES(Element xmlRoot) throws Exception { } @SuppressWarnings("unchecked") @Override public SMTJavaAIError execute(SQLXMLExecArg execArg) throws Exception { List listAggGroupName = (List)execArg._mapSqlArg.get("__AGG_DIM_GROUP_NAMES__"); int lastGroupPos = (int)execArg._mapSqlArg.get("__AGG_DIM_LAST_POS__"); if(lastGroupPos < 0) { lastGroupPos = listAggGroupName.size(); } for(int i = 0; i <= lastGroupPos; i ++) { if(i > 0) execArg._query._sbSQLText.append(","); SMTDuckTimeGroupName groupNameInfo = listAggGroupName.get(i); execArg._query._sbSQLText.append(groupNameInfo._titleCol); } return null; } } /////////////////////////////////////////////////////////////////////////////////////// private static class SQLXMLNodDIM_NAME_ORDER extends SQLXMLNode { public SQLXMLNodDIM_NAME_ORDER(Element xmlRoot) throws Exception { } @SuppressWarnings("unchecked") @Override public SMTJavaAIError execute(SQLXMLExecArg execArg) throws Exception { List listAggGroupName = (List)execArg._mapSqlArg.get("__AGG_DIM_GROUP_NAMES__"); int lastGroupPos = (int)execArg._mapSqlArg.get("__AGG_DIM_LAST_POS__"); if(lastGroupPos < 0) { execArg._query._sbSQLText.append("1"); } for(int i = 0; i <= lastGroupPos; i ++) { if(i > 0) execArg._query._sbSQLText.append(","); SMTDuckTimeGroupName groupNameInfo = listAggGroupName.get(i); execArg._query._sbSQLText.append(groupNameInfo._titleCol); } return null; } } /////////////////////////////////////////////////////////////////////////////////////// private static class SQLXMLNodeAGG_DIM_FILTER_VALUE_SQL extends SQLXMLNode { public SQLXMLNodeAGG_DIM_FILTER_VALUE_SQL(Element xmlRoot) throws Exception { } @SuppressWarnings("unchecked") @Override public SMTJavaAIError execute(SQLXMLExecArg execArg) throws Exception { List listAggGroupName = (List)execArg._mapSqlArg.get("__AGG_DIM_GROUP_NAMES__"); int lastGroupPos = (int)execArg._mapSqlArg.get("__AGG_DIM_LAST_POS__"); if(lastGroupPos < 0) return null; execArg._query._sbSQLText.append(" AND ("); for(int i = 0; i <= lastGroupPos; i ++) { SMTDuckTimeGroupName groupNameInfo = listAggGroupName.get(i); String value = (String) execArg._mapSqlArg.get(groupNameInfo._idCol.toUpperCase()); if(i > 0) execArg._query._sbSQLText.append(" AND "); if(value == null) execArg._query._sbSQLText.append(groupNameInfo._idCol + " IS NULL"); else execArg._query._sbSQLText.append(groupNameInfo._idCol + " = '" + value.replace("'", "''") + "'"); } execArg._query._sbSQLText.append(")"); return null; } } /////////////////////////////////////////////////////////////////////////////////////// private String _dsId; private SQLXMLNode _SQLXMLNode; private static SQLXMLNode createSQLXMLNode(Element xmlRoot) throws Exception { String name = xmlRoot.getName().toUpperCase(); if("SQL".equals(name) || "NAME_SQL".equals(name)|| "VALUE_SQL".equals(name)|| "SIM_SQL".equals(name) || "QUOTA_SQL".equals(name) ) return new SQLXMLNodeSQL(xmlRoot); else if("QUERY_SIM_TO_PARAM".equals(name)) return new SQLXMLNodeQUERY_SIM_TO_PARAM(xmlRoot); else if("PARAM".equals(name)) return new SQLXMLNodePARAM(xmlRoot); else if("TEXT".equals(name)) return new SQLXMLNodeTEXT(xmlRoot); else if("METRICS_ID".equals(name)) return new SQLXMLNodeMETRICS_ID(xmlRoot); else if("DIM_NAME_FILTERS".equals(name)) return new SQLXMLNodeDIM_NAME_FILTERS(xmlRoot); else if("DIM_NAME_MATCH".equals(name)) return new SQLXMLNodeDIM_NAME_MATCH(xmlRoot); else if("DIM_NAME_GROUP".equals(name)) return new SQLXMLNodeDIM_NAME_GROUP(xmlRoot); else if("SIM_FIELD".equals(name)) return new SQLXMLNodeSIM_FIELD(xmlRoot); else if("SIM_VALUE".equals(name)) return new SQLXMLNodeSIM_VALUE(xmlRoot); else if("SIM_MIN_VALUE".equals(name)) return new SQLXMLNodeSIM_MIN_VALUE(xmlRoot); else if("VALUE_FILTER".equals(name)) return new SQLXMLNodeVALUE_FILTER(xmlRoot); else if("AGG_DIM_COL_NAMES".equals(name)) return new SQLXMLNodeAGG_DIM_COL_NAMES(xmlRoot); else if("AGG_DIM_FILTER_VALUE_SQL".equals(name)) return new SQLXMLNodeAGG_DIM_FILTER_VALUE_SQL(xmlRoot); else if("AGG_DIM_COL_TITLES".equals(name)) return new SQLXMLNodeAGG_DIM_COL_TITLES(xmlRoot); else if("DIM_NAME_ORDER".equals(name)) return new SQLXMLNodDIM_NAME_ORDER(xmlRoot); else throw new Exception("unknow SQLXML : " + name); } public SMTMetricSqlXml(Element rootElement) throws Exception { _dsId = SMTStatic.getXmlAttr(rootElement, "ds_id", null); _SQLXMLNode = createSQLXMLNode(rootElement); } public SMTJavaAIError parseSQL(ASTDBMap dbMap, Json jsonArgs, Map mapSqlArg, SMTAIServerRequest tranReq, SQLXMLQuery r_query) throws Exception { r_query._db = dbMap.getDatabase(_dsId); SMTJavaAIError error = null; SQLXMLExecArg execArg = new SQLXMLExecArg(jsonArgs, mapSqlArg, tranReq, r_query); if((error = _SQLXMLNode.execute(execArg)) != null) return error; return null; } }