1 | package tuffy.ra; |
2 | |
3 | import java.sql.PreparedStatement; |
4 | import java.sql.ResultSet; |
5 | import java.sql.SQLException; |
6 | import java.util.ArrayList; |
7 | import java.util.Arrays; |
8 | import java.util.Collection; |
9 | import java.util.Collections; |
10 | import java.util.HashMap; |
11 | import java.util.HashSet; |
12 | import java.util.Set; |
13 | |
14 | |
15 | import tuffy.db.RDB; |
16 | import tuffy.db.SQLMan; |
17 | import tuffy.mln.Clause; |
18 | import tuffy.mln.Literal; |
19 | import tuffy.mln.Predicate; |
20 | import tuffy.mln.Term; |
21 | import tuffy.mln.Type; |
22 | import tuffy.util.Config; |
23 | import tuffy.util.DebugMan; |
24 | import tuffy.util.ExceptionMan; |
25 | import tuffy.util.StringMan; |
26 | import tuffy.util.Timer; |
27 | import tuffy.util.UIMan; |
28 | |
29 | /** |
30 | * A conjunctive query. |
31 | * Used by Datalog and scoping rules. |
32 | */ |
33 | public class ConjunctiveQuery implements Cloneable{ |
34 | |
35 | public String additionalWhereClause = " (1=1) "; |
36 | |
37 | private boolean isScopingRule = false; |
38 | |
39 | public boolean inverseEmbededWeight = false; |
40 | |
41 | public void setScopingRule(boolean isScopingRule) { |
42 | this.isScopingRule = isScopingRule; |
43 | } |
44 | |
45 | public boolean isScopingRule() { |
46 | return isScopingRule; |
47 | } |
48 | |
49 | private Double newTuplePrior = null; |
50 | |
51 | public void setNewTuplePrior(double prior){ |
52 | newTuplePrior = prior; |
53 | } |
54 | |
55 | public Double getNetTuplePrior(){ |
56 | return newTuplePrior; |
57 | } |
58 | |
59 | public boolean isCRFChainRule = false; |
60 | |
61 | public static HashSet<String> indexBuilt = new HashSet<String>(); |
62 | |
63 | private static int idGen = 0; |
64 | public boolean isView = false; |
65 | public boolean isStatic = false; |
66 | public boolean isFictitious = false; |
67 | |
68 | public String allFreeBinding = null; |
69 | |
70 | private static HashMap<Integer, ConjunctiveQuery> objMap = new HashMap<Integer, ConjunctiveQuery>(); |
71 | |
72 | private int id = 0; |
73 | |
74 | public ConjunctiveQuery(){ |
75 | id = (++idGen); |
76 | objMap.put(id, this); |
77 | } |
78 | |
79 | public int getID() { |
80 | return id; |
81 | } |
82 | |
83 | public static ConjunctiveQuery getCqById(int id){ |
84 | return objMap.get(id); |
85 | } |
86 | |
87 | public Clause sourceClause = null; |
88 | |
89 | public ConjunctiveQuery clone(){ |
90 | |
91 | ConjunctiveQuery cloned = new ConjunctiveQuery(); |
92 | try{ |
93 | |
94 | cloned.additionalWhereClause = this.additionalWhereClause; |
95 | cloned.allFreeBinding = this.allFreeBinding; |
96 | cloned.allVariable = (HashSet<String>) this.allVariable.clone(); |
97 | cloned.body = (ArrayList<Literal>) this.body.clone(); |
98 | cloned.constraints = (ArrayList<Expression>) this.constraints.clone(); |
99 | cloned.freeVars = (HashMap<String, Type>) this.freeVars.clone(); |
100 | cloned.head = (Literal) this.head.clone(); |
101 | cloned.inverseEmbededWeight = this.inverseEmbededWeight; |
102 | cloned.isCRFChainRule = this.isCRFChainRule; |
103 | cloned.isFictitious = this.isFictitious; |
104 | cloned.isScopingRule = this.isScopingRule; |
105 | cloned.isStatic = this.isStatic; |
106 | cloned.isView = this.isView; |
107 | cloned.newTuplePrior = this.newTuplePrior; |
108 | cloned.psMap = (HashMap<String, PreparedStatement>) this.psMap.clone(); |
109 | cloned.sourceClause = this.sourceClause; |
110 | cloned.type = this.type; |
111 | cloned.weight = this.weight; |
112 | }catch(Exception e){ |
113 | e.printStackTrace(); |
114 | } |
115 | return cloned; |
116 | } |
117 | |
118 | /** |
119 | * Maps from binding patterns to corresponding prepared statements. Here the binding pattern |
120 | * is a string like "11011", which means the third parameter need to be queried, while other |
121 | * four are provided. |
122 | */ |
123 | public HashMap<String, PreparedStatement> psMap = new HashMap<String, PreparedStatement>(); |
124 | |
125 | public String getAllFreeBinding(){ |
126 | return this.allFreeBinding; |
127 | } |
128 | |
129 | public static void clearIndexHistory(){ |
130 | indexBuilt = new HashSet<String>(); |
131 | } |
132 | |
133 | private double weight = 0; |
134 | |
135 | public void setWeight(double w){ |
136 | weight = w; |
137 | } |
138 | |
139 | public double getWeight(){ |
140 | return weight; |
141 | } |
142 | |
143 | /** |
144 | * Type used by CC. |
145 | */ |
146 | public enum CLUSTERING_RULE_TYPE |
147 | {SOFT_COMPLETE, SOFT_INCOMPLETE, HARD, COULD_LINK_CLIQUE, MUST_LINK_CLIQUE, |
148 | COULD_LINK_PAIRWISE, NODE_LIST, NODE_CLASS, CLASS_TAGS, WOULD_LINK_CLIQUE, SOFT_NEGATIVE, |
149 | HARD_NEGATIVE} |
150 | |
151 | /** |
152 | * Type used by CC. |
153 | */ |
154 | public CLUSTERING_RULE_TYPE type = null; |
155 | |
156 | public Literal head; |
157 | public ArrayList<Literal> body = new ArrayList<Literal>(); |
158 | HashMap<String, Type> freeVars = new HashMap<String, Type>(); |
159 | |
160 | private ArrayList<Expression> constraints = new ArrayList<Expression>(); |
161 | |
162 | /** |
163 | * Add a constraint that must hold. |
164 | * @param e A bool expression that must be TRUE. |
165 | */ |
166 | public void addConstraint(Expression e){ |
167 | constraints.add(e); |
168 | } |
169 | |
170 | public ArrayList<Expression> getConstraint(){ |
171 | return constraints; |
172 | } |
173 | |
174 | public ArrayList<Expression> getConstraint(HashSet<String> allVariables){ |
175 | // return constraints; |
176 | ArrayList<Expression> ret = new ArrayList<Expression>(); |
177 | |
178 | for(Expression e : constraints){ |
179 | int flag = 0; |
180 | for(String v : e.getVars()){ |
181 | if(!allVariables.contains(v)){ |
182 | flag = 1; |
183 | } |
184 | } |
185 | if(flag == 0){ |
186 | ret.add(e); |
187 | } |
188 | } |
189 | |
190 | return ret; |
191 | |
192 | } |
193 | |
194 | public void addConstraintAll(Collection<Expression> es){ |
195 | constraints.addAll(es); |
196 | } |
197 | |
198 | public String toStringInOneLine(){ |
199 | |
200 | String s = ""; |
201 | if(this.sourceClause != null && this.sourceClause.hasEmbeddedWeight()){ |
202 | s = "["+this.sourceClause.getVarWeight()+"] " + head.toString() + |
203 | (isScopingRule ? " :=\t" : " :-\t"); |
204 | }else{ |
205 | s = "["+this.getWeight()+"] " + head.toString() + |
206 | (isScopingRule ? " :=\t" : " :-\t"); |
207 | } |
208 | |
209 | ArrayList<String> a = new ArrayList<String>(); |
210 | for(Literal b : body){ |
211 | a.add(b.toString()); |
212 | } |
213 | for(Expression e : this.constraints){ |
214 | boolean ori = e.changeName; |
215 | e.changeName = true; |
216 | a.add(e.toString()); |
217 | e.changeName = ori; |
218 | } |
219 | s += StringMan.join(", ", a); |
220 | if(newTuplePrior != null){ |
221 | s += "\n### (new tuple prior = " + newTuplePrior + ")"; |
222 | } |
223 | return s; |
224 | |
225 | } |
226 | |
227 | public String toString(){ |
228 | |
229 | String s = ""; |
230 | if(this.sourceClause != null && this.sourceClause.hasEmbeddedWeight()){ |
231 | s = "["+this.sourceClause.getVarWeight()+"] " + head.toString() + |
232 | (isScopingRule ? " :=\n\t" : " :-\n\t"); |
233 | }else{ |
234 | s = "["+this.getWeight()+"] " + head.toString() + |
235 | (isScopingRule ? " :=\n\t" : " :-\n\t"); |
236 | } |
237 | |
238 | ArrayList<String> a = new ArrayList<String>(); |
239 | for(Literal b : body){ |
240 | a.add(b.toString()); |
241 | } |
242 | for(Expression e : this.constraints){ |
243 | boolean ori = e.changeName; |
244 | e.changeName = true; |
245 | a.add(e.toString()); |
246 | e.changeName = ori; |
247 | } |
248 | s += StringMan.join(", \n\t", a); |
249 | if(newTuplePrior != null){ |
250 | s += "\n### (new tuple prior = " + newTuplePrior + ")"; |
251 | } |
252 | return s; |
253 | } |
254 | |
255 | /** |
256 | * Execute this conjunctive query. |
257 | * @param db the DB connection |
258 | * @param truth the truth value for the newly materialized tuples (of the head predicate) |
259 | */ |
260 | public void materialize(RDB db, Boolean truth, ArrayList<String> orderBy){ |
261 | Predicate p = head.getPred(); |
262 | HashMap<String, String> mapVarAttr = new HashMap<String, String>(); |
263 | ArrayList<String> selList = new ArrayList<String>(); |
264 | ArrayList<String> fromList = new ArrayList<String>(); |
265 | ArrayList<String> whereList = new ArrayList<String>(); |
266 | whereList.add("1=1"); |
267 | |
268 | // instantiate dangling variables with the type dict |
269 | for(String v : freeVars.keySet()){ |
270 | Type t = freeVars.get(v); |
271 | String tname = "type" + v; |
272 | fromList.add(t.getRelName() + " " + tname); |
273 | mapVarAttr.put(v, tname + ".constantID"); |
274 | } |
275 | HashMap<String, Type> var2type = new HashMap<String, Type>(); |
276 | ArrayList<String> exceptList = new ArrayList<String>(); |
277 | // variable/constant binding |
278 | for(int i=0; i<body.size();i++) { |
279 | Literal lit = body.get(i); |
280 | int idx = i; |
281 | String relP = lit.getPred().getRelName(); |
282 | if(lit.getSense()){ |
283 | fromList.add(relP + " t" +idx); |
284 | } |
285 | if(!lit.coversAllMaterializedTuples()){ |
286 | if(lit.getSense()){ |
287 | //TODO |
288 | // whereList.add("t" + idx + ".truth = " + |
289 | // (lit.getSense()?"TRUE" : "FALSE")); |
290 | } |
291 | } |
292 | |
293 | ArrayList<Term> terms = lit.getTerms(); |
294 | |
295 | if(lit.getSense()){ |
296 | for(int j=0; j<terms.size(); j++) { |
297 | Term t = terms.get(j); |
298 | String var = t.var(); |
299 | |
300 | var2type.put(var, lit.getPred().getTypeAt(j)); |
301 | |
302 | String attr = "t" + idx + "."+lit.getPred().getArgs().get(j); |
303 | if(t.isConstant()) { |
304 | whereList.add(attr + "=" + t.constant()); |
305 | }else { |
306 | String cattr = mapVarAttr.get(var); |
307 | if(cattr == null) { |
308 | mapVarAttr.put(var, attr); |
309 | }else { |
310 | whereList.add(attr + "=" + cattr); |
311 | } |
312 | } |
313 | } |
314 | }else{ |
315 | ArrayList<String> args = new ArrayList<String>(); |
316 | for(int j=0; j<terms.size(); j++) { |
317 | Term t = terms.get(j); |
318 | if(t.isConstant()){ |
319 | args.add(Integer.toString(t.constant())); |
320 | }else{ |
321 | String var = t.var(); |
322 | var2type.put(var, lit.getPred().getTypeAt(j)); |
323 | String cattr = mapVarAttr.get(var); |
324 | if(cattr == null) { |
325 | // ExceptionMan.die("unsafe rule:\n" + toString()); |
326 | }else { |
327 | args.add(cattr); |
328 | } |
329 | } |
330 | } |
331 | |
332 | exceptList.add("(" + StringMan.commaList(args) + ") NOT IN (SELECT " + |
333 | StringMan.commaList(lit.getPred().getArgs()) + |
334 | " FROM " + lit.getPred().getRelName() + ")"); |
335 | } |
336 | } |
337 | |
338 | /////////////////////////////////// |
339 | whereList.addAll(exceptList); |
340 | |
341 | for(int i=0;i<this.head.getPred().arity();i++){ |
342 | Term t = this.head.getTerms().get(i); |
343 | Type type = this.head.getPred().getTypeAt(i); |
344 | var2type.put(t.toString(), type); |
345 | } |
346 | |
347 | /* |
348 | // variable/constant binding |
349 | for(int i=0; i<body.size();i++) { |
350 | Literal lit = body.get(i); |
351 | int idx = i; |
352 | String relP = lit.getPred().getRelName(); |
353 | fromList.add(relP + " t" +idx); |
354 | if(!lit.coversAllMaterializedTuples()){ |
355 | whereList.add("t" + idx + ".truth = " + |
356 | (lit.getSense()?"TRUE" : "FALSE")); |
357 | } |
358 | ArrayList<Term> terms = lit.getTerms(); |
359 | for(int j=0; j<terms.size(); j++) { |
360 | Term t = terms.get(j); |
361 | String var = t.var(); |
362 | String attr = "t" + idx + "."+lit.getPred().getArgs().get(j); |
363 | if(t.isConstant()) { |
364 | whereList.add(attr + "=" + t.constant()); |
365 | }else { |
366 | String cattr = mapVarAttr.get(var); |
367 | if(cattr == null) { |
368 | mapVarAttr.put(var, attr); |
369 | }else { |
370 | whereList.add(attr + "=" + cattr); |
371 | } |
372 | } |
373 | } |
374 | } |
375 | */ |
376 | |
377 | // express constraints in SQL |
378 | HashSet<String> cvars = new HashSet<String>(); |
379 | int nChangeName = 0; |
380 | for(Expression e : constraints){ |
381 | for(String var : e.getVars()){ |
382 | if(var2type.get(var).isNonSymbolicType()){ |
383 | e.changeName = false; |
384 | } |
385 | } |
386 | if(e.changeName == true){ |
387 | nChangeName ++; |
388 | } |
389 | cvars.addAll(e.getVars()); |
390 | } |
391 | |
392 | HashMap<String, String> mapVarVal = new HashMap<String, String>(); |
393 | HashMap<String, String> mapVarValNotChangeName = new HashMap<String, String>(); |
394 | int idx = 0; |
395 | for(String v : cvars){ |
396 | ++ idx; |
397 | String attr = mapVarAttr.get(v); |
398 | if(attr == null){ |
399 | ExceptionMan.die("unsafe constraints in conjunctive query\n" + toString()); |
400 | } |
401 | if(nChangeName > 0){ |
402 | fromList.add(var2type.get(v).getRelName() + " s" + idx); |
403 | whereList.add("s" + idx + ".constantid = " + attr); |
404 | } |
405 | mapVarVal.put(v, "s" + idx + ".constantvalue"); |
406 | mapVarValNotChangeName.put(v, attr); |
407 | } |
408 | for(Expression e : constraints){ |
409 | if(e.changeName == true){ |
410 | e.bindVariables(mapVarVal); |
411 | whereList.add(e.toSQL()); |
412 | }else{ |
413 | Expression tmpE = e.clone(); |
414 | tmpE.renameVariables(mapVarValNotChangeName); |
415 | whereList.add(tmpE.toString()); |
416 | } |
417 | } |
418 | |
419 | |
420 | // select list |
421 | HashMap<String, String> var2argMap = new HashMap<String, String>(); |
422 | for(int i=0; i<p.arity(); i++){ |
423 | Term t = head.getTerms().get(i); |
424 | String v = null; |
425 | if(t.isConstant()){ |
426 | v = Integer.toString(t.constant()); |
427 | }else{ |
428 | v = mapVarAttr.get(t.var()); |
429 | } |
430 | selList.add(v + " AS " + p.getArgs().get(i)); |
431 | var2argMap.put(t.toString(), p.getArgs().get(i)); |
432 | } |
433 | |
434 | String sql, sub; |
435 | sub = "SELECT " + StringMan.commaList(selList) + |
436 | (fromList.size() > 0 ? " FROM " + StringMan.commaList(fromList) : " ") + |
437 | " WHERE " + SQLMan.andSelCond(whereList) |
438 | + |
439 | "\nEXCEPT\n" + |
440 | " SELECT " + StringMan.commaList(p.getArgs()) + |
441 | " FROM " + p.getRelName() |
442 | ; |
443 | |
444 | |
445 | ArrayList<String> iargs = new ArrayList<String>(); |
446 | //iargs.add(p.nextTupleID()); |
447 | String club = "2"; //evidence |
448 | if(p.hasQuery()) club = "3"; |
449 | if(truth == null || head.coversAllMaterializedTuples() |
450 | || newTuplePrior != null){ |
451 | iargs.add("NULL"); |
452 | iargs.add("0"); |
453 | if(newTuplePrior != null && |
454 | newTuplePrior <= 1 && newTuplePrior >= 0){ |
455 | iargs.add(newTuplePrior.toString()); |
456 | }else{ |
457 | iargs.add("NULL"); |
458 | } |
459 | }else if(truth){ |
460 | iargs.add("TRUE"); |
461 | iargs.add(club); |
462 | iargs.add("NULL"); |
463 | }else{ |
464 | iargs.add("FALSE"); |
465 | iargs.add(club); |
466 | iargs.add("NULL"); |
467 | } |
468 | |
469 | |
470 | for(String a : p.getArgs()){ |
471 | iargs.add("nt." + a); |
472 | } |
473 | |
474 | //p.isInMem = true; |
475 | |
476 | //sql = "INSERT INTO " + p.getRelName() + "(id,truth,club,prior," |
477 | sql = "INSERT INTO " + p.getRelName() + "(truth,club,prior," |
478 | + StringMan.commaList(p.getArgs()) + ")\n"; |
479 | sql += "SELECT " + StringMan.commaList(iargs) + |
480 | " FROM (" + sub + ") nt "; |
481 | |
482 | if(orderBy.size() > 0){ |
483 | ArrayList<String> orders = new ArrayList<String>(); |
484 | for(String o : orderBy){ |
485 | if(var2argMap.containsKey(o)){ |
486 | orders.add(var2argMap.get(o)); |
487 | } |
488 | } |
489 | |
490 | if(orders.size() > 0){ |
491 | sql += " ORDER BY (" + StringMan.commaList(orders) + ")"; |
492 | } |
493 | } |
494 | |
495 | UIMan.verbose(2, sql); |
496 | db.update(sql); |
497 | /* |
498 | String ecost = db.estimateCost(sub); |
499 | // DebugMan.verbose(2, ecost); |
500 | DebugMan.verbose(2, sql); |
501 | DebugMan.verbose(2, "ESTIMATED cost = " + db.estimatedCost + " ; rows = " + db.estimatedRows); |
502 | Timer.start("cqmat"); |
503 | db.update(sql); |
504 | double rtime = Timer.elapsedMilliSeconds("cqmat"); |
505 | DebugMan.verbose(2, Timer.elapsed("cqmat")); |
506 | DebugMan.verbose(2, "COST-RATIO = " + (db.estimatedCost/rtime) + " ; ROW-RATIO = " + |
507 | ((double)db.estimatedRows/db.getLastUpdateRowCount())); |
508 | */ |
509 | } |
510 | |
511 | public String getJoinSQL(HashSet<String> whichToBound){ |
512 | Predicate p = head.getPred(); |
513 | |
514 | HashMap<String, String> mapVarAttr = new HashMap<String, String>(); |
515 | ArrayList<String> selList = new ArrayList<String>(); |
516 | ArrayList<String> fromList = new ArrayList<String>(); |
517 | ArrayList<String> whereList = new ArrayList<String>(); |
518 | ArrayList<String> exceptList = new ArrayList<String>(); |
519 | whereList.add("1=1"); |
520 | |
521 | // instantiate dangling variables with the type dict |
522 | for(String v : freeVars.keySet()){ |
523 | Type t = freeVars.get(v); |
524 | String tname = "type" + v; |
525 | fromList.add(t.getRelName() + " " + tname); |
526 | mapVarAttr.put(v, tname + ".constantID"); |
527 | } |
528 | |
529 | HashMap<String, Type> var2type = new HashMap<String, Type>(); |
530 | // variable/constant binding |
531 | for(int i=0; i<body.size();i++) { |
532 | Literal lit = body.get(i); |
533 | int idx = i; |
534 | String relP = lit.getPred().getRelName(); |
535 | if(lit.getSense()){ |
536 | fromList.add(relP + " t" +idx); |
537 | } |
538 | if(!lit.coversAllMaterializedTuples()){ |
539 | if(lit.getSense()){ |
540 | // whereList.add("t" + idx + ".truth = " + |
541 | // (lit.getSense()?"TRUE" : "FALSE")); |
542 | } |
543 | } |
544 | |
545 | ArrayList<Term> terms = lit.getTerms(); |
546 | |
547 | if(lit.getSense()){ |
548 | for(int j=0; j<terms.size(); j++) { |
549 | Term t = terms.get(j); |
550 | String var = t.var(); |
551 | String attr = "t" + idx + "."+lit.getPred().getArgs().get(j); |
552 | |
553 | var2type.put(var, lit.getPred().getTypeAt(j)); |
554 | |
555 | if(t.isConstant()) { |
556 | whereList.add(attr + "=" + t.constant()); |
557 | }else { |
558 | String cattr = mapVarAttr.get(var); |
559 | if(cattr == null) { |
560 | mapVarAttr.put(var, attr); |
561 | }else { |
562 | |
563 | String a1 = attr; |
564 | a1 = a1.replaceAll("\\..*$", ""); |
565 | if(cattr.contains(a1 + ".")){ |
566 | continue; |
567 | } |
568 | |
569 | whereList.add(attr + "=" + cattr); |
570 | } |
571 | } |
572 | } |
573 | }else{ |
574 | ArrayList<String> args = new ArrayList<String>(); |
575 | for(int j=0; j<terms.size(); j++) { |
576 | Term t = terms.get(j); |
577 | if(t.isConstant()){ |
578 | args.add(Integer.toString(t.constant())); |
579 | }else{ |
580 | String var = t.var(); |
581 | |
582 | var2type.put(var, lit.getPred().getTypeAt(j)); |
583 | |
584 | String cattr = mapVarAttr.get(var); |
585 | if(cattr == null) { |
586 | return null; |
587 | // ExceptionMan.die("write negative predicate in the last\n" + toString()); |
588 | }else { |
589 | args.add(cattr); |
590 | } |
591 | } |
592 | } |
593 | |
594 | exceptList.add("(" + StringMan.commaList(args) + ") NOT IN (SELECT " + |
595 | StringMan.commaList(lit.getPred().getArgs()) + |
596 | " FROM " + lit.getPred().getRelName() + ")"); |
597 | } |
598 | } |
599 | |
600 | /////////////////////////////////// |
601 | whereList.addAll(exceptList); |
602 | |
603 | for(int i=0;i<this.head.getPred().arity();i++){ |
604 | Term t = this.head.getTerms().get(i); |
605 | Type type = this.head.getPred().getTypeAt(i); |
606 | var2type.put(t.toString(), type); |
607 | } |
608 | |
609 | // express constraints in SQL |
610 | HashSet<String> cvars = new HashSet<String>(); |
611 | int nChangeName = 0; |
612 | for(Expression e : constraints){ |
613 | for(String var : e.getVars()){ |
614 | if(var2type.get(var).isNonSymbolicType()){ |
615 | e.changeName = false; |
616 | } |
617 | } |
618 | if(e.changeName == true){ |
619 | nChangeName ++; |
620 | } |
621 | cvars.addAll(e.getVars()); |
622 | } |
623 | |
624 | HashMap<String, String> mapVarVal = new HashMap<String, String>(); |
625 | HashMap<String, String> mapVarValNotChangeName = new HashMap<String, String>(); |
626 | int idx = 0; |
627 | for(String v : cvars){ |
628 | ++ idx; |
629 | String attr = mapVarAttr.get(v); |
630 | if(attr == null){ |
631 | ExceptionMan.die("unsafe constraints in conjunctive query\n" + toString()); |
632 | } |
633 | if(nChangeName > 0){ |
634 | fromList.add(var2type.get(v).getRelName() + " s" + idx); |
635 | whereList.add("s" + idx + ".constantid = " + attr); |
636 | } |
637 | mapVarVal.put(v, "s" + idx + ".constantstring"); |
638 | mapVarValNotChangeName.put(v, attr); |
639 | } |
640 | for(Expression e : constraints){ |
641 | if(e.changeName == true){ |
642 | e.bindVariables(mapVarVal); |
643 | whereList.add(e.toSQL()); |
644 | }else{ |
645 | Expression tmpE = e.clone(); |
646 | tmpE.renameVariables(mapVarValNotChangeName); |
647 | whereList.add(tmpE.toString()); |
648 | } |
649 | } |
650 | |
651 | |
652 | // select list |
653 | for(int i=0; i<p.arity(); i++){ |
654 | Term t = head.getTerms().get(i); |
655 | String v = null; |
656 | if(t.isConstant()){ |
657 | v = Integer.toString(t.constant()); |
658 | }else{ |
659 | v = mapVarAttr.get(t.var()); |
660 | } |
661 | selList.add(v + " AS " + p.getArgs().get(i)); |
662 | |
663 | if( whichToBound.contains(head.getTerms().get(i).toString()) ){ |
664 | // head.getTerms().get(i).toString().equals( whichToBound ) ){ |
665 | whereList.add(v + " = " + "1"); |
666 | } |
667 | |
668 | } |
669 | |
670 | String sub; |
671 | |
672 | String sel = "SELECT "; |
673 | if(this.type == CLUSTERING_RULE_TYPE.COULD_LINK_PAIRWISE |
674 | || this.type == CLUSTERING_RULE_TYPE.COULD_LINK_CLIQUE){ |
675 | sel += "DISTINCT "; |
676 | } |
677 | |
678 | sub = sel + StringMan.commaList(selList) + |
679 | " FROM " + StringMan.join(" JOIN ", fromList) + |
680 | " ON " + SQLMan.andSelCond(whereList) ; |
681 | return sub; |
682 | } |
683 | |
684 | public String getBoundedSQL(HashSet<String> whichToBound){ |
685 | Predicate p = head.getPred(); |
686 | |
687 | HashMap<String, String> mapVarAttr = new HashMap<String, String>(); |
688 | ArrayList<String> selList = new ArrayList<String>(); |
689 | ArrayList<String> fromList = new ArrayList<String>(); |
690 | ArrayList<String> whereList = new ArrayList<String>(); |
691 | ArrayList<String> exceptList = new ArrayList<String>(); |
692 | whereList.add("1=1"); |
693 | |
694 | |
695 | HashMap<String, Type> var2type = new HashMap<String, Type>(); |
696 | // instantiate dangling variables with the type dict |
697 | for(String v : freeVars.keySet()){ |
698 | Type t = freeVars.get(v); |
699 | String tname = "type" + v; |
700 | fromList.add(t.getRelName() + " " + tname); |
701 | mapVarAttr.put(v, tname + ".constantID"); |
702 | } |
703 | |
704 | // variable/constant binding |
705 | for(int i=0; i<body.size();i++) { |
706 | Literal lit = body.get(i); |
707 | int idx = i; |
708 | String relP = lit.getPred().getRelName(); |
709 | if(lit.getSense()){ |
710 | fromList.add(relP + " t" +idx); |
711 | } |
712 | if(!lit.coversAllMaterializedTuples()){ |
713 | if(lit.getSense()){ |
714 | whereList.add("t" + idx + ".truth = " + |
715 | (lit.getSense()?"'1'" : "'0'")); |
716 | } |
717 | } |
718 | |
719 | ArrayList<Term> terms = lit.getTerms(); |
720 | |
721 | if(lit.getSense()){ |
722 | for(int j=0; j<terms.size(); j++) { |
723 | Term t = terms.get(j); |
724 | String var = t.var(); |
725 | var2type.put(var, lit.getPred().getTypeAt(j)); |
726 | String attr = "t" + idx + "."+lit.getPred().getArgs().get(j); |
727 | if(t.isConstant()) { |
728 | whereList.add(attr + "=" + t.constant()); |
729 | }else { |
730 | String cattr = mapVarAttr.get(var); |
731 | if(cattr == null) { |
732 | mapVarAttr.put(var, attr); |
733 | }else { |
734 | whereList.add(attr + "=" + cattr); |
735 | } |
736 | } |
737 | } |
738 | }else{ |
739 | ArrayList<String> args = new ArrayList<String>(); |
740 | for(int j=0; j<terms.size(); j++) { |
741 | Term t = terms.get(j); |
742 | |
743 | if(t.isConstant()){ |
744 | args.add(Integer.toString(t.constant())); |
745 | }else{ |
746 | String var = t.var(); |
747 | |
748 | var2type.put(var, lit.getPred().getTypeAt(j)); |
749 | |
750 | String cattr = mapVarAttr.get(var); |
751 | if(cattr == null) { |
752 | return null; |
753 | // ExceptionMan.die("write negative predicate in the last\n" + toString()); |
754 | }else { |
755 | args.add(cattr); |
756 | } |
757 | } |
758 | } |
759 | |
760 | exceptList.add("(" + StringMan.commaList(args) + ") NOT IN (SELECT " + |
761 | StringMan.commaList(lit.getPred().getArgs()) + |
762 | " FROM " + lit.getPred().getRelName() + ")"); |
763 | } |
764 | } |
765 | |
766 | for(int i=0;i<this.head.getPred().arity();i++){ |
767 | Term t = this.head.getTerms().get(i); |
768 | Type type = this.head.getPred().getTypeAt(i); |
769 | var2type.put(t.toString(), type); |
770 | } |
771 | |
772 | /////////////////////////////////// |
773 | // express constraints in SQL |
774 | HashSet<String> cvars = new HashSet<String>(); |
775 | int nChangeName = 0; |
776 | for(Expression e : constraints){ |
777 | for(String var : e.getVars()){ |
778 | if(var2type.get(var) == null){ |
779 | System.out.println(); |
780 | } |
781 | if(var2type.get(var).isNonSymbolicType()){ |
782 | e.changeName = false; |
783 | } |
784 | } |
785 | if(e.changeName == true){ |
786 | nChangeName ++; |
787 | } |
788 | cvars.addAll(e.getVars()); |
789 | } |
790 | |
791 | HashMap<String, String> mapVarVal = new HashMap<String, String>(); |
792 | HashMap<String, String> mapVarValNotChangeName = new HashMap<String, String>(); |
793 | int idx = 0; |
794 | for(String v : cvars){ |
795 | ++ idx; |
796 | String attr = mapVarAttr.get(v); |
797 | if(attr == null){ |
798 | ExceptionMan.die("unsafe constraints in conjunctive query\n" + toString()); |
799 | } |
800 | if(nChangeName > 0){ |
801 | fromList.add(var2type.get(v).getRelName() + " s" + idx); |
802 | whereList.add("s" + idx + ".constantid = " + attr); |
803 | } |
804 | mapVarVal.put(v, "s" + idx + ".constantvalue"); |
805 | mapVarValNotChangeName.put(v, attr); |
806 | } |
807 | for(Expression e : constraints){ |
808 | if(e.changeName == true){ |
809 | e.bindVariables(mapVarVal); |
810 | whereList.add(e.toSQL()); |
811 | }else{ |
812 | Expression tmpE = e.clone(); |
813 | tmpE.renameVariables(mapVarValNotChangeName); |
814 | whereList.add(tmpE.toString()); |
815 | } |
816 | } |
817 | |
818 | |
819 | // select list |
820 | for(int i=0; i<p.arity(); i++){ |
821 | Term t = head.getTerms().get(i); |
822 | String v = null; |
823 | if(t.isConstant()){ |
824 | v = Integer.toString(t.constant()); |
825 | }else{ |
826 | v = mapVarAttr.get(t.var()); |
827 | } |
828 | selList.add(v + " AS " + p.getArgs().get(i)); |
829 | |
830 | |
831 | if( whichToBound.contains(head.getTerms().get(i).toString()) ){ |
832 | whereList.add(v + " = " + "1"); |
833 | } |
834 | } |
835 | |
836 | String sub; |
837 | |
838 | String sel = "SELECT "; |
839 | if(this.type == CLUSTERING_RULE_TYPE.COULD_LINK_PAIRWISE |
840 | || this.type == CLUSTERING_RULE_TYPE.COULD_LINK_CLIQUE){ |
841 | sel += "DISTINCT "; |
842 | } |
843 | |
844 | sub = sel + StringMan.commaList(selList) + |
845 | (fromList.size() > 0 ? " FROM " + StringMan.commaList(fromList) : " ") + |
846 | " WHERE " + SQLMan.andSelCond(whereList) ; |
847 | return sub; |
848 | } |
849 | |
850 | public HashSet<String> allVariable = new HashSet<String>(); |
851 | |
852 | /** |
853 | * Set the head of this query. |
854 | * @param lit |
855 | */ |
856 | public void setHead(Literal lit){ |
857 | head = lit; |
858 | for(int i=0; i<lit.getPred().arity(); i++){ |
859 | Term t = lit.getTerms().get(i); |
860 | if(t.isVariable() && !allVariable.contains(t.var())) freeVars.put(t.var(), lit.getPred().getTypeAt(i)); |
861 | } |
862 | } |
863 | |
864 | /** |
865 | * Add a body literal. |
866 | * @param lit |
867 | */ |
868 | public void addBodyLit(Literal lit){ |
869 | body.add(lit); |
870 | for(int i=0; i<lit.getPred().arity(); i++){ |
871 | Term t = lit.getTerms().get(i); |
872 | if(lit.getSense()){ |
873 | if(t.isVariable()) freeVars.remove(t.var()); |
874 | allVariable.add(t.var()); |
875 | } |
876 | } |
877 | } |
878 | |
879 | public void buildIndexes(RDB db, Boolean truth, Set<Predicate> IDB, String tableName, |
880 | boolean addM1LessThanM2, ArrayList<String> additionalSel, boolean... forceBuild){ |
881 | |
882 | Predicate p = head.getPred(); |
883 | |
884 | HashMap<String, String> mapVarAttr = new HashMap<String, String>(); |
885 | ArrayList<String> selList = new ArrayList<String>(); |
886 | ArrayList<String> fromList = new ArrayList<String>(); |
887 | ArrayList<String> whereList = new ArrayList<String>(); |
888 | ArrayList<String> exceptList = new ArrayList<String>(); |
889 | whereList.add("1=1"); |
890 | |
891 | HashSet<String> tables = new HashSet<String>(); |
892 | HashSet<String> indices = new HashSet<String>(); |
893 | HashMap<String, String> indicesAssistor = new HashMap<String, String>(); |
894 | |
895 | // instantiate dangling variables with the type dict |
896 | for(String v : freeVars.keySet()){ |
897 | Type t = freeVars.get(v); |
898 | String tname = "type" + v; |
899 | fromList.add(t.getRelName() + " " + tname); |
900 | mapVarAttr.put(v, tname + ".constantID"); |
901 | } |
902 | |
903 | HashMap<String, Type> var2type = new HashMap<String, Type>(); |
904 | |
905 | ArrayList<String> priors = new ArrayList<String>(); |
906 | // variable/constant binding |
907 | for(int i=0; i<body.size();i++) { |
908 | Literal lit = body.get(i); |
909 | int idx = i; |
910 | String relP = lit.getPred().getRelName(); |
911 | tables.add(relP); |
912 | if(lit.getSense()){ |
913 | fromList.add(relP + " t" +idx); |
914 | } |
915 | if(!lit.coversAllMaterializedTuples()){ |
916 | if(lit.getSense()){ |
917 | whereList.add("t" + idx + ".truth <> FALSE "); |
918 | // whereList.add("t" + idx + ".truth = " + |
919 | // (lit.getSense()?"TRUE" : "FALSE")); |
920 | } |
921 | } |
922 | |
923 | if(lit.getPred().hasSoftEvidence()){ |
924 | //priors.add("t"+idx+".prior"); |
925 | priors.add( |
926 | "(CASE WHEN (t" + idx + ".prior) IS NULL THEN 0 " + |
927 | "ELSE (" + |
928 | "(CASE WHEN t" + idx + ".prior >=1 THEN " + Config.hard_weight + |
929 | " WHEN t" + idx + ".prior<=0 THEN -" + Config.hard_weight + |
930 | " ELSE ln(t" + idx + ".prior / (1-t" + idx+ ".prior)) END) " |
931 | + ")::FLOAT END)" |
932 | |
933 | ); |
934 | whereList.add("t" + idx + ".prior > 0.5"); |
935 | } |
936 | |
937 | ArrayList<Term> terms = lit.getTerms(); |
938 | |
939 | if(lit.getSense()){ |
940 | for(int j=0; j<terms.size(); j++) { |
941 | Term t = terms.get(j); |
942 | String var = t.var(); |
943 | |
944 | var2type.put(var, lit.getPred().getTypeAt(j)); |
945 | |
946 | String attr = "t" + idx + "."+lit.getPred().getArgs().get(j); |
947 | if(t.isConstant()) { |
948 | whereList.add(attr + "=" + t.constant()); |
949 | }else { |
950 | String cattr = mapVarAttr.get(var); |
951 | if(cattr == null) { |
952 | if(!lit.getPred().isCurrentlyView) indicesAssistor.put(attr, lit.getPred().getRelName()+"("+lit.getPred().getArgs().get(j)+")"); |
953 | mapVarAttr.put(var, attr); |
954 | }else { |
955 | if(!lit.getPred().isCurrentlyView) indices.add(lit.getPred().getRelName()+"("+lit.getPred().getArgs().get(j)+")"); |
956 | if(indicesAssistor.containsKey(cattr)) indices.add(indicesAssistor.get(cattr)); |
957 | whereList.add(attr + "=" + cattr); |
958 | } |
959 | } |
960 | } |
961 | }else{ |
962 | ArrayList<String> args = new ArrayList<String>(); |
963 | for(int j=0; j<terms.size(); j++) { |
964 | Term t = terms.get(j); |
965 | if(t.isConstant()){ |
966 | args.add(Integer.toString(t.constant())); |
967 | }else{ |
968 | String var = t.var(); |
969 | var2type.put(var, lit.getPred().getTypeAt(j)); |
970 | String cattr = mapVarAttr.get(var); |
971 | if(cattr == null) { |
972 | // ExceptionMan.die("write negative predicate in the last\n" + toString()); |
973 | }else { |
974 | args.add(cattr); |
975 | } |
976 | } |
977 | } |
978 | |
979 | ///////////////////////////////////////////////////TODO:////////////////////////////////////////////////// |
980 | // |
981 | // [10.0] tmp_predicate_1143(mid, eid) :- |
982 | // mcoref(mid, mid1), |
983 | // Mention(mid1, sentID, docID, w1), |
984 | // EntityFeature_WikiTitlePartBefore(eid, w1), |
985 | // MentionFeature_TextLength(mid, len), |
986 | // MentionFeature_TextLength(mid1, len1), |
987 | // !mcoref(mid1, mid2), |
988 | // len <= len1 |
989 | // |
990 | |
991 | exceptList.add("(" + StringMan.commaList(args) + ") NOT IN (SELECT " + |
992 | StringMan.commaList(lit.getPred().getArgs()) + |
993 | " FROM " + lit.getPred().getRelName() + ")"); |
994 | } |
995 | } |
996 | |
997 | /////////////////////////////////// |
998 | whereList.addAll(exceptList); |
999 | |
1000 | for(int i=0;i<this.head.getPred().arity();i++){ |
1001 | Term t = this.head.getTerms().get(i); |
1002 | Type type = this.head.getPred().getTypeAt(i); |
1003 | var2type.put(t.toString(), type); |
1004 | } |
1005 | |
1006 | |
1007 | // express constraints in SQL |
1008 | HashSet<String> cvars = new HashSet<String>(); |
1009 | int nChangeName = 0; |
1010 | for(Expression e : constraints){ |
1011 | for(String var : e.getVars()){ |
1012 | if(var2type.get(var).isNonSymbolicType()){ |
1013 | e.changeName = false; |
1014 | } |
1015 | } |
1016 | if(e.changeName == true){ |
1017 | nChangeName ++; |
1018 | } |
1019 | cvars.addAll(e.getVars()); |
1020 | } |
1021 | |
1022 | HashMap<String, String> mapVarVal = new HashMap<String, String>(); |
1023 | HashMap<String, String> mapVarValNotChangeName = new HashMap<String, String>(); |
1024 | int idx = 0; |
1025 | for(String v : cvars){ |
1026 | ++ idx; |
1027 | String attr = mapVarAttr.get(v); |
1028 | if(attr == null){ |
1029 | ExceptionMan.die("unsafe constraints in conjunctive query\n" + toString()); |
1030 | } |
1031 | if(nChangeName > 0){ |
1032 | fromList.add(var2type.get(v).getRelName() + " s" + idx); |
1033 | whereList.add("s" + idx + ".constantid = " + attr); |
1034 | } |
1035 | mapVarVal.put(v, "s" + idx + ".constantvalue"); |
1036 | mapVarValNotChangeName.put(v, attr); |
1037 | } |
1038 | for(Expression e : constraints){ |
1039 | if(e.changeName == true){ |
1040 | e.bindVariables(mapVarVal); |
1041 | whereList.add(e.toSQL()); |
1042 | }else{ |
1043 | Expression tmpE = e.clone(); |
1044 | tmpE.renameVariables(mapVarValNotChangeName); |
1045 | /////////bug |
1046 | whereList.add(tmpE.toString()); |
1047 | } |
1048 | } |
1049 | |
1050 | // select list |
1051 | for(int i=0; i<p.arity(); i++){ |
1052 | Term t = head.getTerms().get(i); |
1053 | String v = null; |
1054 | if(t.isConstant()){ |
1055 | v = Integer.toString(t.constant()); |
1056 | }else{ |
1057 | v = mapVarAttr.get(t.var()); |
1058 | if(!p.getTypeAt(i).isNonSymbolicType()){ |
1059 | if(indicesAssistor.containsKey(v)) indices.add(indicesAssistor.get(v)); |
1060 | } |
1061 | } |
1062 | selList.add(v + "::" + (p.getTypeAt(i).isNonSymbolicType() ? |
1063 | p.getTypeAt(i).getNonSymbolicTypeInSQL() : "integer") + |
1064 | " AS " + p.getArgs().get(i) ); |
1065 | } |
1066 | |
1067 | |
1068 | ArrayList<String> iargs = new ArrayList<String>(); |
1069 | for(String a : p.getArgs()){ |
1070 | iargs.add("nt." + a); |
1071 | } |
1072 | |
1073 | for(String s : additionalSel){ |
1074 | if(s.equals("weight")){ |
1075 | |
1076 | String expoExp = StringMan.join("+", priors); |
1077 | expoExp = "(2*(exp(" + expoExp + ")/(1+exp(" + expoExp +"))) - 1)"; |
1078 | |
1079 | if(priors.size()==0){ |
1080 | expoExp = "1"; |
1081 | } |
1082 | |
1083 | selList.add((sourceClause.hasEmbeddedWeight()? |
1084 | (this.inverseEmbededWeight == true? "-" : "") + mapVarAttr.get(sourceClause.getVarWeight()) : |
1085 | this.weight) + "*" + expoExp + " AS weight "); |
1086 | iargs.add("nt.weight::float"); |
1087 | }else if(s.equals("prov")){ |
1088 | iargs.add(this.sourceClause.getId() + "::int as prov"); |
1089 | }else if(s.equals("deepprov")){ |
1090 | String deepprov = ""; |
1091 | ArrayList<String> smalls = new ArrayList<String>(); |
1092 | for(String var : mapVarAttr.keySet()){ |
1093 | smalls.add("'" + var + ":' || CAST(" + mapVarAttr.get(var) + " AS TEXT)"); |
1094 | } |
1095 | smalls.add("''"); |
1096 | deepprov = "ARRAY[" + StringMan.commaList(smalls) + "]::TEXT[] AS deepprov"; |
1097 | selList.add(deepprov); |
1098 | iargs.add("nt.deepprov::TEXT[]"); |
1099 | } |
1100 | } |
1101 | |
1102 | String sql, sub; |
1103 | |
1104 | sql = ""; |
1105 | |
1106 | sub = "SELECT " + StringMan.commaList(selList) + |
1107 | (fromList.size() > 0 ? " FROM " + StringMan.commaList(fromList) : " ") + |
1108 | " WHERE " + SQLMan.andSelCond(whereList) ; |
1109 | |
1110 | //build index; |
1111 | if(Config.evidDBSchema == null || (forceBuild.length > 0 && forceBuild[0] == true)){ |
1112 | for(String column : indices){ |
1113 | if(indexBuilt.contains(column)){ |
1114 | continue; |
1115 | } |
1116 | indexBuilt.add(column); |
1117 | |
1118 | String indexName = column.replaceAll("\\(|\\)", "") + Config.getNextGlobalCounter(); |
1119 | String indexsql = "DROP INDEX IF EXISTS inair_" + indexName; |
1120 | db.execute(indexsql); |
1121 | indexsql = "CREATE INDEX inair_" + indexName + " ON " + column; |
1122 | UIMan.verbose(1, indexsql); |
1123 | db.execute(indexsql); |
1124 | } |
1125 | } |
1126 | |
1127 | if(!Config.gp){ |
1128 | for(String table : tables){ |
1129 | db.analyze(table); |
1130 | } |
1131 | } |
1132 | |
1133 | //generates all the binding patterns |
1134 | int intBindingPattern = 0; // 0->variable, 1->constant |
1135 | int nBindings = (int) Math.pow(2, p.arity()); |
1136 | int lBindings = p.arity(); |
1137 | |
1138 | for(int i=0;i<nBindings;i++){ |
1139 | String bindingPattern = Integer.toBinaryString(intBindingPattern); |
1140 | intBindingPattern++; |
1141 | if(bindingPattern.length() < lBindings){ |
1142 | char[] tmp = new char[lBindings - bindingPattern.length()]; |
1143 | Arrays.fill(tmp, '0'); |
1144 | bindingPattern = String.valueOf(tmp) + bindingPattern; |
1145 | } |
1146 | |
1147 | if(!bindingPattern.contains("1")){ |
1148 | this.allFreeBinding = bindingPattern; |
1149 | } |
1150 | |
1151 | ArrayList<String> bindingWhere = new ArrayList<String>(); |
1152 | |
1153 | for(int j=0;j<lBindings;j++){ |
1154 | if(bindingPattern.charAt(j) == '1'){ |
1155 | Term t = head.getTerms().get(j); |
1156 | if(t.isVariable()){ |
1157 | String v = mapVarAttr.get(t.var()); |
1158 | bindingWhere.add("( " + v + " = ? ) "); |
1159 | } |
1160 | } |
1161 | } |
1162 | |
1163 | String newsub = sub; |
1164 | if(bindingWhere.size() > 0){ |
1165 | newsub = sub + " AND " + StringMan.join(" AND ", bindingWhere); |
1166 | } |
1167 | |
1168 | if(tableName == null){ |
1169 | sql = "SELECT " + StringMan.commaList(iargs) + |
1170 | " FROM (" + newsub + ") nt"; |
1171 | }else{ |
1172 | |
1173 | String dropsql = "DROP TABLE IF EXISTS " + tableName; |
1174 | db.update(dropsql); |
1175 | String schemasql = "CREATE TABLE " + tableName + " AS SELECT * FROM " + p.getRelName() + " WHERE 1=2"; |
1176 | db.update(schemasql); |
1177 | |
1178 | sql = "SELECT " + StringMan.commaList(iargs) + |
1179 | " FROM (" + newsub + ") nt"; |
1180 | } |
1181 | |
1182 | //System.err.println(sql); |
1183 | PreparedStatement ps = db.getPrepareStatement(sql); |
1184 | this.psMap.put(bindingPattern, ps); |
1185 | } |
1186 | |
1187 | |
1188 | } |
1189 | |
1190 | public class StringSet{ |
1191 | public String sql; |
1192 | public ArrayList<String> arg = new ArrayList<String>(); |
1193 | public ArrayList<String> as = new ArrayList<String>(); |
1194 | public ArrayList<String> prevas = new ArrayList<String>(); |
1195 | public ArrayList<String> headas = new ArrayList<String>(); |
1196 | public HashMap<String, String> arg2as = new HashMap<String, String>(); |
1197 | public String classAs; |
1198 | public String classAsO; |
1199 | public Predicate headPred; |
1200 | } |
1201 | |
1202 | } |