| 1 | package tuffy.mln; |
| 2 | |
| 3 | import java.io.BufferedReader; |
| 4 | import java.io.File; |
| 5 | import java.io.InputStream; |
| 6 | import java.util.ArrayList; |
| 7 | import java.util.Collections; |
| 8 | import java.util.HashMap; |
| 9 | import java.util.HashSet; |
| 10 | import java.util.Hashtable; |
| 11 | import java.util.zip.GZIPInputStream; |
| 12 | |
| 13 | import tuffy.db.RDB; |
| 14 | import tuffy.parse.InputParser; |
| 15 | import tuffy.ra.ConjunctiveQuery; |
| 16 | import tuffy.ra.Function; |
| 17 | import tuffy.util.Config; |
| 18 | import tuffy.util.DebugMan; |
| 19 | import tuffy.util.ExceptionMan; |
| 20 | import tuffy.util.FileMan; |
| 21 | import tuffy.util.Timer; |
| 22 | import tuffy.util.UIMan; |
| 23 | |
| 24 | /** |
| 25 | * An MLN. Holds the symbol table. |
| 26 | */ |
| 27 | public class MarkovLogicNetwork implements Cloneable{ |
| 28 | private static int idGen = 0; |
| 29 | private int id = 0; |
| 30 | |
| 31 | /** |
| 32 | * The db connection associated with this MLN. |
| 33 | */ |
| 34 | private RDB db = null; |
| 35 | |
| 36 | /** |
| 37 | * Database tables storing intermediate data. |
| 38 | */ |
| 39 | public String relClauses = "clauses"; |
| 40 | public String relAtoms = "atoms"; |
| 41 | public String relTrueAtoms = "true_atoms"; |
| 42 | public String relClausePart = "clause_part"; |
| 43 | public String relAtomPart = "atom_part"; |
| 44 | |
| 45 | /** |
| 46 | * Parser of input. |
| 47 | */ |
| 48 | private InputParser parser; |
| 49 | |
| 50 | public RDB getDB(){ |
| 51 | return db; |
| 52 | } |
| 53 | |
| 54 | /** |
| 55 | * List of all predicates appearing in this MLN. |
| 56 | */ |
| 57 | private ArrayList<Predicate> listPred = new ArrayList<Predicate>(); |
| 58 | |
| 59 | /** |
| 60 | * Map from string name to Predicate object. |
| 61 | */ |
| 62 | private Hashtable<String, Predicate> nameMapPred = |
| 63 | new Hashtable<String, Predicate>(); |
| 64 | |
| 65 | private Hashtable<String, Function> nameMapFunc = |
| 66 | new Hashtable<String, Function>(); |
| 67 | |
| 68 | /** |
| 69 | * Map from string name to Type object. |
| 70 | */ |
| 71 | private Hashtable<String, Type> nameMapType = |
| 72 | new Hashtable<String, Type>(); |
| 73 | |
| 74 | /** |
| 75 | * List of clauses marked as relevant. |
| 76 | */ |
| 77 | private HashSet<Clause> relevantClauses = new HashSet<Clause>(); |
| 78 | |
| 79 | private HashMap<Clause, Clause> unnormal2normal = new HashMap<Clause, Clause>(); |
| 80 | |
| 81 | /** |
| 82 | * List of normalized clauses. |
| 83 | */ |
| 84 | private ArrayList<Clause> listClauses = new ArrayList<Clause>(); |
| 85 | |
| 86 | /** |
| 87 | * List of unnormalized clauses. |
| 88 | */ |
| 89 | public ArrayList<Clause> unnormalizedClauses = new ArrayList<Clause>(); |
| 90 | |
| 91 | /** |
| 92 | * Map from signature of clauses to Clause object. |
| 93 | * For the definition of ``signature'', see {@link Clause#normalize()}. |
| 94 | */ |
| 95 | private Hashtable<String, Clause> sigMap = new Hashtable<String, Clause>(); |
| 96 | |
| 97 | /** |
| 98 | * Map from string name to integer constant ID. |
| 99 | */ |
| 100 | private HashMap<String, Integer> mapConstantID = new HashMap<String, Integer>(); |
| 101 | |
| 102 | /** |
| 103 | * |
| 104 | */ |
| 105 | private HashMap<Predicate, ArrayList<ConjunctiveQuery>> scopes = |
| 106 | new HashMap<Predicate, ArrayList<ConjunctiveQuery>>(); |
| 107 | |
| 108 | private ArrayList<ConjunctiveQuery> scopingRules = new ArrayList<ConjunctiveQuery>(); |
| 109 | |
| 110 | private ArrayList<Predicate> clusteringPredicates = new ArrayList<Predicate>(); |
| 111 | |
| 112 | public HashSet<ConjunctiveQuery> dedupalogRules = new HashSet<ConjunctiveQuery>(); |
| 113 | |
| 114 | |
| 115 | @SuppressWarnings("unchecked") |
| 116 | public Object clone() throws CloneNotSupportedException { |
| 117 | |
| 118 | MarkovLogicNetwork clone=(MarkovLogicNetwork) super.clone(); |
| 119 | |
| 120 | clone.db = db; |
| 121 | |
| 122 | clone.parser = parser; |
| 123 | |
| 124 | clone.listPred = (ArrayList<Predicate>) listPred.clone(); |
| 125 | |
| 126 | clone.nameMapPred = (Hashtable<String, Predicate>) nameMapPred.clone(); |
| 127 | |
| 128 | clone.nameMapFunc = (Hashtable<String, Function>) nameMapFunc.clone(); |
| 129 | |
| 130 | clone.nameMapType = (Hashtable<String, Type>) nameMapType.clone(); |
| 131 | |
| 132 | clone.relevantClauses = (HashSet<Clause>) relevantClauses.clone(); |
| 133 | |
| 134 | clone.listClauses = (ArrayList<Clause>) listClauses.clone(); |
| 135 | |
| 136 | clone.unnormalizedClauses = (ArrayList<Clause>) unnormalizedClauses.clone(); |
| 137 | |
| 138 | clone.sigMap = (Hashtable<String, Clause>) sigMap.clone(); |
| 139 | |
| 140 | // clone.mapConstantID = (HashMap<String, Integer>) mapConstantID.clone(); |
| 141 | |
| 142 | clone.scopes = (HashMap<Predicate, ArrayList<ConjunctiveQuery>>) scopes.clone(); |
| 143 | |
| 144 | clone.scopingRules = (ArrayList<ConjunctiveQuery>) scopingRules.clone(); |
| 145 | |
| 146 | clone.clusteringPredicates = (ArrayList<Predicate>) clusteringPredicates.clone(); |
| 147 | |
| 148 | return clone; |
| 149 | |
| 150 | } |
| 151 | |
| 152 | |
| 153 | public ArrayList<ConjunctiveQuery> getAllDatalogRules(){ |
| 154 | return datalogRules; |
| 155 | } |
| 156 | |
| 157 | /** |
| 158 | * Returns the RDB used by this MLN. |
| 159 | */ |
| 160 | public RDB getRDB() { |
| 161 | return db; |
| 162 | } |
| 163 | |
| 164 | |
| 165 | public int getID(){ |
| 166 | return id; |
| 167 | } |
| 168 | |
| 169 | /** |
| 170 | * Constructor of MLN. {@link MarkovLogicNetwork#parser} will be |
| 171 | * constructed here. |
| 172 | * |
| 173 | */ |
| 174 | public MarkovLogicNetwork(){ |
| 175 | parser = new InputParser(this); |
| 176 | id = (idGen++); |
| 177 | String relp = "mln" + id + "_"; |
| 178 | relAtoms = relp + "atoms"; |
| 179 | relClauses = relp + "clauses"; |
| 180 | relClausePart = relp + "clause_part"; |
| 181 | relAtomPart = relp + "atom_part"; |
| 182 | } |
| 183 | |
| 184 | /** |
| 185 | * Marks a clause as relevant. Called by KBMC. |
| 186 | * |
| 187 | * @see tuffy.ground.KBMC#run() |
| 188 | */ |
| 189 | public void setClauseAsRelevant(Clause c){ |
| 190 | relevantClauses.add(c); |
| 191 | } |
| 192 | |
| 193 | /** |
| 194 | * Returns the set of relevant clauses. |
| 195 | */ |
| 196 | public HashSet<Clause> getRelevantClauses(){ |
| 197 | return relevantClauses; |
| 198 | } |
| 199 | |
| 200 | /** |
| 201 | * Registers a new, unnormalized clause. |
| 202 | * |
| 203 | * @param c the clause to be registered |
| 204 | */ |
| 205 | public void registerClause(Clause c){ |
| 206 | if(c == null) return; |
| 207 | unnormalizedClauses.add(c); |
| 208 | } |
| 209 | |
| 210 | /** |
| 211 | * Add a scoping rule |
| 212 | * @param cq |
| 213 | */ |
| 214 | public void registerScopingRule(ConjunctiveQuery cq){ |
| 215 | Predicate p = cq.head.getPred(); |
| 216 | ArrayList<ConjunctiveQuery> qs = scopes.get(p); |
| 217 | if(qs == null){ |
| 218 | qs = new ArrayList<ConjunctiveQuery>(); |
| 219 | scopes.put(p, qs); |
| 220 | } |
| 221 | qs.add(cq); |
| 222 | cq.setScopingRule(true); |
| 223 | scopingRules.add(cq); |
| 224 | } |
| 225 | |
| 226 | ArrayList<ConjunctiveQuery> datalogRules = new ArrayList<ConjunctiveQuery>(); |
| 227 | |
| 228 | ArrayList<ConjunctiveQuery> intermediateRules = new ArrayList<ConjunctiveQuery>(); |
| 229 | |
| 230 | ArrayList<ConjunctiveQuery> postprocRules = new ArrayList<ConjunctiveQuery>(); |
| 231 | |
| 232 | /** |
| 233 | * Add a datalog rule |
| 234 | * @param cq |
| 235 | */ |
| 236 | public void registerDatalogRule(ConjunctiveQuery cq){ |
| 237 | datalogRules.add(cq); |
| 238 | } |
| 239 | |
| 240 | public void registerPostprocRule(ConjunctiveQuery cq){ |
| 241 | postprocRules.add(cq); |
| 242 | } |
| 243 | |
| 244 | public void registerIntermediateRule(ConjunctiveQuery cq){ |
| 245 | intermediateRules.add(cq); |
| 246 | } |
| 247 | |
| 248 | |
| 249 | /** |
| 250 | * Test whether a predicate is scoped |
| 251 | */ |
| 252 | public boolean isScoped(Predicate p){ |
| 253 | return scopes.containsKey(p); |
| 254 | } |
| 255 | |
| 256 | /** |
| 257 | * Execute the scoping rules for a predicate |
| 258 | * @param p the target predicate |
| 259 | */ |
| 260 | private void applyScopeForPred(Predicate p){ |
| 261 | for(ConjunctiveQuery cq : scopes.get(p)){ |
| 262 | UIMan.verbose(1, cq.toString()); |
| 263 | cq.materialize(db, p.isClosedWorld() ? false : null, new ArrayList<String>()); |
| 264 | int ni = db.getLastUpdateRowCount(); |
| 265 | UIMan.verbose(1, "### Inserted " + UIMan.comma(ni) + (ni!=1 ? " new tuples" : " new tuple") + "\n"); |
| 266 | db.analyze(p.getRelName()); |
| 267 | } |
| 268 | } |
| 269 | |
| 270 | |
| 271 | /** |
| 272 | * Execute all Datalog rules |
| 273 | */ |
| 274 | public void executeAllDatalogRules(){ |
| 275 | if(datalogRules.isEmpty()) return; |
| 276 | |
| 277 | long total = 0; |
| 278 | UIMan.println(">>> Executing Datalog rules..."); |
| 279 | for(ConjunctiveQuery cq : datalogRules){ |
| 280 | Predicate p = cq.head.getPred(); |
| 281 | UIMan.verbose(1, cq.toString()); |
| 282 | Timer.start("datalogq"); |
| 283 | cq.buildIndexes(db, null, null, null, false, new ArrayList<String>()); |
| 284 | cq.materialize(db, true, new ArrayList<String>()); |
| 285 | int ni = db.getLastUpdateRowCount(); |
| 286 | total += ni; |
| 287 | UIMan.verbose(1, "### inserted " + UIMan.comma(ni) + |
| 288 | (ni!=1 ? " new tuples" : " new tuple")); |
| 289 | UIMan.verbose(1, "### current cardinality of '" + |
| 290 | p.getName() + "' = " + |
| 291 | UIMan.comma(db.countTuples(p.getRelName()))); |
| 292 | String tm = Timer.elapsed("datalogq"); |
| 293 | UIMan.verbose(1, "### took time " + tm + |
| 294 | "\n"); |
| 295 | db.analyze(p.getRelName()); |
| 296 | } |
| 297 | } |
| 298 | |
| 299 | |
| 300 | /** |
| 301 | * Execute all Postprocessing rules |
| 302 | */ |
| 303 | public void executeAllPostprocRules(){ |
| 304 | UIMan.println(">>> Executing Postprocessing rules..."); |
| 305 | for(ConjunctiveQuery cq : postprocRules){ |
| 306 | UIMan.verbose(1, cq.toString()); |
| 307 | String sql = "DELETE FROM " + cq.head.getPred().getRelName(); |
| 308 | db.update(sql); |
| 309 | db.vacuum(cq.head.getPred().getRelName()); |
| 310 | cq.materialize(db, true, new ArrayList<String>()); |
| 311 | int ni = db.getLastUpdateRowCount(); |
| 312 | UIMan.verbose(1, "### inserted " + ni + (ni!=1 ? " new tuples" : " new tuple") + "\n"); |
| 313 | } |
| 314 | } |
| 315 | |
| 316 | public void executeAllIntermediateRules(){ |
| 317 | UIMan.println(">>> Executing intermediate rules..."); |
| 318 | for(ConjunctiveQuery cq : intermediateRules){ |
| 319 | UIMan.verbose(1, cq.toString()); |
| 320 | String sql = "DELETE FROM " + cq.head.getPred().getRelName(); |
| 321 | db.update(sql); |
| 322 | db.vacuum(cq.head.getPred().getRelName()); |
| 323 | cq.materialize(db, true, new ArrayList<String>()); |
| 324 | int ni = db.getLastUpdateRowCount(); |
| 325 | UIMan.verbose(1, "### inserted " + ni + (ni!=1 ? " new tuples" : " new tuple") + "\n"); |
| 326 | Predicate p = cq.head.getPred(); |
| 327 | db.analyze(p.getRelName()); |
| 328 | } |
| 329 | } |
| 330 | |
| 331 | |
| 332 | /** |
| 333 | * Execute all scoping rules |
| 334 | * |
| 335 | * @return true iff there is at least one scoping rule |
| 336 | */ |
| 337 | public boolean applyAllScopes(){ |
| 338 | if(scopes.isEmpty()) return false; |
| 339 | UIMan.println(">>> Applying scoping rules..."); |
| 340 | ArrayList<Predicate> up = new ArrayList<Predicate>(); |
| 341 | for(Predicate p : scopes.keySet()){ |
| 342 | if(p.isClosedWorld() && !p.isCompletelySepcified()){ |
| 343 | applyScopeForPred(p); |
| 344 | }else{ |
| 345 | up.add(p); |
| 346 | } |
| 347 | } |
| 348 | for(Predicate p : up){ |
| 349 | if(!p.isCompletelySepcified()){ |
| 350 | applyScopeForPred(p); |
| 351 | } |
| 352 | } |
| 353 | return true; |
| 354 | } |
| 355 | |
| 356 | /** |
| 357 | * Get the clause object by integer ID. Accepts negative id, and will |
| 358 | * translate it into positive. Does not accept zero id or id larger |
| 359 | * than the number of clauses, and will return null. |
| 360 | * @param id ID of wanted clause. |
| 361 | */ |
| 362 | public Clause getClauseById(int id){ |
| 363 | if(id < 0) id = -id; |
| 364 | if(id < 1 || id > listClauses.size()) return null; |
| 365 | return listClauses.get(id-1); |
| 366 | } |
| 367 | |
| 368 | /** |
| 369 | * Normalize all clauses. If the signature of this clause is |
| 370 | * as the same as some some existing clauses in {@link MarkovLogicNetwork#listClauses}, |
| 371 | * then {@link Clause#absorb(Clause)} this new clause. If not |
| 372 | * absorbed, this new clause is set an ID sequentially and a name |
| 373 | * Clause$id. Predicates in this clause is registered |
| 374 | * by {@link Predicate#addRelatedClause(Clause)}. |
| 375 | * |
| 376 | * @see Clause#normalize() |
| 377 | * @see Clause#absorb(Clause) |
| 378 | */ |
| 379 | public void normalizeClauses(){ |
| 380 | listClauses.clear(); |
| 381 | for(Clause c : unnormalizedClauses){ |
| 382 | |
| 383 | // applyScopes(c); |
| 384 | if(c.hasEmbeddedWeight()){ |
| 385 | listClauses.add(c); |
| 386 | int id = listClauses.size(); |
| 387 | c.setId(id); |
| 388 | c.setName("Clause" + id); |
| 389 | for(Predicate p : c.getReferencedPredicates()){ |
| 390 | p.addRelatedClause(c); |
| 391 | } |
| 392 | this.unnormal2normal.put(c, c); |
| 393 | continue; |
| 394 | } |
| 395 | Clause tmpc = c; |
| 396 | c = c.normalize(); |
| 397 | |
| 398 | this.unnormal2normal.put(tmpc, c); |
| 399 | |
| 400 | if(c == null) continue; |
| 401 | c.checkVariableSafety(); |
| 402 | Clause ec = sigMap.get(c.getSignature()); |
| 403 | if(ec == null){ |
| 404 | listClauses.add(c); |
| 405 | sigMap.put(c.getSignature(), c); |
| 406 | int id = listClauses.size(); |
| 407 | c.setId(id); |
| 408 | c.setName("Clause" + id); |
| 409 | for(Predicate p : c.getReferencedPredicates()){ |
| 410 | p.addRelatedClause(c); |
| 411 | } |
| 412 | }else{ |
| 413 | ec.absorb(c); |
| 414 | } |
| 415 | } |
| 416 | for(Clause c : listClauses){ |
| 417 | UIMan.verbose(2, "\n" + c.toString()); |
| 418 | } |
| 419 | } |
| 420 | |
| 421 | /** |
| 422 | * Finalize the definitions of all clauses, i.e., prepare |
| 423 | * the database table used by each clause, including |
| 424 | * 1) instance table for each clause; 2) SQL needed to |
| 425 | * ground this clause. |
| 426 | * Call this when all clauses have been parsed. |
| 427 | */ |
| 428 | public void finalizeClauseDefinitions(RDB adb){ |
| 429 | for(Clause c : listClauses){ |
| 430 | c.prepareForDB(adb); |
| 431 | } |
| 432 | } |
| 433 | |
| 434 | /** |
| 435 | * Return the type of a given name; create if this type does not exist. |
| 436 | */ |
| 437 | public Type getOrCreateTypeByName(String name){ |
| 438 | Type t= nameMapType.get(name); |
| 439 | if(t == null){ |
| 440 | t = new Type(name); |
| 441 | nameMapType.put(name, t); |
| 442 | } |
| 443 | return t; |
| 444 | } |
| 445 | |
| 446 | |
| 447 | /** |
| 448 | * Call materialize() for all types. This will put |
| 449 | * the domain members of each type into corresponding |
| 450 | * database tables. |
| 451 | * |
| 452 | * @see Type#storeConstantList(RDB) |
| 453 | */ |
| 454 | private void materializeAllTypes(RDB adb){ |
| 455 | for(Type t : nameMapType.values()){ |
| 456 | t.storeConstantList(adb); |
| 457 | } |
| 458 | } |
| 459 | |
| 460 | |
| 461 | |
| 462 | /** |
| 463 | * Return the set of all predicates. |
| 464 | */ |
| 465 | public HashSet<Predicate> getAllPred() { |
| 466 | return new HashSet<Predicate>(listPred); |
| 467 | } |
| 468 | |
| 469 | public ArrayList<Predicate> getAllPredOrderByName(){ |
| 470 | ArrayList<String> pnames = new ArrayList<String>(); |
| 471 | for(Predicate p : listPred){ |
| 472 | pnames.add(p.getName()); |
| 473 | } |
| 474 | Collections.sort(pnames); |
| 475 | ArrayList<Predicate> ps = new ArrayList<Predicate>(); |
| 476 | for(String pn : pnames){ |
| 477 | ps.add(this.getPredByName(pn)); |
| 478 | } |
| 479 | return ps; |
| 480 | } |
| 481 | |
| 482 | |
| 483 | |
| 484 | /** |
| 485 | * Register a new predicate. Here by ``register'' |
| 486 | * it means 1) set ID for this predicate sequentially; 2) push |
| 487 | * it into {@link MarkovLogicNetwork#listPred}; 3) |
| 488 | * building the map from predicate name to this predicate. |
| 489 | */ |
| 490 | public void registerPred(Predicate p){ |
| 491 | if(nameMapPred.containsKey(p.getName())){ |
| 492 | ExceptionMan.die("Duplicate predicate definitions - " + p.getName()); |
| 493 | } |
| 494 | if(Predicate.isBuiltInPredName(p.getName())){ |
| 495 | System.err.println("WARNING: user-defined predicate '" + |
| 496 | p.getName() + "' will be overridden by the built-in one!"); |
| 497 | return; |
| 498 | } |
| 499 | p.setMLN(this); |
| 500 | p.setID(listPred.size()); |
| 501 | listPred.add(p); |
| 502 | nameMapPred.put(p.getName(), p); |
| 503 | } |
| 504 | |
| 505 | /** |
| 506 | * Return the predicate of the given name; null if such predicate does not exist. |
| 507 | */ |
| 508 | public Predicate getPredByName(String name) { |
| 509 | Predicate bip = Predicate.getBuiltInPredByName(name); |
| 510 | if(bip != null){ |
| 511 | return bip; |
| 512 | } |
| 513 | return nameMapPred.get(name); |
| 514 | } |
| 515 | |
| 516 | |
| 517 | /** |
| 518 | * Get a function by its name; can be built-in. |
| 519 | * @param name |
| 520 | * |
| 521 | */ |
| 522 | public Function getFunctionByName(String name) { |
| 523 | Function f = Function.getBuiltInFunctionByName(name); |
| 524 | if(f != null){ |
| 525 | return f; |
| 526 | } |
| 527 | return nameMapFunc.get(name); |
| 528 | } |
| 529 | |
| 530 | |
| 531 | /** |
| 532 | * Return all unnormalized clauses as read from the input file. |
| 533 | */ |
| 534 | public ArrayList<Clause> getAllUnnormalizedClauses(){ |
| 535 | return unnormalizedClauses; |
| 536 | } |
| 537 | |
| 538 | |
| 539 | /** |
| 540 | * Return all normalized clauses. |
| 541 | */ |
| 542 | public ArrayList<Clause> getAllNormalizedClauses(){ |
| 543 | return listClauses; |
| 544 | } |
| 545 | |
| 546 | /** |
| 547 | * Return assigned ID of a constant symbol. |
| 548 | * If this symbol is a new one, a new ID will be assigned to it, |
| 549 | * and the symbol table will be updated. |
| 550 | */ |
| 551 | public int getSymbolID(String symbol, Type type) { |
| 552 | Integer id = mapConstantID.get(symbol); |
| 553 | if(id == null) { |
| 554 | id = mapConstantID.size() + 1; |
| 555 | mapConstantID.put(symbol, id); |
| 556 | if(Config.learning_mode){ |
| 557 | Clause.mappingFromID2Const.put(id, symbol); |
| 558 | } |
| 559 | } |
| 560 | if(type != null && !type.isNonSymbolicType()) { |
| 561 | type.addConstant(id); |
| 562 | } |
| 563 | return id; |
| 564 | |
| 565 | } |
| 566 | |
| 567 | /** |
| 568 | * Ground and store all query atoms. |
| 569 | * |
| 570 | * @see Predicate#addQuery(Atom) |
| 571 | * @see Predicate#groundAndStoreAtom(Atom) |
| 572 | */ |
| 573 | public void storeAllQueries(){ |
| 574 | for(Predicate p : getAllPred()){ |
| 575 | if(p.isClosedWorld()){ |
| 576 | continue; |
| 577 | } |
| 578 | p.storeQueries(); |
| 579 | } |
| 580 | } |
| 581 | |
| 582 | /** |
| 583 | * Store all evidences into the database by flushing the "buffers". |
| 584 | * These tuples are pushed into the relational table {@link Predicate#getRelName()} |
| 585 | * in the database. |
| 586 | */ |
| 587 | public void storeAllEvidence(){ |
| 588 | for(Predicate p : getAllPred()){ |
| 589 | p.flushEvidence(); |
| 590 | } |
| 591 | } |
| 592 | |
| 593 | /** |
| 594 | * Close all file handles used by each predicate in {@link MarkovLogicNetwork#listPred}. |
| 595 | */ |
| 596 | public void closeFiles(){ |
| 597 | for(Predicate p : listPred){ |
| 598 | p.closeFiles(); |
| 599 | } |
| 600 | } |
| 601 | |
| 602 | /** |
| 603 | * Parse multiple MLN program files. |
| 604 | * |
| 605 | * @param progFiles list of MLN program files (in Alchemy format) |
| 606 | */ |
| 607 | public void loadPrograms(String[] progFiles){ |
| 608 | for(String f : progFiles){ |
| 609 | String g = FileMan.getGZIPVariant(f); |
| 610 | if(g == null){ |
| 611 | ExceptionMan.die("non-existent file: " + f); |
| 612 | }else{ |
| 613 | f = g; |
| 614 | } |
| 615 | UIMan.println(">>> Parsing program file: " + f); |
| 616 | parser.parseProgramFile(f); |
| 617 | } |
| 618 | normalizeClauses(); |
| 619 | } |
| 620 | |
| 621 | public void loadProgramsButNotNormalizeClauses(String[] progFiles){ |
| 622 | for(String f : progFiles){ |
| 623 | String g = FileMan.getGZIPVariant(f); |
| 624 | if(g == null){ |
| 625 | ExceptionMan.die("non-existent file: " + f); |
| 626 | }else{ |
| 627 | f = g; |
| 628 | } |
| 629 | UIMan.println(">>> Parsing program file: " + f); |
| 630 | parser.parseProgramFile(f); |
| 631 | } |
| 632 | } |
| 633 | |
| 634 | /** |
| 635 | * Parse multiple MLN evidence files. If file size is larger |
| 636 | * than 1MB, then uses a file stream |
| 637 | * incrementally parse this file. Can also accept .gz file (see {@link GZIPInputStream#GZIPInputStream(InputStream)}). |
| 638 | * |
| 639 | * @param evidFiles list of MLN evidence files (in Alchemy format) |
| 640 | */ |
| 641 | public void loadEvidences(String[] evidFiles){ |
| 642 | int chunkSize = Config.evidence_file_chunk_size; |
| 643 | for(String f : evidFiles){ |
| 644 | String g = FileMan.getGZIPVariant(f); |
| 645 | if(g == null){ |
| 646 | ExceptionMan.die("File does not exist: " + f); |
| 647 | }else{ |
| 648 | f = g; |
| 649 | } |
| 650 | UIMan.println(">>> Parsing evidence file: " + f); |
| 651 | |
| 652 | if(FileMan.getFileSize(f) <= chunkSize){ |
| 653 | parser.parseEvidenceFile(f); |
| 654 | }else{ |
| 655 | try{ |
| 656 | long lineOffset = 0, lastChunkLines = 0; |
| 657 | BufferedReader reader = FileMan.getBufferedReaderMaybeGZ(f); |
| 658 | StringBuilder sb = new StringBuilder(); |
| 659 | String line = reader.readLine(); |
| 660 | while(line != null){ |
| 661 | sb.append(line); |
| 662 | sb.append("\n"); |
| 663 | lastChunkLines ++; |
| 664 | if(sb.length() >= chunkSize){ |
| 665 | parser.parseEvidenceString(sb.toString(), lineOffset); |
| 666 | sb.delete(0, sb.length()); |
| 667 | sb = new StringBuilder(); |
| 668 | lineOffset += lastChunkLines; |
| 669 | lastChunkLines = 0; |
| 670 | UIMan.print("."); |
| 671 | } |
| 672 | line = reader.readLine(); |
| 673 | } |
| 674 | reader.close(); |
| 675 | if(sb.length() > 0){ |
| 676 | parser.parseEvidenceString(sb.toString(), lineOffset); |
| 677 | } |
| 678 | UIMan.println(); |
| 679 | }catch(Exception e){ |
| 680 | ExceptionMan.handle(e); |
| 681 | } |
| 682 | } |
| 683 | try { |
| 684 | DebugMan.runGC(); |
| 685 | } catch (Exception e) { |
| 686 | e.printStackTrace(); |
| 687 | } |
| 688 | } |
| 689 | } |
| 690 | |
| 691 | /** |
| 692 | * Parse multiple MLN query files. |
| 693 | * |
| 694 | * @param queryFiles list of MLN query files (in Alchemy format) |
| 695 | */ |
| 696 | public void loadQueries(String[] queryFiles){ |
| 697 | for(String f : queryFiles){ |
| 698 | String g = FileMan.getGZIPVariant(f); |
| 699 | if(g == null){ |
| 700 | ExceptionMan.die("non-existent file: " + f); |
| 701 | }else{ |
| 702 | f = g; |
| 703 | } |
| 704 | UIMan.println(">>> Parsing query file: " + f); |
| 705 | parser.parseQueryFile(f); |
| 706 | } |
| 707 | } |
| 708 | |
| 709 | |
| 710 | /** |
| 711 | * Read in the query atoms provided by the command line. |
| 712 | */ |
| 713 | public void parseQueryCommaList(String queryAtoms){ |
| 714 | parser.parseQueryCommaList(queryAtoms); |
| 715 | } |
| 716 | |
| 717 | /** |
| 718 | * Prepare the database for each predicate and clause. |
| 719 | * @see Predicate#prepareDB(RDB) |
| 720 | * @see MarkovLogicNetwork#finalizeClauseDefinitions(RDB) |
| 721 | */ |
| 722 | public void prepareDB(RDB adb){ |
| 723 | db = adb; |
| 724 | for(Predicate p : listPred){ |
| 725 | p.prepareDB(adb); |
| 726 | } |
| 727 | finalizeClauseDefinitions(adb); |
| 728 | } |
| 729 | |
| 730 | public void setDB(RDB adb){ |
| 731 | this.db = adb; |
| 732 | } |
| 733 | |
| 734 | /** |
| 735 | * Clean up temporary data in DB and working dir, including |
| 736 | * 1) drop schema in PostgreSQL; 2) remove directory. |
| 737 | * |
| 738 | * @return true on success |
| 739 | */ |
| 740 | public boolean cleanUp(){ |
| 741 | closeFiles(); |
| 742 | return db.dropSchema(Config.db_schema) && |
| 743 | FileMan.removeDirectory(new File(Config.getWorkingDir())); |
| 744 | } |
| 745 | |
| 746 | /** |
| 747 | * Stores constants and evidence into database table. |
| 748 | * |
| 749 | * @see MarkovLogicNetwork#materializeAllTypes(RDB) |
| 750 | * @see MarkovLogicNetwork#storeAllEvidence() |
| 751 | */ |
| 752 | public void materializeTables(){ |
| 753 | UIMan.verbose(1, ">>> Storing symbol tables..."); |
| 754 | UIMan.verbose(1, "### constants = " + mapConstantID.size()); |
| 755 | db.createConstantTable(mapConstantID); |
| 756 | mapConstantID = null; |
| 757 | try { |
| 758 | DebugMan.runGC(); |
| 759 | DebugMan.runGC(); |
| 760 | DebugMan.runGC(); |
| 761 | } catch (Exception e) { |
| 762 | e.printStackTrace(); |
| 763 | } |
| 764 | materializeAllTypes(db); |
| 765 | UIMan.println(">>> Storing evidence..."); |
| 766 | storeAllEvidence(); |
| 767 | } |
| 768 | |
| 769 | } |