main_memory_storage_manager.hxx

Go to the documentation of this file.

//#include "main_memory_storage_manager.h"
// This file is included in a header file now.
#include "timer.h"
#include <algorithm>

template<class T>
STLMainMemory_Storage_Manager<T>::STLMainMemory_Storage_Manager(struct main_memory_storage_spec *s) {
  strategy = s->strategy;
  low_water_it_cached = _itable.end();
  high_water_it_cached = _itable.begin();
}

template<class T>
STLMainMemory_Storage_Manager<T>::STLMainMemory_Storage_Manager(Hazy_Database *db_conn, struct main_memory_storage_spec *s, struct hazy_model &hm) : Storage_Manager<T>(db_conn) {
  strategy = s->strategy;
  this->entity_table_name = s->entity_table_name;
  if(s->bRebuild)
    this->loadFromDatabase(hm);
  
  if(hm.isHazy()) {
    std::sort(_itable.begin(), _itable.end(), STLMainMemory_Storage_Manager<T>::cluster_sort_predicate);
    
    struct internal_record l, h;
    
    l.eps = hm.low_water; h.eps = hm.high_water;
    
    low_water_it_cached  = std::lower_bound( _itable.begin() , _itable.end(), l,
                                             STLMainMemory_Storage_Manager<T>::cluster_sort_predicate);
    high_water_it_cached = std::upper_bound( _itable.begin() , _itable.end(), h,
                                             STLMainMemory_Storage_Manager<T>::cluster_sort_predicate);
  }
}

template<class T>
void
STLMainMemory_Storage_Manager<T>::
resort(struct hazy_model &hm) {
  internal_iterator it;
  LOGGING_ONLY(Timer t1(true););
  switch( strategy ) {
    // Only hazy resorts the table
  case hazy_model::EAGER_HAZY:   
    {
      LOGGING_ONLY(Timer update_eps(true); int nClassified = 0; double classify_time = 0.0; double update_ex_table_time = 0.0;);
      // Now we update the epsilon values of the internal table
      for(it = _itable.begin(); it != _itable.end(); it ++) {
        LOGGING_ONLY(Timer classify_timer(true););
        double _eps = classify(hm._model, it->v);
        LOGGING_ONLY(classify_time += classify_timer.stop(););
        // If the entities label may have changed, then update it.
        if(it->eps >= hm.low_water || it->eps <= hm.high_water) {
          LOGGING_ONLY(Timer update_ex_table_timer(true););
          _eager_external_table[it->k] = _eps > 0;
          LOGGING_ONLY(update_ex_table_time += update_ex_table_timer.stop(););
        }
        // update the label in _table for every entity
        it->eps = _eps ;
        LOGGING_ONLY(nClassified++;);
      }    
      LOGGING_ONLY(std::cout << "time for only classification: " << classify_time << std::endl);
      LOGGING_ONLY(std::cout << "time for only update external table: " << update_ex_table_time << std::endl;);
      LOGGING_ONLY(std::cout << "\tUpdated the epsilon value" << nClassified << " in " << update_eps.stop() << std::endl;  );
      
      // And now resort with the new epsilon values...
      LOGGING_ONLY(Timer internal_resort(true););
      std::sort(_itable.begin(), _itable.end(), STLMainMemory_Storage_Manager<T>::cluster_sort_predicate);
      LOGGING_ONLY(std::cout << "\t Internal Resort: " << internal_resort.stop() << std::endl;);
      
      struct internal_record l, h;
      
      l.eps = 0.0; h.eps = 0.0;
      
      LOGGING_ONLY(Timer bound_finder(true););
      low_water_it_cached  = std::lower_bound( _itable.begin() , _itable.end(), l,
                                               STLMainMemory_Storage_Manager<T>::cluster_sort_predicate);
      high_water_it_cached = std::upper_bound( _itable.begin() , _itable.end(), h,
                                               STLMainMemory_Storage_Manager<T>::cluster_sort_predicate);
      LOGGING_ONLY(std::cout << "\t Found the bound: " << bound_finder.stop() << std::endl;);
      LOGGING_ONLY(std::cout << "model is: " << hm._model << std::endl;);
    }
    break;
  case hazy_model::LAZY_HAZY:    
    {
      // Now we update the epsilon values of the internal table
      for(it = _itable.begin(); it != _itable.end(); it ++) {
        double _eps = classify(hm._model, it->v);
        // update the label in _table for every entity
        it->eps = _eps ;
      }    
      // And now resort with the new epsilon values...
      std::sort(_itable.begin(), _itable.end(), STLMainMemory_Storage_Manager<T>::cluster_sort_predicate);
      
      struct internal_record l, h;
      
      l.eps = 0.0; h.eps = 0.0;
      
      low_water_it_cached  = std::lower_bound( _itable.begin() , _itable.end(), l,
                                               STLMainMemory_Storage_Manager<T>::cluster_sort_predicate);
      high_water_it_cached = std::upper_bound( _itable.begin() , _itable.end(), h,
                                               STLMainMemory_Storage_Manager<T>::cluster_sort_predicate);
    }
    break;    
  case hazy_model::EAGER_NAIVE:
  case hazy_model::LAZY_NAIVE:
    break;
  default:
    throw UnknownStrategyException(__FILE__, __LINE__, "[resort]");
  }
  LOGGING_ONLY(std::cout << "resort takes: " << t1.stop() << std::endl;);
}

template<class T>
void
STLMainMemory_Storage_Manager<T>::
incrementalUpdate(struct hazy_model &hm, double &waste_time) {
  LOGGING_ONLY(std::cout << "in mm storage manager, incremental update, strategy: " << strategy << std::endl;);
  waste_time = 0.0;
  LOGGING_ONLY(Timer total_timer(true););
  switch (strategy) {
  case hazy_model::EAGER_HAZY:
    {
      Timer waste_timer;
      internal_iterator low_water_it, high_water_it;
      struct internal_record l,h;
      int nTotalTuples = 0, nWastedTuples = 0;  
      LOGGING_ONLY(int tuplesChangeLabels = 0;);
      l.eps = hm.low_water; h.eps = hm.high_water;
      low_water_it  = std::lower_bound( _itable.begin() , low_water_it_cached, l, 
                                        STLMainMemory_Storage_Manager<T>::cluster_sort_predicate);
      high_water_it = std::upper_bound( high_water_it_cached, _itable.end(), h,
                                        STLMainMemory_Storage_Manager<T>::cluster_sort_predicate);
      
      //set new iterators
      low_water_it_cached = low_water_it;
      high_water_it_cached = high_water_it;
      
      waste_timer.start();
      for(; low_water_it != high_water_it; low_water_it++ ) {
        nTotalTuples++;      
        bool nVal = (classify(hm._model, low_water_it->v) > 0.0);
        LOGGING_ONLY(if(nVal != _eager_external_table[low_water_it->k]) tuplesChangeLabels ++; );
        VERBOSE_ONLY(std::cout << "\t\t Updating a label: " << low_water_it->k << " to " << nVal << " from " << _eager_external_table[low_water_it->k] << " v=" << low_water_it->v << std::endl;);
        _eager_external_table[low_water_it->k] = nVal;
        // if it's 0 then it's a wasted tuple
        nWastedTuples += (1 - (int) nVal );
      }
      waste_time = (nTotalTuples > 0) ? waste_timer.stop() * (double) nWastedTuples/ (double) nTotalTuples : 0.0;
      LOGGING_ONLY(std::cout << "incremental update: tuples between low & high water: " << nTotalTuples << std::endl;);
      LOGGING_ONLY(std::cout << "\nTotal Tuples=" << nTotalTuples << " nChangedLabels=" << tuplesChangeLabels << std::endl;);
      LOGGING_ONLY(std::cout << "\tWaste Ratio: " << ((double) nWastedTuples / (double) nTotalTuples) << " waste time is: " << waste_time << " of " << waste_timer.getElapsedTime() << std::endl;);
    }
    break;
  case hazy_model::EAGER_NAIVE:
    {
      key_iterator_type lz;
      LOGGING_ONLY(Timer t1(true););

      for(lz = _entity_table.begin(); lz != _entity_table.end(); lz++) {
        bool nVal = classify(hm._model, lz->second) > 0.0;      
        _eager_external_table[lz->first] = nVal;
      }

      LOGGING_ONLY(std::cout << "n-rescan update: " << t1.stop(););
    }
    break;      
  case hazy_model::LAZY_HAZY:
  case hazy_model::LAZY_NAIVE:
    break;
  default:
    throw UnknownStrategyException(__FILE__, __LINE__, "[incrementalUpdate]");
  }
  LOGGING_ONLY(std::cout << "\ttotal time is: " << total_timer.stop() << std::endl;);
}


template<class T>
void
STLMainMemory_Storage_Manager<T>::
insertEntity(struct hazy_model &m, key e, T x)
{
  struct internal_record ir;
  if(m.isHazy() ) {
    ir.k   = e;
    ir.eps = classify(m._model, x);
    ir.v   = x;
  }
  switch(strategy) {
  case hazy_model::LAZY_NAIVE:
    _entity_table[e] = x;       
    break;
  case hazy_model::LAZY_HAZY:
    _entity_table[e] = x;
    _itable.push_back(ir);
    break;
  case hazy_model::EAGER_NAIVE:
    _eager_external_table[e] = classify(m._model, x) > 0;
    _entity_table[e] = x;
    break;
  case hazy_model::EAGER_HAZY:
    {      
      _eager_external_table[e] = ir.eps > 0;
      _itable.push_back(ir);
    }
    break;
  default:
    throw UnknownStrategyException(__FILE__, __LINE__, "[insertEntity]");
    return;
  }
  VERBOSE_ONLY(std::cout << "entity inserted, id: " << e << std::endl;);
}

template<class T>
void 
STLMainMemory_Storage_Manager<T>::
getEntityClass(key e, sClass &c, struct hazy_model &hm) {
  switch (strategy) {
  case hazy_model::LAZY_NAIVE:
    c = classify(hm._model, _entity_table[e]) > 0.0 ? 1 : 0;
    break;
  case hazy_model::LAZY_HAZY:
    // We could potentially avoid the dot product here...
    c = classify(hm._model, _entity_table[e]) > 0.0 ? 1 : 0;
    break;
  case hazy_model::EAGER_NAIVE:
  case hazy_model::EAGER_HAZY:
    c = _eager_external_table[e];      
    break;
  default:
    throw UnknownStrategyException(__FILE__, __LINE__, "[insert entity]");
  }
  LOGGING_ONLY(std::cout << "label of " << e << " is: " << c << std::endl;);
}

template<class T>
void 
STLMainMemory_Storage_Manager<T>::
getNumInClass(sClass c, int &nClass, struct hazy_model &hm, double &waste_time) {
  
  
  switch (strategy) {
  case hazy_model::LAZY_NAIVE: 
    {
      LOGGING_ONLY(std::cout << "in get num in class for lazy naive" << std::endl;);
      key_iterator_type lz;
      for(lz = _entity_table.begin(); lz != _entity_table.end(); lz++) {
        nClass += classify(hm._model, lz->second) > 0.0 ? 1 : 0;
      }
      break;
    }
  case hazy_model::LAZY_HAZY:      
    {
      int nTotalTuples = 0, nWastedTuples = 0;  
      struct internal_record l;
      l.eps = hm.low_water; 
      internal_iterator  lazy_hazy_it = std::lower_bound( _itable.begin() , _itable.end(), l, 
                                                          STLMainMemory_Storage_Manager<T>::cluster_sort_predicate);
      Timer waste_timer(true);
      bool past_high_water = false;
      double high_water = hm.high_water;
      LOGGING_ONLY(int first_item = 0;);
      for(; lazy_hazy_it != _itable.end(); lazy_hazy_it++) {
        nTotalTuples++;
        LOGGING_ONLY(if(first_item == 0) std::cout << "first scanned tuple id: " << lazy_hazy_it->k << std::endl;);
        if(!past_high_water) { 
          nClass += classify(hm._model, lazy_hazy_it->v) > 0.0 ? 1 : 0; 
          past_high_water = lazy_hazy_it->eps > high_water;
        } else {
          nClass++;
        }
        LOGGING_ONLY(first_item ++;);
      }
      nWastedTuples = nTotalTuples - nClass;
      
      waste_time = (nTotalTuples > 0) ? waste_timer.stop() * ((double) nWastedTuples) / ((double) nTotalTuples): 0.0;
      LOGGING_ONLY(std::cout << "tuples between low&high water: " << nTotalTuples << std::endl;);
      LOGGING_ONLY(std::cout << "\tWaste Ratio: " << ((double) nWastedTuples) / ((double) nTotalTuples) <<  " waste time: " << waste_time << std::endl;);
      break;
    }
  case hazy_model::EAGER_NAIVE: 
    {    
      for(  std::map<key,bool>::iterator it  = _eager_external_table.begin();
            it != _eager_external_table.end();
            it++) {
        VERBOSE_ONLY(std::cout << it->second << " ";); 
        nClass += it->second ? 1 : 0;
      }
      
      VERBOSE_ONLY(std::cout << std::endl;);
      break;
    }
    
    // There does not appear to be an easy way to fix this, because it requires clustering...
  case hazy_model::EAGER_HAZY:
    {
      LOGGING_ONLY(int nScanned = 0;);
      for(  internal_iterator it = low_water_it_cached;
            it != high_water_it_cached; 
            it++) {
        nClass += _eager_external_table[it->k] ? 1 : 0;
        LOGGING_ONLY(if(nScanned == 0) std::cout << "first scanned tuple id: " << it->k << std::endl;);
        LOGGING_ONLY(nScanned++;);
      }
      for(  internal_iterator it = high_water_it_cached;
            it != _itable.end(); 
            it++) {
        nClass++;
      }
            
      LOGGING_ONLY(std::cout << "[Eager:Hazy] scanned " << nScanned << " number " << nClass << std::endl;);
      break;
    }
  default:
    throw UnknownStrategyException(__FILE__, __LINE__, "[numInclass]");
  }
  LOGGING_ONLY(std::cout << "num in class " << c << " is: " << nClass << std::endl;);
}

---