incremental_sgd.hxx

Go to the documentation of this file.

//#include "incremental_sgd.h"
// This file is included in the header function.
#define HINGELOSS 1
#define SMOOTHHINGELOSS 2
#define SQUAREDHINGELOSS 3
#define LOGLOSS 10
#define LOGLOSSMARGIN 11

#define LOSS HINGELOSS

#define BIAS 1


inline
double loss(double z)
{
#if LOSS == LOGLOSS
  if (z > 18)
    return exp(-z);
  if (z < -18)
    return -z;
  return log(1+exp(-z));
#elif LOSS == LOGLOSSMARGIN
  if (z > 18)
    return exp(1-z);
  if (z < -18)
    return 1-z;
  return log(1+exp(1-z));
#elif LOSS == SMOOTHHINGELOSS
  if (z < 0)
    return 0.5 - z;
  if (z < 1)
    return 0.5 * (1-z) * (1-z);
  return 0;
#elif LOSS == SQUAREDHINGELOSS
  if (z < 1)
    return 0.5 * (1 - z) * (1 - z);
  return 0;
#elif LOSS == HINGELOSS
  if (z < 1)
    return 1 - z;
  return 0;
#else
# error "Undefined loss"
#endif
}


inline 
double dloss(double z)
{ 
#if LOSS == LOGLOSS
  if (z > 18)
    return exp(-z);
  if (z < -18)
    return 1;
  return 1 / (exp(z) + 1);
#elif LOSS == LOGLOSSMARGIN
  if (z > 18)
    return exp(1-z);
  if (z < -18)
    return 1;
  return 1 / (exp(z-1) + 1);
#elif LOSS == SMOOTHHINGELOSS
  if (z < 0)
    return 1;
  if (z < 1)
    return 1-z;
  return 0;
#elif LOSS == SQUAREDHINGELOSS
  if (z < 1)
    return (1 - z);
  return 0;
#else
  if (z < 1)
    return 1;
  return 0;
#endif
}

template<class T>
IncrementalSGD<T>::
IncrementalSGD(int dim, double l) : lambda(l),_m(dim) { 
  // Shift t in order to have a
  // reasonable initial learning rate.
  // This assumes |x| \approx 1.
  double maxw = 1.0 / sqrt(lambda);
  double typw = sqrt(maxw);
  double eta0 = typw / std::max(1.0,dloss(-typw));
  _m.t = 1 / (eta0 * lambda);    
} 

template<class T>
void
IncrementalSGD<T>::resetModel() {
  _m.resetModel();
  double maxw = 1.0 / sqrt(lambda);
  double typw = sqrt(maxw);
  double eta0 = typw / std::max(1.0,dloss(-typw));
  _m.t = 1 / (eta0 * lambda);    
}

template<class T>
void
IncrementalSGD<T>::oneStep(double y, T x) {
  double eta = 1.0 / (lambda * _m.t);
  double s = 1 - eta * lambda;
  _m.wscale *= s;
  if (_m.wscale < 1e-9) {
    _m.w.scale(_m.wscale);
    _m.wscale = 1;
  }
  double wx = dot(_m.w,x) * _m.wscale;
  double z = y * (wx + _m.bias);
#if LOSS < LOGLOSS
  if (z < 1)
#endif
    {
      double etd = eta * dloss(z);
      _m.w.add(x, etd * y / _m.wscale);
#if BIAS
      // Slower rate on the bias because
      // it learns at each iteration.
      _m.bias += etd * y * 0.01;
#endif
    }
  _m.t += 1;
}

template<class T>
bool
IncrementalSGD<T>::classifyExample(T x) {
  double wx = dot(_m.w,x) * _m.wscale;
  double product = wx + _m.bias;
  return product > 0;
}


// TrueSGD
template<class T>
bool
TrueSGD<T>::addExample(int y, T ex) { oneStep(y,ex); return true; }

template<class T>
bool 
ReservoirSGD<T>::addExample(int y, T ex)
{ 
  _r.addExample(y,ex);
  if(!_r.isFull()) { return false; }      
  // otherwise we have work to do.
  for(int i = 0; i < RESERVOIR_ITERATIONS; i ++) {
    for(int j = 0; j < _r.getCurrentSize(); j ++) {
      oneStep(_r.getExampleClass(j), _r.getExampleFeatureVector(j));        
    }
  }
  
  return true;
}

---