Enumerations | |
enum | t_storage_manager { SM_HYBRID, SM_MAIN_MEMORY, SM_ONDISK } |
Functions | |
template<class T> | |
void | read_file_into_vectors (const char *filename, vector< int > &entity_ids, vector< T > &entity_vectors) |
Reads entities from the file. | |
template<class T> | |
int | getLabel (struct hazy_model &hm, T s) |
Classifies given entity vector with the given model and returns the class. | |
template<class T> | |
void | loadEntitiesToSMAndVectors (Storage_Manager< T > *sm, struct hazy_model &hm, const char *filename, vector< int > &entity_ids, vector< T > &entity_vectors) |
Reads entities from the file and load them into both storage manager and vectors. | |
template<class T> | |
void | testGetEntityClass (Storage_Manager< T > *sm, struct hazy_model &hm, vector< int > entity_ids, vector< T > entity_vectors, bool &method_successful) |
For all entities in the vector, gets their labels from storage manager and compare the classes retrieved by this class and check whether they are same or not. | |
template<class T> | |
void | testGetNumInClass (Storage_Manager< T > *sm, struct hazy_model &hm, vector< int > entity_ids, vector< T > entity_vectors, bool &method_successful) |
For all entities in the vector, gets their labels from storage manager and compare the classes retrieved by this class and check whether they are same or not Compares # of the entities in class 1 that come from storage manager and calculated in this class First, num in class is retrieved from storage manager and then for all entities label is calculated in this class. | |
template<class T> | |
void | test1 (Storage_Manager< T > *sm, const char *filename, struct hazy_model &hm, IncrementalSGD< T > *sgd, const char *training_file_name, bool &method_successful) |
Tests whether for all entities, we correctly get the class of the entity correctly. | |
template<class T> | |
void | test2 (Storage_Manager< T > *sm, const char *filename, struct hazy_model &hm, IncrementalSGD< T > *sgd, const char *training_file_name, bool &method_successful) |
Tests whether we correctly get the number in class. | |
template<class T> | |
void | test3 (Storage_Manager< T > *sm, const char *filename, struct hazy_model &hm, IncrementalSGD< T > *sgd, const char *training_file_name, bool &method_successful) |
Tests that incremental updates are working without any error. | |
template<class T> | |
void | test4 (Storage_Manager< T > *sm, const char *filename, struct hazy_model &hm, IncrementalSGD< T > *sgd, const char *training_file_name, bool &method_successful) |
Tests that resort at each step is working without any error. | |
template<class T> | |
void | test5 (Storage_Manager< T > *sm, const char *filename, struct hazy_model &hm, IncrementalSGD< T > *sgd, const char *training_file_name, bool &method_successful, bool isTest1) |
This method tests whether we successfully update and read together Tests that when we do incremental update at each step, then either get entity test or get num in class test works correctly (determined by isTest1 boolean) Meaning that whether we successfully read after each update or not. | |
template<class T> | |
void | test6 (Storage_Manager< T > *sm, const char *filename, struct hazy_model &hm, IncrementalSGD< T > *sgd, const char *training_file_name, bool &method_successful, bool isTest1) |
This method tests whether we successfully resort and read together Tests that when we do resort at each step, then either get entity test or get num in class test works correctly (determined by isTest1 boolean) Meaning that whether we successfully read after each resort or not. | |
void | copyModel (struct model &model, struct hazy_model &hm) |
Copies into model from hazy model. | |
void | holder_difference_models (struct model current, struct model old, double m_factor, double &delta_w, double &delta_b) |
Calculates delta differences from the given models. | |
template<class T> | |
void | test7 (Storage_Manager< T > *sm, const char *filename, struct hazy_model &hm, IncrementalSGD< T > *sgd, const char *training_file_name, bool &method_successful, bool isTest1) |
This method tests whether we successfully update and read together It simulates hazy algorithm by doing both resort and incremental update together. | |
template<class T> | |
void | run_test (Storage_Manager< T > *sm, struct hazy_model &hm, const char *filename, IncrementalSGD< T > *sgd, const char *training_file_name, bool &method_successful, int testNum) |
Executes the test with the given parameter. | |
template<class T> | |
void | prepare_ondisk_storage_manager (struct ondisk_storage_spec *s, IncrementalSGD< T > *sgd, const char *filename, const char *training_file_name, bool &method_successful, int testNum, struct hazy_model &hm, bool dense, Hazy_Database *db_conn) |
Creates ondisk storage manager. | |
template<class T> | |
void | prepare_hybrid_storage_manager (struct ondisk_storage_spec *s, IncrementalSGD< T > *sgd, const char *filename, const char *training_file_name, bool &method_successful, int testNum, struct hazy_model &hm, bool dense, Hazy_Database *db_conn, int nBuffer, bool useeps) |
Creates hybrid storage manager. | |
template<class T> | |
void | prepare_mm_storage_manager (struct main_memory_storage_spec *s, IncrementalSGD< T > *sgd, const char *filename, const char *training_file_name, bool &method_successful, int testNum, struct hazy_model &hm, Hazy_Database *db_conn) |
Creates main memory storage manager. | |
template<class T> | |
void | initialize (double lambda, string entity_table_name, string entity_file_name, string training_file_name, bool &method_successful, bool bDense, t_storage_manager storage_manager) |
Initializes the storage manager, spec and others required for the test. | |
int | main (int argc, char **argv) |
Variables | |
int | buffer_size = 10 |
enum t_storage_manager |
void copyModel | ( | struct model & | model, | |
struct hazy_model & | hm | |||
) |
Copies into model from hazy model.
[out] | model | model to be copied from given hazy model |
[in] | hm | hazy model |
int getLabel | ( | struct hazy_model & | hm, | |
T | s | |||
) |
Classifies given entity vector with the given model and returns the class.
[in] | hm | hazy model |
[in] | s | feature vector of the entity |
void holder_difference_models | ( | struct model | current, | |
struct model | old, | |||
double | m_factor, | |||
double & | delta_w, | |||
double & | delta_b | |||
) |
Calculates delta differences from the given models.
[in] | current | current model |
[in] | old | last model from the sort |
[in] | m_factor | m factor |
[out] | delta_w | delta value between w's |
[out] | delta_b | delta value between biases |
void initialize | ( | double | lambda, | |
string | entity_table_name, | |||
string | entity_file_name, | |||
string | training_file_name, | |||
bool & | method_successful, | |||
bool | bDense, | |||
t_storage_manager | storage_manager | |||
) |
Initializes the storage manager, spec and others required for the test.
[in] | lambda | |
[in] | entity_table_name | name of the table that contains entities |
[in] | entity_file_name | name of the file that contains entities |
[in] | training_file_name | name of the file that contains training examples |
[out] | method_successful | success |
[in] | bDense | whether feature vectors are dene or not |
[in] | storage_manager | storage manager |
void loadEntitiesToSMAndVectors | ( | Storage_Manager< T > * | sm, | |
struct hazy_model & | hm, | |||
const char * | filename, | |||
vector< int > & | entity_ids, | |||
vector< T > & | entity_vectors | |||
) |
Reads entities from the file and load them into both storage manager and vectors.
[in] | sm | storage manager |
[in] | hm | hazy model |
[in] | filename | name of the file that contains entities |
[out] | entity_ids | the vector for entity ids |
[out] | entity_vectors | the vector for entity vectors |
int main | ( | int | argc, | |
char ** | argv | |||
) |
void prepare_hybrid_storage_manager | ( | struct ondisk_storage_spec * | s, | |
IncrementalSGD< T > * | sgd, | |||
const char * | filename, | |||
const char * | training_file_name, | |||
bool & | method_successful, | |||
int | testNum, | |||
struct hazy_model & | hm, | |||
bool | dense, | |||
Hazy_Database * | db_conn, | |||
int | nBuffer, | |||
bool | useeps | |||
) |
Creates hybrid storage manager.
[in] | s | ondisk storage spec |
[in] | sgd | sgd model |
[in] | filename | name of the file that contains entities |
[in] | training_file_name | name of the file that contains training examples |
[out] | method_successful | success |
[in] | testNum | which test to execute |
[in] | hm | hazy model |
[in] | dense | whether feature vectors are dense or not |
[in] | db_conn | database connection |
[in] | nBuffer | buffer size |
[in] | useeps | whether to use eps map or not |
void prepare_mm_storage_manager | ( | struct main_memory_storage_spec * | s, | |
IncrementalSGD< T > * | sgd, | |||
const char * | filename, | |||
const char * | training_file_name, | |||
bool & | method_successful, | |||
int | testNum, | |||
struct hazy_model & | hm, | |||
Hazy_Database * | db_conn | |||
) |
Creates main memory storage manager.
[in] | s | main memory storage spec |
[in] | sgd | sgd model |
[in] | filename | name of the file that contains entities |
[in] | training_file_name | name of the file that contains training examples |
[out] | method_successful | success |
[in] | testNum | which test to execute |
[in] | hm | hazy model |
[in] | db_conn | database connection |
void prepare_ondisk_storage_manager | ( | struct ondisk_storage_spec * | s, | |
IncrementalSGD< T > * | sgd, | |||
const char * | filename, | |||
const char * | training_file_name, | |||
bool & | method_successful, | |||
int | testNum, | |||
struct hazy_model & | hm, | |||
bool | dense, | |||
Hazy_Database * | db_conn | |||
) |
Creates ondisk storage manager.
[in] | s | ondisk storage spec |
[in] | sgd | sgd model |
[in] | filename | name of the file that contains entities |
[in] | training_file_name | name of the file that contains training examples |
[out] | method_successful | success |
[in] | testNum | which test to execute |
[in] | hm | hazy model |
[in] | dense | whether feature vectors are dense or not |
[in] | db_conn | database connection |
void read_file_into_vectors | ( | const char * | filename, | |
vector< int > & | entity_ids, | |||
vector< T > & | entity_vectors | |||
) |
Reads entities from the file.
[in] | filename | name of the file that contains entities |
[out] | entity_ids | the vector that contains entity ids |
[out] | entity_vectors | the vector that contains entity vectors |
void run_test | ( | Storage_Manager< T > * | sm, | |
struct hazy_model & | hm, | |||
const char * | filename, | |||
IncrementalSGD< T > * | sgd, | |||
const char * | training_file_name, | |||
bool & | method_successful, | |||
int | testNum | |||
) |
Executes the test with the given parameter.
[in] | sm | storage manager |
[in] | hm | hazy model |
[in] | filename | name of the file that contains the entities |
[in] | sgd | sgd model |
[in] | training_file_name | name of the training file |
[out] | method_successful | success |
[in] | testNum | which test to execute |
void test1 | ( | Storage_Manager< T > * | sm, | |
const char * | filename, | |||
struct hazy_model & | hm, | |||
IncrementalSGD< T > * | sgd, | |||
const char * | training_file_name, | |||
bool & | method_successful | |||
) |
Tests whether for all entities, we correctly get the class of the entity correctly.
[in] | sm | storage manager |
[in] | filename | name of the file that contains entities |
[in] | hm | hazy model |
[in] | sgd | sgd model |
[in] | training_file_name | name of the file that contains training examples |
[out] | method_successful | success |
void test2 | ( | Storage_Manager< T > * | sm, | |
const char * | filename, | |||
struct hazy_model & | hm, | |||
IncrementalSGD< T > * | sgd, | |||
const char * | training_file_name, | |||
bool & | method_successful | |||
) |
Tests whether we correctly get the number in class.
[in] | sm | storage manager |
[in] | filename | name of the file that contains entities |
[in] | hm | hazy model |
[in] | sgd | sgd model |
[in] | training_file_name | name of the file that contains training examples |
[out] | method_successful | success |
void test3 | ( | Storage_Manager< T > * | sm, | |
const char * | filename, | |||
struct hazy_model & | hm, | |||
IncrementalSGD< T > * | sgd, | |||
const char * | training_file_name, | |||
bool & | method_successful | |||
) |
Tests that incremental updates are working without any error.
[in] | sm | storage manager |
[in] | filename | name of the file that contains entities |
[in] | hm | hazy model |
[in] | sgd | sgd model |
[in] | training_file_name | name of the file that contains training examples |
[out] | method_successful | success |
void test4 | ( | Storage_Manager< T > * | sm, | |
const char * | filename, | |||
struct hazy_model & | hm, | |||
IncrementalSGD< T > * | sgd, | |||
const char * | training_file_name, | |||
bool & | method_successful | |||
) |
Tests that resort at each step is working without any error.
[in] | sm | storage manager |
[in] | filename | name of the file that contains entities |
[in] | hm | hazy model |
[in] | sgd | sgd model |
[in] | training_file_name | name of the file that contains training examples |
[out] | method_successful | success |
void test5 | ( | Storage_Manager< T > * | sm, | |
const char * | filename, | |||
struct hazy_model & | hm, | |||
IncrementalSGD< T > * | sgd, | |||
const char * | training_file_name, | |||
bool & | method_successful, | |||
bool | isTest1 | |||
) |
This method tests whether we successfully update and read together Tests that when we do incremental update at each step, then either get entity test or get num in class test works correctly (determined by isTest1 boolean) Meaning that whether we successfully read after each update or not.
[in] | sm | storage manager |
[in] | filename | name of the file that contains entities |
[in] | hm | hazy model |
[in] | sgd | sgd model |
[in] | training_file_name | name of the file that contains training examples |
[out] | method_successful | success |
[in] | isTest1 | if this is true, get entity class is tested, else get num in class is tested |
void test6 | ( | Storage_Manager< T > * | sm, | |
const char * | filename, | |||
struct hazy_model & | hm, | |||
IncrementalSGD< T > * | sgd, | |||
const char * | training_file_name, | |||
bool & | method_successful, | |||
bool | isTest1 | |||
) |
This method tests whether we successfully resort and read together Tests that when we do resort at each step, then either get entity test or get num in class test works correctly (determined by isTest1 boolean) Meaning that whether we successfully read after each resort or not.
[in] | sm | storage manager |
[in] | filename | name of the file that contains entities |
[in] | hm | hazy model |
[in] | sgd | sgd model |
[in] | training_file_name | name of the file that contains training examples |
[out] | method_successful | success |
[in] | isTest1 | if this is true, get entity class is tested, else get num in class is tested |
void test7 | ( | Storage_Manager< T > * | sm, | |
const char * | filename, | |||
struct hazy_model & | hm, | |||
IncrementalSGD< T > * | sgd, | |||
const char * | training_file_name, | |||
bool & | method_successful, | |||
bool | isTest1 | |||
) |
This method tests whether we successfully update and read together It simulates hazy algorithm by doing both resort and incremental update together.
However, instead of skiing strategy, it does resort at every 5 step and incremental update otherwise. Tests that when we do incremental update or resort at each step, then either get entity test or get num in class test works correctly (determined by isTest1 boolean) Meaning that whether we successfully read after each update or not.
[in] | sm | storage manager |
[in] | filename | name of the file that contains entities |
[in] | hm | hazy model |
[in] | sgd | sgd model |
[in] | training_file_name | name of the file that contains training examples |
[out] | method_successful | success |
[in] | isTest1 | if this is true, get entity class is tested, else get num in class is tested |
void testGetEntityClass | ( | Storage_Manager< T > * | sm, | |
struct hazy_model & | hm, | |||
vector< int > | entity_ids, | |||
vector< T > | entity_vectors, | |||
bool & | method_successful | |||
) |
For all entities in the vector, gets their labels from storage manager and compare the classes retrieved by this class and check whether they are same or not.
[in] | sm | storage manager |
[in] | hm | hazy model |
[in] | entity_ids | id of the entities |
[in] | entity_vectors | vectors of the entities |
[out] | method_successful | success |
void testGetNumInClass | ( | Storage_Manager< T > * | sm, | |
struct hazy_model & | hm, | |||
vector< int > | entity_ids, | |||
vector< T > | entity_vectors, | |||
bool & | method_successful | |||
) |
For all entities in the vector, gets their labels from storage manager and compare the classes retrieved by this class and check whether they are same or not Compares # of the entities in class 1 that come from storage manager and calculated in this class First, num in class is retrieved from storage manager and then for all entities label is calculated in this class.
Then, we find the num in class and we compare it with the result of storage manager
[in] | sm | storage manager |
[in] | hm | hazy model |
[in] | entity_ids | id of the entities |
[in] | entity_vectors | vectors of the entities |
[out] | method_successful | success |
int buffer_size = 10 |