Class to implement various performance measures.
based on: Fawcett, T: March 2004, ROC Graphs: Notes and Practical Considerations for Researchers and input from Sonnenburg, S: Feburary 2008, various discussions
Definition at line 45 of file PerformanceMeasures.h.
Public Member Functions | |
CPerformanceMeasures () | |
CPerformanceMeasures (CLabels *true_labels, CLabels *output) | |
virtual | ~CPerformanceMeasures () |
bool | set_true_labels (CLabels *true_labels) |
CLabels * | get_true_labels () const |
bool | set_output (CLabels *output) |
CLabels * | get_output () const |
int32_t | get_num_labels () const |
void | get_ROC (float64_t **result, int32_t *num, int32_t *dim) |
float64_t | get_auROC () |
float64_t | get_aoROC () |
void | get_PRC (float64_t **result, int32_t *num, int32_t *dim) |
float64_t | get_auPRC () |
float64_t | get_aoPRC () |
void | get_DET (float64_t **result, int32_t *num, int32_t *dim) |
float64_t | get_auDET () |
float64_t | get_aoDET () |
void | get_all_accuracy (float64_t **result, int32_t *num, int32_t *dim) |
float64_t | get_accuracy (float64_t threshold=0) |
void | get_all_error (float64_t **result, int32_t *num, int32_t *dim) |
float64_t | get_error (float64_t threshold=0) |
void | get_all_fmeasure (float64_t **result, int32_t *num, int32_t *dim) |
float64_t | get_fmeasure (float64_t threshold=0) |
void | get_all_CC (float64_t **result, int32_t *num, int32_t *dim) |
float64_t | get_CC (float64_t threshold=0) |
void | get_all_WRAcc (float64_t **result, int32_t *num, int32_t *dim) |
float64_t | get_WRAcc (float64_t threshold=0) |
void | get_all_BAL (float64_t **result, int32_t *num, int32_t *dim) |
float64_t | get_BAL (float64_t threshold=0) |
virtual const char * | get_name () const |
Protected Member Functions | |
void | init_nolabels () |
float64_t | trapezoid_area (float64_t x1, float64_t x2, float64_t y1, float64_t y2) |
void | create_sortedROC () |
void | compute_ROC (float64_t **result) |
void | compute_accuracy (float64_t **result, int32_t *num, int32_t *dim, bool do_error=false) |
void | compute_PRC (float64_t **result) |
void | compute_DET (float64_t **result) |
void | compute_confusion_matrix (float64_t threshold, int32_t *tp, int32_t *fp, int32_t *fn, int32_t *tn) |
Protected Attributes | |
CLabels * | m_true_labels |
CLabels * | m_output |
int32_t | m_num_labels |
int32_t | m_all_true |
int32_t | m_all_false |
int32_t * | m_sortedROC |
float64_t | m_auROC |
float64_t | m_auPRC |
float64_t | m_auDET |
default constructor
Definition at line 18 of file PerformanceMeasures.cpp.
CPerformanceMeasures | ( | CLabels * | true_labels, | |
CLabels * | output | |||
) |
constructor
true_labels | true labels as seen in real world | |
output | output labels/hypothesis from a classifier |
Definition at line 27 of file PerformanceMeasures.cpp.
~CPerformanceMeasures | ( | ) | [virtual] |
Definition at line 39 of file PerformanceMeasures.cpp.
void compute_accuracy | ( | float64_t ** | result, | |
int32_t * | num, | |||
int32_t * | dim, | |||
bool | do_error = false | |||
) | [protected] |
compute ROC accuracy/error
result | where the result will be stored | |
num | number of points | |
dim | dimension == 2 (output, accuracy/error) | |
do_error | if error instead of accuracy shall be computed |
Definition at line 385 of file PerformanceMeasures.cpp.
void compute_confusion_matrix | ( | float64_t | threshold, | |
int32_t * | tp, | |||
int32_t * | fp, | |||
int32_t * | fn, | |||
int32_t * | tn | |||
) | [protected] |
compute confusion matrix
caller has to delete[]
threshold | threshold to compute against | |
tp | storage of true positives or NULL if unused | |
fp | storage of false positives or NULL if unused | |
fn | storage of false negatives or NULL if unused | |
tn | storage of true negatives or NULL if unused |
Definition at line 148 of file PerformanceMeasures.cpp.
void compute_DET | ( | float64_t ** | result | ) | [protected] |
compute DET points and auDET
result | where the result will be stored |
Definition at line 332 of file PerformanceMeasures.cpp.
void compute_PRC | ( | float64_t ** | result | ) | [protected] |
compute PRC points and auPRC
result | where the result will be stored |
Definition at line 282 of file PerformanceMeasures.cpp.
void compute_ROC | ( | float64_t ** | result | ) | [protected] |
compute ROC points and auROC
Definition at line 208 of file PerformanceMeasures.cpp.
void create_sortedROC | ( | ) | [protected] |
create index for ROC sorting
Definition at line 119 of file PerformanceMeasures.cpp.
get classifier's accuracy at given threshold
threshold | all values below are considered negative examples (default 0) |
Definition at line 373 of file PerformanceMeasures.cpp.
void get_all_accuracy | ( | float64_t ** | result, | |
int32_t * | num, | |||
int32_t * | dim | |||
) |
get classifier's accuracies (swig compatible)
accuracy = (true positives + true negatives) / all labels
caller has to free
result | storage of accuracies in 2 dim array: (output, accuracy), sorted by output | |
num | number of accuracy points | |
dim | dimension == 2 (output, accuracy) |
Definition at line 413 of file PerformanceMeasures.cpp.
void get_all_BAL | ( | float64_t ** | result, | |
int32_t * | num, | |||
int32_t * | dim | |||
) |
get classifier's Balanced Error (swig compatible)
BAL = 0.5 * ( true positives / all true labels + true negatives / all false labels )
caller has to free
result | storage of BAL in 2 dim array: (output, BAL), sorted by output | |
num | number of BAL points | |
dim | dimension == 2 (output, BAL) |
Definition at line 585 of file PerformanceMeasures.cpp.
void get_all_CC | ( | float64_t ** | result, | |
int32_t * | num, | |||
int32_t * | dim | |||
) |
get classifier's Cross Correlation coefficients (swig compatible)
CC = ( true positives * true negatives
also check http://en.wikipedia.org/wiki/Correlation
caller has to free
result | storage of CCs in 2 dim array: (output, CC), sorted by output | |
num | number of CC points | |
dim | dimension == 2 (output, CC) |
Definition at line 494 of file PerformanceMeasures.cpp.
void get_all_error | ( | float64_t ** | result, | |
int32_t * | num, | |||
int32_t * | dim | |||
) |
get classifier's error rates (swig compatible)
value is 1 - accuracy
caller has to free
result | storage of errors in 2 dim array: (output, error), sorted by output | |
num | number of accuracy points | |
dim | dimension == 2 (output, error) |
Definition at line 419 of file PerformanceMeasures.cpp.
void get_all_fmeasure | ( | float64_t ** | result, | |
int32_t * | num, | |||
int32_t * | dim | |||
) |
get classifier's F-measure (swig compatible)
F-measure = 2 / (1 / precision + 1 / recall)
caller has to free
result | storage of F-measure in 2 dim array (output, fmeasure), sorted by output | |
num | number of accuracy points | |
dim | dimension == 2 (output, fmeasure) |
Definition at line 456 of file PerformanceMeasures.cpp.
void get_all_WRAcc | ( | float64_t ** | result, | |
int32_t * | num, | |||
int32_t * | dim | |||
) |
get classifier's Weighted Relative Accuracy (swig compatible)
WRAcc = ( true positives / (true positives + false negatives) )
caller has to free
result | storage of WRAccs in 2 dim array: (output, WRAcc), sorted by output | |
num | number of WRAcc points | |
dim | dimension == 2 (output, WRAcc) |
Definition at line 541 of file PerformanceMeasures.cpp.
float64_t get_aoDET | ( | ) |
return area over Detection Error Tradeoff curve
value is 1 - auDET
Definition at line 223 of file PerformanceMeasures.h.
float64_t get_aoPRC | ( | ) |
return area over Precision Recall Curve
value is 1 - auPRC
Definition at line 180 of file PerformanceMeasures.h.
float64_t get_aoROC | ( | ) |
return area over Reveiver Operating Curve
value is 1 - auROC
Definition at line 138 of file PerformanceMeasures.h.
float64_t get_auDET | ( | ) |
return area under Detection Error Tradeoff curve
calculated by adding trapezoids
Definition at line 205 of file PerformanceMeasures.h.
float64_t get_auPRC | ( | ) |
return area under Precision Recall Curve
calculated by adding trapezoids
Definition at line 163 of file PerformanceMeasures.h.
float64_t get_auROC | ( | ) |
return area under Receiver Operating Curve
calculated by adding trapezoids
Definition at line 120 of file PerformanceMeasures.h.
get classifier's Balanced Error at threshold 0
Definition at line 569 of file PerformanceMeasures.cpp.
get classifier's Cross Correlation coefficient at threshold
Definition at line 480 of file PerformanceMeasures.cpp.
void get_DET | ( | float64_t ** | result, | |
int32_t * | num, | |||
int32_t * | dim | |||
) |
get Detection Error Tradeoff curve for previously given labels (swig compatible)
DET point = false positives / all false labels, false negatives / all false labels
caller has to free
result | where computed DET values will be stored | |
num | number of labels/examples | |
dim | dimension == 2 (false positive rate, false negative rate) |
Definition at line 323 of file PerformanceMeasures.cpp.
get classifier's error at threshold
value is 1 - accuracy0
threshold | all values below are considered negative examples (default 0) |
Definition at line 270 of file PerformanceMeasures.h.
get classifier's F-measure at threshold 0
Definition at line 427 of file PerformanceMeasures.cpp.
virtual const char* get_name | ( | void | ) | const [virtual] |
get the name of tghe object
Implements CSGObject.
Definition at line 383 of file PerformanceMeasures.h.
int32_t get_num_labels | ( | ) | const |
get number of labels in output/true labels
Definition at line 98 of file PerformanceMeasures.h.
CLabels* get_output | ( | ) | const |
get classifier's output labels/hypothesis
Definition at line 88 of file PerformanceMeasures.h.
void get_PRC | ( | float64_t ** | result, | |
int32_t * | num, | |||
int32_t * | dim | |||
) |
get Precision Recall Curve for previously given labels (swig compatible)
PRC point = true positives / all true labels, true positives / (true positives + false positives)
caller has to free
result | where computed ROC values will be stored | |
num | number of labels/examples | |
dim | dimension == 2 (recall, precision) |
Definition at line 273 of file PerformanceMeasures.cpp.
void get_ROC | ( | float64_t ** | result, | |
int32_t * | num, | |||
int32_t * | dim | |||
) |
get Receiver Operating Curve for previously given labels (swig compatible)
ROC point = false positives / all false labels, true positives / all true labels
caller has to free
result | where computed ROC values will be stored | |
num | number of labels/examples | |
dim | dimensions == 2 (false positive rate, true positive rate) |
Definition at line 200 of file PerformanceMeasures.cpp.
CLabels* get_true_labels | ( | ) | const |
get true labels as seen in real world
Definition at line 71 of file PerformanceMeasures.h.
get classifier's Weighted Relative Accuracy at threshold 0
Definition at line 522 of file PerformanceMeasures.cpp.
void init_nolabels | ( | ) | [protected] |
initialise values independent from true labels/output
Definition at line 53 of file PerformanceMeasures.cpp.
bool set_output | ( | CLabels * | output | ) |
set output labels/hypothesis from a classifier
output | output labels |
Definition at line 103 of file PerformanceMeasures.cpp.
bool set_true_labels | ( | CLabels * | true_labels | ) |
set true labels as seen in real world
true_labels | true labels |
Definition at line 63 of file PerformanceMeasures.cpp.
calculate trapezoid area for auROC
x1 | x coordinate of point 1 | |
x2 | x coordinate of point 2 | |
y1 | y coordinate of point 1 | |
y2 | y coordinate of point 2 |
Definition at line 136 of file PerformanceMeasures.cpp.
int32_t m_all_false [protected] |
number of negative examples in true_labels
Definition at line 456 of file PerformanceMeasures.h.
int32_t m_all_true [protected] |
number of positive examples in true_labels
Definition at line 454 of file PerformanceMeasures.h.
area under DET; 1 - area over DET
Definition at line 466 of file PerformanceMeasures.h.
area under PRC; 1 - area over PRC
Definition at line 464 of file PerformanceMeasures.h.
area under ROC; 1 - area over ROC
Definition at line 462 of file PerformanceMeasures.h.
int32_t m_num_labels [protected] |
number of true labels/outputs/accuracies/ROC points
Definition at line 451 of file PerformanceMeasures.h.
output labels/hypothesis from a classifier
Definition at line 449 of file PerformanceMeasures.h.
int32_t* m_sortedROC [protected] |
array of size num_labels with indices of true_labels/output sorted to fit ROC algorithm
Definition at line 460 of file PerformanceMeasures.h.
CLabels* m_true_labels [protected] |
true labels/examples as seen in real world
Definition at line 447 of file PerformanceMeasures.h.