% knn_demo.m
% Explain the K Nearest Neighbor (KNN) classification algorithm
% Dr. David Pan, UAH

N = 10;     % Number of samples per class
% X is the data array, C is the class label vector
[X, C] = dataset_generate(N);
% Plot the data samples of the two classes
figure;
gscatter(X(:,1),X(:,2), C, 'rb','ox');
axis square
axis equal

K = 5;  % Look at the K nearest neighbors 

L = length(X);

Xin = mean(X)      % An example input query vector

% Calculate the Euclidean distances
Dist = zeros(1, L);
for i = 1:L
    Dist(i) = norm(X(i,:) - Xin);
end

% Sort the distances
[D_sorted, Index] = sort(Dist);
% First 5 values of the index vector: Index[1:5]

% 9    10    17     2    15

% Look at the first K classes
Classes = C(Index(1:K));
% Classes = 1, 1, 2, 1, 2 
% Posterior Prob. for Class 1: 3/5 = 60%

% The predicted label for the input vector Xin (majority vote)
label = mode(Classes)
% When there are multiple values occurring equally frequently, mode 
% returns the smallest of those values (tie breaker)

% Using Matlab toolbox
Model = fitcknn(X, C, 'NumNeighbors',K);
Model.Prior
%ans =
%    0.5000    0.5000
[label,score,cost] = predict(Model,Xin)

% With preprocessing (data normalization)
% Plot the standardized samples after normalization
X_std_1 = zeros(L, 2);
X_std = zeros(L, 2);

Mu = zeros(1,2);
Std = zeros(1,2);

for i = 1:2
    Mu(i) = mean(X(:,i));
    Std(i) = std(X(:,i));
    X_std_1(:,i) = (X(:,i) - Mu(i))/Std(i);
end

% Alternatively, use the function
X_std = normalize(X);

isequal(X_std, X_std_1)

figure;
gscatter(X_std(:,1),X_std(:,2),C,'rb','ox')
axis square
axis equal

% Normalize the Xin
for i = 1:2
    Xin_std(:,i) = (Xin(:,i) - Mu(i))/Std(i);
end

hold on;
plot(Xin_std(1), Xin_std(2), 'gd');

% Repeat the KNN algorithm 

% Calculate the Euclidean distances
Dist_std = zeros(1, L);
for i = 1:L
    Dist_std(i) = norm(X_std(i,:) - Xin_std);
end

% Sort the distances
[D_sorted_std, Index_std] = sort(Dist_std);
% First 5 values of the index vector are different from above
%Index_std(1:5)
%ans =
%     9    10    15     2    17

% Look at the first K classes
Classes_std = C(Index_std(1:K));

% The predicted label for the input vector Xin (majority vote)
label_std = mode(Classes_std)

% Using Matlab toolbox
Model_std = fitcknn(X, C, 'NumNeighbors',K, 'Standardize', 1);
Mu
Model_std.Mu
Std
Model_std.Sigma
[label,score,cost] = predict(Model_std,Xin)