Label Smoothing

1
#include <iostream>
2
#include <vector>
3
#include <stdexcept>
4
#include <iomanip>
5
6
// Create a smoothed label for a K-class problem.
7
// y_ls = (1 - epsilon) * one_hot + (epsilon / K) * 1
8
std::vector<double> smoothLabel(int true_class, int K, double epsilon) {
9
    if (K <= 1) throw std::invalid_argument("K must be > 1");
10
    if (true_class < 0 || true_class >= K) throw std::out_of_range("true_class out of range");
11
    if (epsilon < 0.0 || epsilon >= 1.0) throw std::invalid_argument("epsilon must be in [0,1)");
12
13
    std::vector<double> y(K, epsilon / K); // start with uniform mass epsilon/K
14
    y[true_class] += (1.0 - epsilon);      // add (1 - epsilon) to the true class
15
    return y;
16
}
17
18
// Batch version: produce smoothed labels for each example in a mini-batch
19
std::vector<std::vector<double>> smoothBatch(const std::vector<int>& labels, int K, double epsilon) {
20
    std::vector<std::vector<double>> Y;
21
    Y.reserve(labels.size());
22
    for (int t : labels) {
23
        Y.push_back(smoothLabel(t, K, epsilon));
24
    }
25
    return Y;
26
}
27
28
int main() {
29
    int K = 5;
30
    int true_class = 2; // 0-based index
31
    double epsilon = 0.1;
32
33
    auto y = smoothLabel(true_class, K, epsilon);
34
    std::cout << std::fixed << std::setprecision(4);
35
    std::cout << "Smoothed label (single): ";
36
    for (double v : y) std::cout << v << ' ';
37
    std::cout << "\n";
38
39
    std::vector<int> batch_labels = {2, 0, 4};
40
    auto Y = smoothBatch(batch_labels, K, epsilon);
41
    std::cout << "Batch smoothed labels:\n";
42
    for (const auto& row : Y) {
43
        for (double v : row) std::cout << v << ' ';
44
        std::cout << '\n';
45
    }
46
47
    return 0;
48
}
49

1
#include <iostream>
2
#include <vector>
3
#include <cmath>
4
#include <stdexcept>
5
#include <iomanip>
6
7
// Compute a numerically stable softmax of logits.
8
std::vector<double> softmax(const std::vector<double>& z) {
9
    if (z.empty()) throw std::invalid_argument("logits vector is empty");
10
    double max_z = z[0];
11
    for (double v : z) if (v > max_z) max_z = v; // for stability
12
    double sum_exp = 0.0;
13
    std::vector<double> p(z.size());
14
    for (size_t i = 0; i < z.size(); ++i) {
15
        p[i] = std::exp(z[i] - max_z);
16
        sum_exp += p[i];
17
    }
18
    for (double& v : p) v /= sum_exp;
19
    return p;
20
}
21
22
std::vector<double> smoothLabel(int true_class, int K, double epsilon) {
23
    if (K <= 1) throw std::invalid_argument("K must be > 1");
24
    if (true_class < 0 || true_class >= K) throw std::out_of_range("true_class out of range");
25
    if (epsilon < 0.0 || epsilon >= 1.0) throw std::invalid_argument("epsilon must be in [0,1)");
26
    std::vector<double> y(K, epsilon / K);
27
    y[true_class] += (1.0 - epsilon);
28
    return y;
29
}
30
31
// Cross-entropy with label smoothing: L = -sum_i y_ls[i] * log p[i]
32
// Takes raw logits, applies softmax, then computes the loss.
33
double crossEntropySmoothedFromLogits(const std::vector<double>& logits, int true_class, double epsilon) {
34
    auto p = softmax(logits);
35
    auto y_ls = smoothLabel(true_class, static_cast<int>(p.size()), epsilon);
36
37
    const double eps = 1e-15; // clamp to avoid log(0)
38
    double loss = 0.0;
39
    for (size_t i = 0; i < p.size(); ++i) {
40
        double pi = std::min(std::max(p[i], eps), 1.0 - eps);
41
        loss += -y_ls[i] * std::log(pi);
42
    }
43
    return loss;
44
}
45
46
// For comparison: standard cross-entropy with hard label
47
double crossEntropyHardFromLogits(const std::vector<double>& logits, int true_class) {
48
    auto p = softmax(logits);
49
    const double eps = 1e-15;
50
    double pt = std::min(std::max(p[true_class], eps), 1.0 - eps);
51
    return -std::log(pt);
52
}
53
54
int main() {
55
    std::vector<double> logits = {1.2, -0.3, 2.1, 0.0}; // example scores
56
    int true_class = 2; // correct class index
57
    double epsilon = 0.1;
58
59
    double L_hard = crossEntropyHardFromLogits(logits, true_class);
60
    double L_smooth = crossEntropySmoothedFromLogits(logits, true_class, epsilon);
61
62
    std::cout << std::fixed << std::setprecision(6);
63
    std::cout << "Hard-label CE:    " << L_hard << "\n";
64
    std::cout << "Smoothed-label CE:" << L_smooth << "\n";
65
66
    return 0;
67
}
68

1
#include <iostream>
2
#include <vector>
3
#include <cmath>
4
#include <iomanip>
5
6
std::vector<double> softmax(const std::vector<double>& z) {
7
    double max_z = z[0];
8
    for (double v : z) if (v > max_z) max_z = v;
9
    double sum_exp = 0.0;
10
    std::vector<double> p(z.size());
11
    for (size_t i = 0; i < z.size(); ++i) {
12
        p[i] = std::exp(z[i] - max_z);
13
        sum_exp += p[i];
14
    }
15
    for (double &v : p) v /= sum_exp;
16
    return p;
17
}
18
19
std::vector<double> smoothLabel(int true_class, int K, double epsilon) {
20
    std::vector<double> y(K, epsilon / K);
21
    y[true_class] += (1.0 - epsilon);
22
    return y;
23
}
24
25
// Gradient wrt logits z: grad = p - y_ls
26
std::vector<double> gradLogitsLabelSmoothed(const std::vector<double>& logits, int true_class, double epsilon) {
27
    auto p = softmax(logits);
28
    auto y_ls = smoothLabel(true_class, static_cast<int>(p.size()), epsilon);
29
    std::vector<double> g(p.size());
30
    for (size_t i = 0; i < p.size(); ++i) g[i] = p[i] - y_ls[i];
31
    return g;
32
}
33
34
int main() {
35
    std::vector<double> logits = {2.0, 0.5, -1.0};
36
    int true_class = 0;
37
    double epsilon = 0.1;
38
39
    auto g_smooth = gradLogitsLabelSmoothed(logits, true_class, epsilon);
40
41
    // For comparison, gradient with hard labels: p - y (where y is one-hot)
42
    auto p = softmax(logits);
43
    std::vector<double> g_hard = p;
44
    g_hard[true_class] -= 1.0;
45
46
    std::cout << std::fixed << std::setprecision(6);
47
    std::cout << "Gradient (hard):   ";
48
    for (double v : g_hard) std::cout << v << ' ';
49
    std::cout << "\n";
50
51
    std::cout << "Gradient (smooth): ";
52
    for (double v : g_smooth) std::cout << v << ' ';
53
    std::cout << "\n";
54
55
    return 0;
56
}
57