Gradient Clipping & Normalization

1
#include <bits/stdc++.h>
2
using namespace std;
3
4
// Clamp each component to [-c, c]
5
vector<double> clipByValue(const vector<double>& g, double c) {
6
    vector<double> out = g;
7
    for (double &x : out) {
8
        // Note: copying then modifying ensures original is unchanged
9
    }
10
    for (size_t i = 0; i < out.size(); ++i) {
11
        if (out[i] > c) out[i] = c;
12
        else if (out[i] < -c) out[i] = -c;
13
        // else unchanged
14
    }
15
    return out;
16
}
17
18
// Rescale the entire vector so that its L2 norm <= c (direction preserved)
19
vector<double> clipByNorm(const vector<double>& g, double c, double eps = 1e-12) {
20
    double sqsum = 0.0;
21
    for (double x : g) sqsum += x * x;
22
    double norm = sqrt(sqsum);
23
    double scale = 1.0;
24
    if (norm > c) scale = c / (norm + eps); // If already small, keep scale=1
25
    vector<double> out(g.size());
26
    for (size_t i = 0; i < g.size(); ++i) out[i] = g[i] * scale;
27
    return out;
28
}
29
30
void printVec(const string& name, const vector<double>& v) {
31
    cout << name << ": [";
32
    for (size_t i = 0; i < v.size(); ++i) {
33
        cout << fixed << setprecision(4) << v[i] << (i + 1 == v.size() ? "" : ", ");
34
    }
35
    cout << "]\n";
36
}
37
38
int main() {
39
    vector<double> g = {20.0, -0.5, 100.0, -7.2, 0.01};
40
    double c_val = 5.0;  // value clipping threshold
41
    double c_norm = 3.0; // norm clipping threshold
42
43
    auto gv = clipByValue(g, c_val);
44
    auto gn = clipByNorm(g, c_norm);
45
46
    // Compute norms for display
47
    auto l2 = [](const vector<double>& v){ double s=0; for(double x: v) s+=x*x; return sqrt(s); };
48
49
    printVec("Original g", g);
50
    cout << "||g||2 = " << l2(g) << "\n";
51
    printVec("Clip by value (c=5)", gv);
52
    cout << "||g_v||2 = " << l2(gv) << "\n";
53
    printVec("Clip by norm (c=3)", gn);
54
    cout << "||g_n||2 = " << l2(gn) << "\n";
55
56
    return 0;
57
}
58

1
#include <bits/stdc++.h>
2
using namespace std;
3
4
struct LinReg {
5
    // Model: y = a * x + b
6
    double a = 0.0;
7
    double b = 0.0;
8
};
9
10
// Compute gradients of MSE over a mini-batch
11
pair<double,double> batchGrad(const LinReg& m, const vector<double>& X, const vector<double>& Y) {
12
    // dL/da = (2/N) * sum (a*x_i + b - y_i) * x_i
13
    // dL/db = (2/N) * sum (a*x_i + b - y_i)
14
    double dA = 0.0, dB = 0.0;
15
    size_t N = X.size();
16
    for (size_t i = 0; i < N; ++i) {
17
        double pred = m.a * X[i] + m.b;
18
        double err = pred - Y[i];
19
        dA += err * X[i];
20
        dB += err;
21
    }
22
    double scale = 2.0 / static_cast<double>(N);
23
    return {scale * dA, scale * dB};
24
}
25
26
// Clip a 2D gradient (a,b) by L2 norm threshold c
27
pair<double,double> clip2ByNorm(double ga, double gb, double c, double eps = 1e-12) {
28
    double n = sqrt(ga*ga + gb*gb);
29
    double s = (n > c) ? (c / (n + eps)) : 1.0;
30
    return {ga * s, gb * s};
31
}
32
33
int main() {
34
    // Generate synthetic data: y = 3x + 2 with noise
35
    std::mt19937 rng(42);
36
    std::normal_distribution<double> noise(0.0, 0.1);
37
38
    vector<double> X, Y;
39
    for (int i = 0; i < 200; ++i) {
40
        double x = (i - 100) / 10.0; // spread inputs
41
        double y = 3.0 * x + 2.0 + noise(rng);
42
        X.push_back(x);
43
        Y.push_back(y);
44
    }
45
46
    LinReg model;
47
    double lr = 0.5;       // Deliberately large to show stabilization via clipping
48
    double clip_c = 1.0;   // Norm clipping threshold
49
50
    // SGD with mini-batches
51
    size_t epochs = 30;
52
    size_t batch = 20;
53
54
    for (size_t e = 0; e < epochs; ++e) {
55
        // Shuffle indices for each epoch
56
        vector<size_t> idx(X.size());
57
        iota(idx.begin(), idx.end(), 0);
58
        shuffle(idx.begin(), idx.end(), rng);
59
60
        for (size_t s = 0; s < X.size(); s += batch) {
61
            size_t t = min(s + batch, X.size());
62
            vector<double> xb, yb;
63
            xb.reserve(t - s); yb.reserve(t - s);
64
            for (size_t i = s; i < t; ++i) { xb.push_back(X[idx[i]]); yb.push_back(Y[idx[i]]); }
65
66
            auto [ga, gb] = batchGrad(model, xb, yb);
67
            // Clip gradients by L2 norm before the update
68
            auto [gac, gbc] = clip2ByNorm(ga, gb, clip_c);
69
70
            // Parameter update
71
            model.a -= lr * gac;
72
            model.b -= lr * gbc;
73
        }
74
75
        // Compute MSE for monitoring
76
        double mse = 0.0;
77
        for (size_t i = 0; i < X.size(); ++i) {
78
            double err = (model.a * X[i] + model.b) - Y[i];
79
            mse += err * err;
80
        }
81
        mse /= X.size();
82
        cout << "Epoch " << e+1 << ": a=" << model.a << ", b=" << model.b << ", MSE=" << mse << "\n";
83
    }
84
85
    cout << "Learned parameters: a=" << model.a << ", b=" << model.b << " (target ~ 3, 2)\n";
86
    return 0;
87
}
88

1
#include <bits/stdc++.h>
2
using namespace std;
3
4
// Clip multiple gradient tensors by a single global L2 norm threshold.
5
// Returns the scaling factor actually applied.
6
double clipByGlobalNorm(vector<vector<double>>& grads, double c, double eps = 1e-12) {
7
    // Compute global norm = sqrt(sum_j sum_i g_{j,i}^2) over finite entries
8
    long double sqsum = 0.0L;
9
    for (const auto& g : grads) {
10
        for (double x : g) {
11
            if (std::isfinite(x)) sqsum += static_cast<long double>(x) * static_cast<long double>(x);
12
        }
13
    }
14
    double global_norm = sqrt((double)sqsum);
15
    double scale = (global_norm > c) ? (c / (global_norm + eps)) : 1.0;
16
17
    // Apply the same scale to all gradients; also sanitize non-finite values to 0
18
    for (auto& g : grads) {
19
        for (double& x : g) {
20
            if (!std::isfinite(x)) x = 0.0; // defensive: drop NaN/Inf contributions
21
            x *= scale;
22
        }
23
    }
24
    return scale;
25
}
26
27
void printGrads(const vector<vector<double>>& G) {
28
    cout << fixed << setprecision(4);
29
    for (size_t j = 0; j < G.size(); ++j) {
30
        cout << "Tensor " << j << ": [";
31
        for (size_t i = 0; i < G[j].size(); ++i) cout << G[j][i] << (i+1==G[j].size()?"":", ");
32
        cout << "]\n";
33
    }
34
}
35
36
int main() {
37
    // Suppose we have gradients for W1 (6 params), b1 (3 params), and W2 (4 params)
38
    vector<vector<double>> grads = {
39
        { 10.0, -8.0,  2.0,  0.5,  -0.1,  4.0 },   // W1
40
        {  3.0,  100.0, -2.0 },                    // b1 (contains a large outlier)
41
        { -5.0,  1.0,   std::numeric_limits<double>::infinity(), -0.2 } // W2 with Inf
42
    };
43
44
    cout << "Before clipping:\n";
45
    printGrads(grads);
46
47
    double c = 5.0; // global norm threshold
48
    double scale = clipByGlobalNorm(grads, c);
49
50
    cout << "\nApplied global scale = " << scale << "\n";
51
    cout << "After clipping:\n";
52
    printGrads(grads);
53
54
    // Compute resulting global norm for verification
55
    long double sqsum = 0.0L;
56
    for (const auto& g : grads) for (double x : g) sqsum += x * x;
57
    cout << "Resulting global L2 norm = " << sqrt((double)sqsum) << "\n";
58
59
    return 0;
60
}
61