JAX CUDA Optimization Guide: XLA and GPU Acceleration

# Install JAX with CUDA 12 support
pip install --upgrade "jax[cuda12_pip]" -f https://storage.googleapis.com/jax-releases/jax_cuda_releases.html

# Or for CUDA 11
pip install --upgrade "jax[cuda11_pip]" -f https://storage.googleapis.com/jax-releases/jax_cuda_releases.html

# Verify installation
python -c "import jax; print(f'JAX {jax.__version__}'); print(f'Devices: {jax.devices()}')"

# Check GPU backend
python -c "import jax; print(jax.default_backend())"  # Should print 'gpu'

import jax
import jax.numpy as jnp
from jax import grad, jit, vmap

# Simple function - automatically runs on GPU if available
def predict(params, x):
    w, b = params
    return jnp.dot(x, w) + b

def loss_fn(params, x, y):
    pred = predict(params, x)
    return jnp.mean((pred - y) ** 2)

# JIT compile for GPU optimization
loss_fn_jit = jit(loss_fn)

# Automatic differentiation
grad_fn = jit(grad(loss_fn))

# Initialize parameters on GPU
key = jax.random.PRNGKey(0)
w = jax.random.normal(key, (784, 10))
b = jnp.zeros(10)
params = (w, b)

# Training step
@jit
def train_step(params, x, y, lr=0.01):
    grads = grad(loss_fn)(params, x, y)
    new_params = jax.tree_map(lambda p, g: p - lr * g, params, grads)
    return new_params

# Batch processing with vmap (automatic vectorization)
batched_predict = vmap(predict, in_axes=(None, 0))

# Training loop
for epoch in range(100):
    params = train_step(params, x_train, y_train)
    if epoch % 10 == 0:
        loss = loss_fn_jit(params, x_train, y_train)
        print(f"Epoch {epoch}, Loss: {loss:.4f}")

import jax
import jax.numpy as jnp
from jax import pmap, jit, grad
from jax.sharding import PositionalSharding

# Check available devices
devices = jax.devices()
n_devices = len(devices)
print(f"Training on {n_devices} GPUs")

# Model definition
def mlp(params, x):
    for w, b in params[:-1]:
        x = jax.nn.relu(jnp.dot(x, w) + b)
    w, b = params[-1]
    return jnp.dot(x, w) + b

def loss_fn(params, x, y):
    logits = mlp(params, x)
    return jnp.mean(jax.nn.softmax_cross_entropy_with_logits(logits, y))

# Parallelized training step
@pmap
def train_step(params, x, y, lr=0.001):
    grads = grad(loss_fn)(params, x, y)
    # All-reduce gradients across devices
    grads = jax.lax.pmean(grads, axis_name='devices')
    new_params = jax.tree_map(lambda p, g: p - lr * g, params, grads)
    return new_params

# Initialize and replicate params across devices
def init_params(key, layers):
    params = []
    for in_dim, out_dim in zip(layers[:-1], layers[1:]):
        key, subkey = jax.random.split(key)
        w = jax.random.normal(subkey, (in_dim, out_dim)) * 0.01
        b = jnp.zeros(out_dim)
        params.append((w, b))
    return params

key = jax.random.PRNGKey(0)
params = init_params(key, [784, 512, 256, 10])

# Replicate params to all devices
params = jax.device_put_replicated(params, devices)

# Split batch across devices
def split_batch(x, y):
    batch_size = x.shape[0] // n_devices
    x = x.reshape(n_devices, batch_size, -1)
    y = y.reshape(n_devices, batch_size, -1)
    return x, y

# Training loop
for epoch in range(100):
    x_split, y_split = split_batch(x_train, y_train)
    params = train_step(params, x_split, y_split)