CUDA Memory Management Best Practices

// Anti-pattern: allocating in loop
void process_batches(int num_batches, int batch_size) {
    for (int i = 0; i < num_batches; i++) {
        float* d_data;
        cudaMalloc(&d_data, batch_size * sizeof(float));  // Slow!

        process_kernel<<<grid, block>>>(d_data);

        cudaFree(d_data);  // Also slow!
    }
}

// CUDA 11.2+ memory pool
void process_with_pool(int num_batches, int batch_size) {
    // Create memory pool
    cudaMemPool_t mempool;
    cudaDeviceGetDefaultMemPool(&mempool, 0);

    // Set pool to release threshold (keep memory for reuse)
    uint64_t threshold = UINT64_MAX;
    cudaMemPoolSetAttribute(mempool, cudaMemPoolAttrReleaseThreshold, &threshold);

    cudaStream_t stream;
    cudaStreamCreate(&stream);

    for (int i = 0; i < num_batches; i++) {
        float* d_data;
        // Stream-ordered async allocation - uses pool
        cudaMallocAsync(&d_data, batch_size * sizeof(float), stream);

        process_kernel<<<grid, block, 0, stream>>>(d_data);

        // Async free - memory returned to pool
        cudaFreeAsync(d_data, stream);
    }

    cudaStreamSynchronize(stream);
}

// Manual memory pool for older CUDA
class MemoryPool {
    std::vector<void*> free_blocks;
    size_t block_size;

public:
    MemoryPool(size_t size) : block_size(size) {}

    void* allocate() {
        if (!free_blocks.empty()) {
            void* ptr = free_blocks.back();
            free_blocks.pop_back();
            return ptr;
        }
        void* ptr;
        cudaMalloc(&ptr, block_size);
        return ptr;
    }

    void deallocate(void* ptr) {
        free_blocks.push_back(ptr);
    }
};

// Pinned memory for fast CPU-GPU transfer
void pinned_transfer_example() {
    float* h_pinned;
    cudaMallocHost(&h_pinned, size);  // Pinned (page-locked)

    // 2x faster transfer than pageable memory
    cudaMemcpyAsync(d_data, h_pinned, size,
                    cudaMemcpyHostToDevice, stream);

    cudaFreeHost(h_pinned);
}