For enabling nvenc support in peertube i had to add binds to the nvidia libs on host machine

services:
  peertube:
    image: chocobozzz/peertube:production-bookworm
    restart: unless-stopped
    container_name: peertube
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: 1
              capabilities:
                - gpu
    ports:
      - 29511:9000
      - 1935:1935
    volumes:
      - /storage/peertube/data:/data
      - ./docker-volume/config:/config
      - /usr/lib/libnvidia-encode.so.1:/usr/lib/x86_64-linux-gnu/libnvidia-encode.so.1:ro
      - /usr/lib/libnvcuvid.so.1:/usr/lib/x86_64-linux-gnu/libnvcuvid.so.1:ro

Without the libs it should fallback to x264 cpu encoding.