Got it working with nginx. Works perfectly. Here is the config. Also need to add a subpath in the jellyfin network setting.
location /jellyfin {
proxy_pass http://192.168.x.x:8096/jellyfin;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Host $host;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_redirect off;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $http_connection;
proxy_buffering off;
}