Three (3) steps to completely eliminate IPV6 on your system. Step 3 applies only if you are using SSH in daemon mode (on a server). The following should work for all Debian.
Step 1
sudo vi /etc/sysctl.conf
# to disable IPv6 on all interfaces system wide
net.ipv6.conf.all.disable_ipv6 = 1
# Uncomment the next two lines to enable Spoof protection (reverse-path filter)
# Turn on Source Address Verification in all interfaces to prevent some spoofing attacks
net.ipv4.conf.default.rp_filter=1
net.ipv4.conf.all.rp_filter=1
# Do not accept ICMP redirects (prevent MITM attacks)
net.ipv4.conf.all.accept_redirects = 0
net.ipv6.conf.all.accept_redirects = 0
# Do not send ICMP redirects (we are not a router)
net.ipv4.conf.all.send_redirects = 0
# Do not accept IP source route packets (we are not a router)
net.ipv4.conf.all.accept_source_route = 0
net.ipv6.conf.all.accept_source_route = 0
# a helpful addition I use to avoid excess swapping
vm.swappiness=0
sudo sysctl -p /etc/sysctl.conf
Step 2
sudo echo 'net.ipv6.conf.all.disable_ipv6 = 1' > /etc/sysctl.d/90-disable-ipv6.conf
sudo sysctl -p -f /etc/sysctl.d/90-disable-ipv6.conf
Step 3
sudo vi /etc/ssh/sshd_config
addressFamily inet
sudo service ssh restart