Remove class at top of page isn't always working - html

Question

I've got a header that I want to disappear (move above the top of the browser screen) when the user scrolls down, then when the user scrolls back up I want the header to reappear (move back down) and also to gain a solid white background behind it.
All of this is working fine, but then I want the white background to be removed if the user scrolls back up to the very top of the page again (so it's just the header visible but with no white background).
I've cobbled together the code for this, and it's pretty much working - but, for some reason, in certain circumstances, the white background isn't being removed?!
It seems to be if you've scrolled back up enough to make the header reappear with the white background, but not all the way top the top, and then scroll the rest of the way then the white background doesn't disappear for some reason.
Sorry, I've probably not explained that very well, but here's my code. If someone is able to replicate the bug I'm getting and offer an explanation I'd be very grateful!
Thanks
// Hide Header on on scroll down
var didScroll;
var lastScrollTop = 0;
var delta = 30;
var navbarHeight = $('header').outerHeight();
$(window).scroll(function(event){
didScroll = true;
});
setInterval(function() {
if (didScroll) {
hasScrolled();
didScroll = false;
}
}, 30);
function hasScrolled() {
var st = $(this).scrollTop();
// Make sure they scroll more than delta
if(Math.abs(lastScrollTop - st) <= delta)
return;
// If they scrolled down and are past the navbar, add class .nav-up.
// This is necessary so you never see what is "behind" the navbar.
if (st > lastScrollTop && st > navbarHeight){
// Scroll Down
$('header').removeClass('header-down').addClass('header-up').removeClass('header-solid');
} else {
// Scroll Up
if(st + $(window).height() < $(document).height()) {
$('header').removeClass('header-up').addClass('header-down').addClass('header-solid');
}
}
lastScrollTop = st;
};
$(function () {
$(window).scroll(function () {
var top_offset = $(window).scrollTop();
if (top_offset == 0) {
$('header').removeClass('header-solid');
}
})
});
body {
margin: 0;
background: url('https://images.unsplash.com/3/doctype-hi-res.jpg?ixid=MXwxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHw%3D&ixlib=rb-1.2.1&auto=format&fit=crop&w=2246&q=80');
background-size: cover;
}
.wrapper {
width: 100%;
height: 3000px;
}
header {
display: flex;
width: 100%;
padding: 20px 50px;
position: fixed;
box-sizing: border-box;
transition: 300ms all ease-in-out;
margin-top: 0px;
}
.header-logo {
width: 200px;
}
.header-nav {
flex-grow: 1;
text-align: center;
}
.header-socials {
width: 200px;
text-align: right;
}
.header-up {
margin-top: -60px;
}
.header-down {
margin-top: 0px;
}
.header-solid {
background-color: #FFFFFF;
}
<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.3.1/jquery.min.js"></script>
<div class="wrapper">
<header class="header-down">
<div class="header-logo">
<img src="#" class="img-header-logo"/>
</div>
<div class="header-nav">
One Two Three Four Five
</div>
<div class="header-socials">
Social Links
</div>
</header>
</div>