编辑。
我现在明白问题所在了。试着保存滚动的最后一个Y位置。

let lastScrollY = 0;
window.addEventListener('scroll', function(e) {
    if (window.scrollY == 0 || window.scrollY<lastScrollY) {
      document.getElementById('search-bar-scroll').classList.add('scrolled');
    } else {
      document.getElementById('search-bar-scroll').classList.remove('scrolled');
    }
    lastScrollY = window.scrollY;
});

* {
    margin: 0;
    padding: 0;
  }
  
  img,
  fieldset {
    border: none;
  }
  
  body *,
  *:after,
  *:before {
    box-sizing: border-box;
  }
  
  html,
  body {
    height: 250vh;
  }
  
  .main-wrapper {
    max-width: 1400px;
    margin-left: auto;
    margin-right: auto;
  }
  
  .search-bar-scroll {
    background: #1e5da5;
    position: fixed;
    height: auto;
    width: 100%;
    left: 0;
    top: 0;
    opacity: 0;
    transition: 0.3s ease;
  }
  
  .scrolled {
    opacity: 1;
    transition: 150ms all;
  }
  
  #header {
    background: #1e5da5;
    padding: 1rem;
    position: sticky;
    top: 0;
    left: 0;
    z-index: 2;
  }

<div class="search-bar-scroll-container">
  <div class="search-bar-scroll scrolled" id="search-bar-scroll">
    <fieldset class="home-header-search">
      <div action="#" id="scroll-input-container">
        <p>Hello World</p>
      </div>
    </fieldset>
  </div>
</div>

感谢布兰登帮助我的答案，效果很好，这个答案是对Rango使用wheel事件的评论/建议的回应，它的工作就像一个魅力!
我是这么做的：

let scrollPosition = 0;
let scrollElement = document.getElementById("search-bar-scroll");

document.addEventListener("wheel", function(e) {
  if (e.deltaY > 0) {
    scrollElement.style.display = "none";
  } else {
    scrollElement.style.display = "block";
  }
});

* {
  margin: 0;
  padding: 0;
}

img,
fieldset {
  border: none;
}

body *,
*:after,
*:before {
  box-sizing: border-box;
}

html,
body {
  height: 150vh;
}

.main-wrapper {
  max-width: 1400px;
  margin-left: auto;
  margin-right: auto;
}

.search-bar-scroll {
  background: #1e5da5;
  position: fixed;
  height: auto;
  width: 100%;
  left: 0;
  top: 0;
}

<div class="search-bar-scroll" id="search-bar-scroll">
  <fieldset class="home-header-search">
    <div action="#" id="scroll-input-container">
      <p>Hello World</p>
    </div>
  </fieldset>
</div>