Hive解决方案。使用 row_number() 用于删除重复项。看看怎么做 protocol_key 以及 path_key 是计算出来的，它们用于 row_number() partition by 条款：

with your_data as (--use your table instead of this
select stack( 4,
'https://www.cnn.com/2019/09/20/politics/',
'https://www.cnn.com/2019/09/20/politics',
'http://www.cnn.com/2019/09/20/politics/',
'http://www.cnn.com/2019/09/20/politics') as url
) --your table

select url from
(
select s.url, --s.protocol, s.protocol_key, s.host, s.path, s.path_key, --columns for debugging
       row_number() over(partition by s.protocol_key, s.host, s.path_key order by s.protocol desc, s.path desc) rn --https and path with/ are preferred
from       
(--parse url
select t.url, s.protocol, s.host, s.path, regexp_replace(s.path,'/$','') as path_key, regexp_replace(s.protocol,'s$','') as protocol_key
  from your_data t
       lateral view parse_url_tuple(url, 'PROTOCOL','HOST', 'PATH')s as protocol, host, path
)s)s where rn = 1
;

结果：

https://www.cnn.com/2019/09/20/politics/