我们可以在没有.explode()的情况下完成它;也许这将有助于解决您的内存问题：

# test case

df = pd.DataFrame([
    [list('abc'), list('xyz'), 0.1],
    [list('wxy'), list('abd'), 0.2],
], columns=['A', 'B', 'target'])

字符串
然后又道：

a = df['A'].apply('|'.join).str.get_dummies()
b = df['B'].apply('|'.join).str.get_dummies()
cols = a.columns.union(b.columns)

out = pd.concat([
    a.reindex(cols, axis=1, fill_value=0) - b.reindex(cols, axis=1, fill_value=0),
    df.drop(['A', 'B'], axis=1),
], axis=1)
out

>>> out
   a  b  c  d  w  x  y  z  target
0  1  1  1  0  0 -1 -1 -1     0.1
1 -1 -1  0 -1  1  1  1  0     0.2

型

如果所有列表都有相同数量的值，则可以使用两个crosstab：

idx = df['idx'].repeat(3) # use here the list length as value

out = (pd.crosstab(idx, np.array(df['A'].tolist()).ravel())
       .sub(pd.crosstab(idx, np.array(df['B'].tolist()).ravel()),
            fill_value=0)
       .astype(int).rename_axis(columns=None).reset_index()
      )

字符串
输出量：

idx  a  b  c  d  w  x  y  z
0    1  1  1  1  0  0 -1 -1 -1
1    2 -1 -1  0 -1  1  1  1  0

型

您可以尝试：

new_df = (
    pd.DataFrame(
        (dict.fromkeys(A, 1) | dict.fromkeys(B, -1) for A, B in zip(df.A, df.B))
    )
    .fillna(0)
    .astype(int)
    .assign(target=df.target, idx=df["idx"])
)
print(new_df)

字符串
印刷品：

a  b  c  x  y  z  w  d  target  idx
0  1  1  1 -1 -1 -1  0  0     0.1    1
1 -1 -1  0  1  1  0  1 -1     0.2    2

型
快速基准：

from random import sample
from timeit import timeit

def setup_df(n=250_000, n_cats=25_000):
    data = []
    for i in range(n):
        data.append(
            {
                "idx": i,
                "A": list(map(str, sample(range(n_cats), k=11))),
                "B": list(map(str, sample(range(n_cats), k=11))),
                "target": 0.1,
            }
        )
    return pd.DataFrame(data)

def fn1(df):
    new_df = (
        pd.DataFrame(
            (dict.fromkeys(A, 1) | dict.fromkeys(B, -1) for A, B in zip(df.A, df.B))
        )
        .fillna(0)
        .astype(int)
        .assign(target=df.target, idx=df["idx"])
    )

    return new_df

def fn2(df):
    idx = df["idx"].repeat(11)  # use here the list length as value

    out = (
        pd.crosstab(idx, np.array(df["A"].tolist()).ravel())
        .sub(pd.crosstab(idx, np.array(df["B"].tolist()).ravel()), fill_value=0)
        .astype(int)
        .rename_axis(columns=None)
        .reset_index()
    )

    return out

t1 = timeit(
    "fn1(df)", setup="df = setup_df(n=1000, n_cats=100)", globals=globals(), number=1
)
print(t1)
t2 = timeit(
    "fn2(df)", setup="df = setup_df(n=1000, n_cats=100)", globals=globals(), number=1
)
print(t2)

型
印刷品：

0.01216180098708719
0.1373176029883325

型

最后，我选择了一种方法，我在列上进行排序，一次排序一列。速度慢，但内存效率高，因为我可以在继续排序下一列之前将每列更改为稀疏dtype（也许有更好的方法？我不确定）

# gather all vals in cols A and B
all_vals = []
for rowa,rowb in zip(data.A, data.B):
    all_vals += rowa + rowb

unq_categories = set(all_vals)

# sparse df creation
empty = np.empty([len(data.index),len(unq_categories)])
spr_arr = csr_matrix(empty)
sdf = pd.DataFrame(dtype="int8").sparse.from_spmatrix(spr_arr, columns=unq_ids)

for category in sdf:
    # transform column (implicitly turns it dense)
    sdf[category] = data.A.map(lambda category_list: 1 if category in category_list else 0).add(data.B.map(lambda category_list: -1 if category in category_list else 0))
    # make column sparse again to save memory
    sdf[category] = sdf[category].astype(pd.SparseDtype("int8",0))

字符串