正如@Pace所建议的，你应该考虑将输出文件的创建移出阅读循环，比如：

import sys

import pandas as pd
import pyarrow as pa

SPLIT_ROWS = 1     ### used one line chunks for a small test

def main():
    # Write out to file
    with pa.OSFile('test.arrow', 'wb') as sink:     ### no append mode yet
        with pa.RecordBatchFileWriter(sink, table.schema) as writer:
            for split in pd.read_csv('data.csv', chunksize=SPLIT_ROWS):
                table = pa.Table.from_pandas(split)
                writer.write_table(table)

if __name__ == "__main__":
    main()

如果您希望指定特定的输入和输出文件，也不必使用sys.stdin.buffer。您可以运行脚本如下：

python test.py

通过使用with语句，writer和sink都将在之后自动关闭（在本例中，当main()返回时），这意味着不需要包含显式的close()调用。

解决方案改编自@Martin-Evans代码：
按照@Pace的建议，在for循环后关闭文件

import sys

import pandas as pd
import pyarrow as pa

SPLIT_ROWS = 1000000

def main():
    schema = pa.Table.from_pandas(pd.read_csv('Data.csv',nrows=2)).schema 
    ### reads first two lines to define schema 

    with pa.OSFile('test.arrow', 'wb') as sink:
        with pa.RecordBatchFileWriter(sink, schema) as writer:            
            for split in pd.read_csv('Data.csv',chunksize=SPLIT_ROWS):
                table = pa.Table.from_pandas(split)
                writer.write_table(table)

            writer.close()

if __name__ == "__main__":
    main()

在2023年你不需要Pandas来做这个。你可以通过csv使用箭头chunk：

import pyarrow as pa
from pyarrow import csv

schema =  pa.schema([
        ('time', pa.timestamp('ms', None)),
        ('deviceid', pa.utf8())
])
convert_dict = {
  'time': pa.timestamp('ms', None),
  'deviceid': pa.utf8()
}
convert_options = pa.csv.ConvertOptions(
    column_types=convert_dict
    , strings_can_be_null=True
    , quoted_strings_can_be_null=True
    ,timestamp_parsers=["%Y-%m-%d %H:%M:%S"],
)

arrowfile = "data_dst.arrow"
csvfile = "data_src.csv"

with pa.OSFile(arrowfile, 'wb') as sink:     ### no append mode yet
   with pa.csv.open_csv(csvfile, convert_options=convert_options) as reader:
       with pa.RecordBatchFileWriter(sink, schema) as writer:
           for next_chunk in reader:
               if next_chunk is None:
                    break
               if writer is None:
                    break
               next_table = pa.Table.from_batches([next_chunk])
               writer.write_table(next_table)