如何在sql中批量选择大表中的记录

wtlkbnrh  于 2021-07-29  发布在  Java
关注(0)|答案(1)|浏览(417)

我有一个包含数十亿条记录的表。我想用c# while 循环。我曾经 Offset 以及 Fetch Next ,但要得到结果需要太长时间。如果表中包含的记录数低于10万条,则可以正常工作。批量选择记录的最佳方法是什么?
样本代码

  1. int Count = 500000 // Select Count(id) from table
  2. int OffSet = 1; //Next Time 100000
  3. int FetchNext = 100000; //Next Time 200000
  4. while (OffSet < Count)
  5. {
  6. _strsqlcommand = "Select id from table ORDER BY id OFFSET " + OffSet + " ROWS FETCH NEXT " + FetchNext + " ROWS ONLY"
  7. _sqlCommand = new SqlCommand(_strsqlcommand, _sqlConnection);
  8. _sqlDataReader = _sqlCommand.ExecuteReader();
  9. OffSet += FetchNext;
  10. }
xqkwcwgp

xqkwcwgp1#

如果您的id是一个数字,并且您的网络带宽足够好:
首先,找出最大值和最小值 Id .
其次,在where子句中创建一系列数字(对许多记录进行排序是一项高成本的操作)。使用 Where Id > @A and Id<@B 比…快得多 fetch next ).
使用tpl从数据库中提取数据。

  1. namespace BatchLoad
  2. {
  3. class Program
  4. {
  5. static void Main(string[] args)
  6. {
  7. var start = DateTime.Now;
  8. int MinId = 169328112;
  9. int MaxId = 505690099;
  10. int BatchCount = 1000000;
  11. List<QueryExe> Alllist=new List<QueryExe>(BatchCount);
  12. var stack =new ConcurrentStack<int>();
  13. int i = MinId;
  14. int index = 0;
  15. while(i<MaxId+1)
  16. {
  17. int minid = i;
  18. int maxid = i + BatchCount;
  19. string q = $"SELECT [Id] FROM YourTable with(nolock) WHERE Id>={minid} and Id<{maxid} ";
  20. string c = "Data Source=.;Initial Catalog=YourDatabase;Persist Security Info=True;User ID=??;Password=!!!;MultipleActiveResultSets=True";
  21. i = maxid;
  22. Alllist.Add(new QueryExe(q,c, index));
  23. index++;
  24. }
  25. long ProccessCount = 0;
  26. Parallel.ForEach(Alllist, new ParallelOptions {MaxDegreeOfParallelism = 100}, command =>
  27. {
  28. Task.Yield();
  29. var temp = command.GetId();
  30. if(temp?.Count>0)stack.PushRange(temp.ToArray());
  31. Interlocked.Add(ref ProccessCount,1);
  32. var donitems = Interlocked.Read(ref ProccessCount);
  33. if (donitems %10 == 0)
  34. {
  35. Console.WriteLine($"{donitems} / {Alllist.Count} TotalM={stack.Count/1000000} Total={stack.Count}");
  36. }
  37. });
  38. GC.Collect();
  39. Console.WriteLine($"Done>{DateTime.Now.Subtract(start).TotalMilliseconds} count ={stack.Count/1000000}");
  40. Console.ReadLine();
  41. }
  42. }
  43. public class QueryExe
  44. {
  45. private string Q = "";
  46. private string C = "";
  47. private int i = 0;
  48. public QueryExecutor(string Q, string C, int i)
  49. {
  50. this.Q = Q;
  51. this.C = C;
  52. this.i = i;
  53. }
  54. public List<int> GetId()
  55. {
  56. var result = new List<int>();
  57. try
  58. {
  59. SqlConnection conn = new SqlConnection(C);
  60. SqlCommand command = new SqlCommand(this.Q, conn);
  61. command.CommandTimeout = 180;
  62. using (conn)
  63. {
  64. conn.Open();
  65. using (SqlDataReader reader = command.ExecuteReader())
  66. {
  67. while (reader.Read())
  68. {
  69. result.Add(reader.GetInt32(0));
  70. }
  71. }
  72. }
  73. }
  74. catch (Exception ex)
  75. {
  76. Console.WriteLine($"Exception>{i}");
  77. }
  78. return result;
  79. }
  80. }
  81. }
展开查看全部

相关问题