我有以下程序从数据库中获取超过100000个URL,并使用c#httpwebrequest对象获取响应并保存到数据库中。
要获得100000个URL的响应,需要在单个服务器上运行很长时间。我想把这个进程分成线程,并希望在多个服务器上运行。
有没有可能在c语言的mapreduce中使用这种类型的代码,以及线程和tpl?
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Text;
using System.Threading.Tasks;
using System.Runtime.Serialization.Json;
using System.Net;
using System.Runtime.Serialization;
using System.IO;
namespace Test
{
class Program
{
static void Main(string[] args)
{
TestApp testapp = new TestApp();
var urls = testapp.GetURls();
//get string from urls
foreach (var url in urls)
{
var responseString=testapp.GetURLString(url);
//now save response into database
testapp.saveResponseToDB(responseString);
}
}
}
public class TestApp
{
public void saveResponseToDB(string response)
{
//save into db
}
public List<string> GetURls()
{
var urls =new List<string>();
//fetching more then 100000 urls from database.
return urls;
}
public string GetURLString(string requestUrl)
{
HttpWebRequest request = WebRequest.Create(requestUrl) as HttpWebRequest;
using (HttpWebResponse response = request.GetResponse() as HttpWebResponse)
{
StreamReader reader = new StreamReader(response.GetResponseStream());
// Read the content.
string responseFromServer = reader.ReadToEnd();
// return the content.
return responseFromServer;
}
}
}
}
暂无答案!
目前还没有任何答案,快来回答吧!