From c0952aa2c3a694f3109da4193a2e6164cbda2715 Mon Sep 17 00:00:00 2001 From: YukiCoco Date: Sun, 30 Jan 2022 00:40:47 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E5=92=8C=20BUFF=20=E5=B8=82=E5=9C=BA?= =?UTF-8?q?=E5=BB=BA=E7=AB=8B=E8=BF=9E=E6=8E=A5=E5=A4=B1=E8=B4=A5=E5=AF=BC?= =?UTF-8?q?=E8=87=B4=E7=A8=8B=E5=BA=8F=E5=B4=A9=E6=BA=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ChpStmScraper/Program.cs | 302 +++++++++++++++++++++------------------ 1 file changed, 160 insertions(+), 142 deletions(-) diff --git a/ChpStmScraper/Program.cs b/ChpStmScraper/Program.cs index b5adfa6..5496c9e 100644 --- a/ChpStmScraper/Program.cs +++ b/ChpStmScraper/Program.cs @@ -48,175 +48,193 @@ public Program(Queue _logs, ScraperDbContext _context, object contextLoc { httpService = new HttpService(Configuration.ProxyUrl); } - httpService.GetWithCookie(Configuration.BuffUrl + $"?game={Configuration.GameKind.ToString().ToLower()}&min_price={Configuration.MinSellPrice}&max_price={Configuration.MaxSellPrice}&page_num={pageNum}", new Cookie("session", Configuration.BuffSession), result => + try { - try + httpService.GetWithCookie(Configuration.BuffUrl + $"?game={Configuration.GameKind.ToString().ToLower()}&min_price={Configuration.MinSellPrice}&max_price={Configuration.MaxSellPrice}&page_num={pageNum}", new Cookie("session", Configuration.BuffSession), result => { - result.EnsureSuccessStatusCode(); - } - catch (HttpRequestException ex) - { - // TODO:这里会导致 Response status code does not indicate success: 429 (Too Many Requests). - Console.WriteLine(ex.Message); - Thread.Sleep(TimeSpan.FromSeconds(10)); - return; - } - ThreadPool.QueueUserWorkItem(state => - { - HttpService httpService = new HttpService(); - var jsonObj = JObject.Parse(result.Content.ReadAsStringAsync().Result); - //页面数修改 - lock (o4) + try { - maxPageNum = int.Parse(jsonObj["data"]["total_page"].ToString()); + result.EnsureSuccessStatusCode(); } - foreach (var jItem in jsonObj["data"]["items"]) + catch (HttpRequestException ex) { - lock(o) + // TODO:这里会导致 Response status code does not indicate success: 429 (Too Many Requests). + Console.WriteLine(ex.Message); + Thread.Sleep(TimeSpan.FromSeconds(10)); + return; + } + ThreadPool.QueueUserWorkItem(state => + { + HttpService httpService = new HttpService(); + var jsonObj = JObject.Parse(result.Content.ReadAsStringAsync().Result); + //页面数修改 + lock (o4) { - if(isPause) - { - pageNum--; - lock (o3) - { - currentSyncThread--; - } - return; - } + maxPageNum = int.Parse(jsonObj["data"]["total_page"].ToString()); } - var name = Helper.UnicodeToString(jItem["name"].ToString()); - lock (o5) + foreach (var jItem in jsonObj["data"]["items"]) { - if (lastScraperName == name) + lock (o) { - continue; + if (isPause) + { + pageNum--; + lock (o3) + { + currentSyncThread--; + } + return; + } } - else + var name = Helper.UnicodeToString(jItem["name"].ToString()); + lock (o5) { - lastScraperName = name; - } - } - var buffSellNum = int.Parse(jItem["sell_num"].ToString()); - switch (jItem["game"].ToString() == "csgo" ? Goods.GameKind.CSGO : Goods.GameKind.DOTA2) - { - case Goods.GameKind.CSGO: - if (buffSellNum < Configuration.MinSellCount) + if (lastScraperName == name) { continue; } - break; - case Goods.GameKind.DOTA2: - if (buffSellNum < Configuration.MinSellCount) + else { - continue; + lastScraperName = name; } - break; - } - //开始在 STEAM 查询 - string steamMarketUrl = jItem["steam_market_url"].ToString(); - try - { - httpService.GetWithCookie(steamMarketUrl, Configuration.SteamCookies, result => + } + var buffSellNum = int.Parse(jItem["sell_num"].ToString()); + switch (jItem["game"].ToString() == "csgo" ? Goods.GameKind.CSGO : Goods.GameKind.DOTA2) { - if (!string.IsNullOrEmpty(Configuration.ProxyUrl)) - HttpClient.DefaultProxy = new WebProxy(Configuration.ProxyUrl); - string marketResult = result.Content.ReadAsStringAsync().Result; - //Debug.WriteLine(result.RequestMessage.RequestUri.AbsoluteUri); - //重复直到获取的到值 - // while (getResult.StatusCode != HttpStatusCode.OK) - // { - // try - // { - // //marketResult = await client2.GetStringAsync(steamMarketUrl); - // getResult = await client2.SendAsync(message); - // marketResult = await getResult.Content.ReadAsStringAsync(); - // } - // catch (HttpRequestException ex) - // { - // client2.CancelPendingRequests(); - // Console.WriteLine("出现错误：" + ex.Message); - // //这里会出现 429，等待即可 - // //Thread.Sleep(TimeSpan.FromSeconds(120)); - // } - // } - //Debug.WriteLine(marketResult); - //获取 steamID 正则 - Regex regex = new Regex(@"(?<=Market_LoadOrderSpread\(\s).*(?=\s\);)", RegexOptions.Multiline); - var marketId = regex.Match(marketResult).Value; - if (string.IsNullOrEmpty(marketId)) - { - Thread.Sleep(TimeSpan.FromSeconds(10)); - throw new Exception("错误：找不到MarketID，可能是爬取速度过快"); - } - string steamApiUrl = $"https://steamcommunity.com/market/itemordershistogram?country=CN&language=schinese¤cy=23&item_nameid={marketId}&two_factor=0"; - - var apiResult = httpService.Get(steamApiUrl); - JObject steamJsonObj = JObject.Parse(apiResult.Content.ReadAsStringAsync().Result); - //获取 steam 价格正则 - regex = new Regex(@"(?<=数量<\/th><\/tr>¥ ).*?(?=<\/td>)", RegexOptions.Multiline); - var marketSellPrice = regex.Match(steamJsonObj["sell_order_table"].ToString()).Value; - regex = new Regex(@"(?<=数量<\/th><\/tr>¥ ).*?(?=<)", RegexOptions.Multiline); - var marketBuyPrice = regex.Match(steamJsonObj["buy_order_table"].ToString()).Value; - regex = new Regex(@"(?<=>)\d*(?=<\/span>\s个出售中)", RegexOptions.Multiline); - var SteamSellNum = regex.Match(steamJsonObj["sell_order_summary"].ToString()).Value; - Goods item = new Goods() - { - Kind = jItem["game"].ToString() == "csgo" ? Goods.GameKind.CSGO : Goods.GameKind.DOTA2, - Name = name, - BuffBuyPrice = Helper.String2Double(jItem["buy_max_price"].ToString()), - BuffSellPrice = Helper.String2Double(jItem["sell_min_price"].ToString()), - SteamSellPrice = Helper.String2Double(marketSellPrice), - SteamBuyPrice = Helper.String2Double(marketBuyPrice), - SteamSellNum = Helper.String2Int(SteamSellNum), - BuffSellNum = buffSellNum - }; - if (item.BuffSellPrice != 0 && item.SteamBuyPrice != 0) - item.SteamBuyRadio = Math.Round(item.BuffSellPrice / (item.SteamBuyPrice / 1.15), 2); - else - item.SteamBuyRadio = 0; - if (item.BuffSellPrice != 0 && item.SteamSellPrice != 0) - item.SteamSellRadio = Math.Round(item.BuffSellPrice / (item.SteamSellPrice / 1.15), 2); - else - item.SteamSellRadio = 0; - lock (o2) + case Goods.GameKind.CSGO: + if (buffSellNum < Configuration.MinSellCount) + { + continue; + } + break; + case Goods.GameKind.DOTA2: + if (buffSellNum < Configuration.MinSellCount) + { + continue; + } + break; + } + //开始在 STEAM 查询 + string steamMarketUrl = jItem["steam_market_url"].ToString(); + try + { + httpService.GetWithCookie(steamMarketUrl, Configuration.SteamCookies, result => { - var querry = context.Goods.Find(item.Name); - if (querry == null) + if (!string.IsNullOrEmpty(Configuration.ProxyUrl)) + HttpClient.DefaultProxy = new WebProxy(Configuration.ProxyUrl); + string marketResult = result.Content.ReadAsStringAsync().Result; + //Debug.WriteLine(result.RequestMessage.RequestUri.AbsoluteUri); + //重复直到获取的到值 + // while (getResult.StatusCode != HttpStatusCode.OK) + // { + // try + // { + // //marketResult = await client2.GetStringAsync(steamMarketUrl); + // getResult = await client2.SendAsync(message); + // marketResult = await getResult.Content.ReadAsStringAsync(); + // } + // catch (HttpRequestException ex) + // { + // client2.CancelPendingRequests(); + // Console.WriteLine("出现错误：" + ex.Message); + // //这里会出现 429，等待即可 + // //Thread.Sleep(TimeSpan.FromSeconds(120)); + // } + // } + //Debug.WriteLine(marketResult); + //获取 steamID 正则 + Regex regex = new Regex(@"(?<=Market_LoadOrderSpread\(\s).*(?=\s\);)", RegexOptions.Multiline); + var marketId = regex.Match(marketResult).Value; + if (string.IsNullOrEmpty(marketId)) { - context.Goods.Add(item); + Thread.Sleep(TimeSpan.FromSeconds(10)); + throw new Exception("错误：找不到MarketID，可能是爬取速度过快"); } + string steamApiUrl = $"https://steamcommunity.com/market/itemordershistogram?country=CN&language=schinese¤cy=23&item_nameid={marketId}&two_factor=0"; + + var apiResult = httpService.Get(steamApiUrl); + JObject steamJsonObj = JObject.Parse(apiResult.Content.ReadAsStringAsync().Result); + //获取 steam 价格正则 + regex = new Regex(@"(?<=数量<\/th><\/tr>¥ ).*?(?=<\/td>)", RegexOptions.Multiline); + var marketSellPrice = regex.Match(steamJsonObj["sell_order_table"].ToString()).Value; + regex = new Regex(@"(?<=数量<\/th><\/tr>¥ ).*?(?=<)", RegexOptions.Multiline); + var marketBuyPrice = regex.Match(steamJsonObj["buy_order_table"].ToString()).Value; + regex = new Regex(@"(?<=>)\d*(?=<\/span>\s个出售中)", RegexOptions.Multiline); + var SteamSellNum = regex.Match(steamJsonObj["sell_order_summary"].ToString()).Value; + Goods item = new Goods() + { + Kind = jItem["game"].ToString() == "csgo" ? Goods.GameKind.CSGO : Goods.GameKind.DOTA2, + Name = name, + BuffBuyPrice = Helper.String2Double(jItem["buy_max_price"].ToString()), + BuffSellPrice = Helper.String2Double(jItem["sell_min_price"].ToString()), + SteamSellPrice = Helper.String2Double(marketSellPrice), + SteamBuyPrice = Helper.String2Double(marketBuyPrice), + SteamSellNum = Helper.String2Int(SteamSellNum), + BuffSellNum = buffSellNum + }; + if (item.BuffSellPrice != 0 && item.SteamBuyPrice != 0) + item.SteamBuyRadio = Math.Round(item.BuffSellPrice / (item.SteamBuyPrice / 1.15), 2); + else + item.SteamBuyRadio = 0; + if (item.BuffSellPrice != 0 && item.SteamSellPrice != 0) + item.SteamSellRadio = Math.Round(item.BuffSellPrice / (item.SteamSellPrice / 1.15), 2); else + item.SteamSellRadio = 0; + lock (o2) { - //中国时间 - item.UpdateTime = DateTime.UtcNow.AddHours(8); - context.Entry(querry).CurrentValues.SetValues(item); + var querry = context.Goods.Find(item.Name); + if (querry == null) + { + context.Goods.Add(item); + } + else + { + //中国时间 + item.UpdateTime = DateTime.UtcNow.AddHours(8); + context.Entry(querry).CurrentValues.SetValues(item); + } + context.SaveChanges(); + //Console.WriteLine($"正在查询第{currentPageNum}页，物品名：{item.Name}"); + logs.Enqueue($"正在查询第{currentPageNum}页，物品名：{item.Name}"); + while (logs.Count > 20) logs.Dequeue(); } - context.SaveChanges(); - //Console.WriteLine($"正在查询第{currentPageNum}页，物品名：{item.Name}"); - logs.Enqueue($"正在查询第{currentPageNum}页，物品名：{item.Name}"); - while(logs.Count > 20) logs.Dequeue(); - } - }); + }); + } + catch (System.Exception e) + { + Console.WriteLine(e.Message); + Console.WriteLine(e.StackTrace); + continue; + } + } - catch (System.Exception e) + //保存数据库 + lock (o2) { - Console.WriteLine(e.Message); - Console.WriteLine(e.StackTrace); - continue; + context.SaveChanges(); } - - } - //保存数据库 - lock (o2) - { - context.SaveChanges(); - } - lock (o3) - { - currentSyncThread--; - } + lock (o3) + { + currentSyncThread--; + } + }); }); - }); + } + catch (Exception ex) + { + //访问 BUFF 错误 + Console.WriteLine(ex.Message); + Thread.Sleep(TimeSpan.FromSeconds(10)); + lock (o) + { + if (pageNum != 1) pageNum--; + } + lock (o3) + { + currentSyncThread--; + } + return; + } } }); }