Scrapping running hadless or viewable

2021-01-17 23:10:07 +01:00
parent fb86fd60cc
commit 2dd9eae9c3
8 changed files with 301 additions and 2 deletions
--- a/StockDal/StockMarketRepository.cs
+++ b/StockDal/StockMarketRepository.cs
@ -0,0 +1,192 @@
+using OpenQA.Selenium;
+using OpenQA.Selenium.Chrome;
+using StockDal.Interface;
+using StockDomain;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace StockDal
+{
+    public class StockMarketRepository : IStockMarketRepository
+    {
+        public Dictionary<string, DiTraderStockRow> StockMarketList { get; set; }
+        public DiTraderStockRow SaveRow { get; set; }
+        public StringBuilder TextResults { get; set; }
+        public bool ViewBrowser { get; set; }
+
+        public IWebDriver driver;
+
+        public StockMarketRepository()
+        {
+            ViewBrowser = false;
+        }
+
+        private void Find_Data()
+        {
+            TextResults = new StringBuilder();
+            StockMarketList = new Dictionary<string, DiTraderStockRow>();
+            IList<IWebElement> searchElements = driver.FindElements(By.TagName("tbody"));
+            foreach (IWebElement i in searchElements)
+            {
+
+                HtmlAgilityPack.HtmlDocument htmlDocument = new HtmlAgilityPack.HtmlDocument();
+                var text = i.GetAttribute("innerHTML");
+                htmlDocument.LoadHtml(text);
+                var inputs = htmlDocument.DocumentNode.Descendants("tr").ToList();
+                foreach (var items in inputs)
+                {
+                    HtmlAgilityPack.HtmlDocument htmlDocument1 = new HtmlAgilityPack.HtmlDocument();
+                    htmlDocument1.LoadHtml(items.InnerHtml);
+                    var tds = htmlDocument1.DocumentNode.Descendants("td").ToList();
+                    var appendText = "";
+                    var fNr = 0;
+                    foreach (var item in tds)
+                    {
+                        var intext = item.InnerText.Replace("\r\n", "");
+                        appendText += appendText.Length == 0 ? intext : " " + intext;
+                        if (tds.Count == 10)
+                        {
+                            AddValueToListRow(fNr++, intext);
+                        }
+                    }
+                    if (!string.IsNullOrEmpty(appendText))
+                    {
+                        TextResults.Append(appendText + "\r\n");
+                    }
+                }
+
+                TextResults.Append("\r\n");
+            }
+            var oxe = StockMarketList;
+        }
+        private void AddValueToListRow(int pos, string value)
+        {
+            switch (pos)
+            {
+                case 0:
+                    {
+                        SaveRow = new DiTraderStockRow();
+                        SaveRow.StockName = value;
+                        break;
+                    }
+                case 1:
+                    {
+                        SaveRow.ProcChange = string.IsNullOrWhiteSpace(value) ? 0 : decimal.Parse(value);
+                        break;
+                    }
+                case 2:
+                    {
+                        SaveRow.RealChange = string.IsNullOrWhiteSpace(value) ? 0 : decimal.Parse(value);
+                        break;
+                    }
+                case 3:
+                    {
+                        SaveRow.BuyPrice = string.IsNullOrWhiteSpace(value) ? 0 : decimal.Parse(value);
+                        break;
+                    }
+                case 4:
+                    {
+                        SaveRow.SellPrice = string.IsNullOrWhiteSpace(value) ? 0 : decimal.Parse(value);
+                        break;
+                    }
+                case 5:
+                    {
+                        SaveRow.LatestPrice = string.IsNullOrWhiteSpace(value) ? 0 : decimal.Parse(value);
+                        break;
+                    }
+                case 6:
+                    {
+                        SaveRow.HighestPrice = string.IsNullOrWhiteSpace(value) ? 0 : decimal.Parse(value);
+                        break;
+                    }
+                case 7:
+                    {
+                        SaveRow.LowestPrice = string.IsNullOrWhiteSpace(value) ? 0 : decimal.Parse(value);
+                        break;
+                    }
+                case 8:
+                    {
+
+                        SaveRow.Volume = string.IsNullOrWhiteSpace(value) ? 0 : long.Parse(value.Replace(" ", ""));
+                        break;
+                    }
+                case 9:
+                    {
+                        SaveRow.TimeOfDay = TimeSpan.Parse(value);
+                        //StockMarketList.Add(SaveRow.StockName, SaveRow);
+                        try
+                        {
+                            StockMarketList.Add(SaveRow.StockName, SaveRow);
+                        }
+                        catch (ArgumentException ae)
+                        {
+                            StockMarketList.Add(SaveRow.StockName + "-2", SaveRow);
+                        }
+                        break;
+                    }
+                default:
+                    break;
+            }
+        }
+
+        private void OpenBrowser()
+        {
+            var driverService = ChromeDriverService.CreateDefaultService();
+            driverService.HideCommandPromptWindow = true;
+
+            if (ViewBrowser)
+            {
+                driver = new ChromeDriver(driverService);
+            }
+            else
+            {
+                var options = new ChromeOptions();
+                options.AddArgument("headless");
+                driver = new ChromeDriver(driverService, options);
+            }
+
+            try
+            {
+                //var url0 = "https://money.cnn.com/data/hotstocks/index.html";
+                var url = "https://trader.di.se/index.php/stocklist/index/2055?list=7126";
+                //var url1 = "https://www.finansportalen.se/aktiekurser/";
+
+                driver.Navigate().GoToUrl(url);
+            }
+            catch
+            {
+                throw;
+            }
+
+        }
+
+        public void LoadStockMarketList()
+        {
+            OpenBrowser();
+            Find_Data();
+        }
+
+
+        public void LoadStockMarketList(bool viewBrowser)
+        {
+            ViewBrowser = viewBrowser;
+            OpenBrowser();
+            Find_Data();
+        }
+
+        public void RefreshMarketList()
+        {
+            driver.Navigate().Refresh();
+            Find_Data();
+        }
+
+        public void Clean()
+        {
+            driver.Quit();
+        }
+
+    }
+}