Scrapping running hadless or viewable

RepositoryPattern/Program.cs

@@ -22,7 +22,7 @@ namespace RepositoryPattern
             Application.EnableVisualStyles();
             Application.SetCompatibleTextRenderingDefault(false);
             Container = Configure();
-            Application.Run(new frmInitial(Container.Resolve<IProductRepository>(),Container.Resolve<IStockMemberRepository>()));
+            Application.Run(new frmInitial(Container.Resolve<IProductRepository>(),Container.Resolve<IStockMemberRepository>(),Container.Resolve<IStockMarketRepository>()));
         }
         /// <summary>
         /// Setting dependency injection
@@ -33,6 +33,7 @@ namespace RepositoryPattern
             var builder = new ContainerBuilder();
             builder.RegisterType<ProductRepository>().As<IProductRepository>();
             builder.RegisterType<StockMemberRepository>().As<IStockMemberRepository>();
+            builder.RegisterType<StockMarketRepository>().As<IStockMarketRepository>();
             builder.RegisterType<frmInitial>();
             return builder.Build();
         }

RepositoryPattern/RepositoryPattern.csproj

@@ -9,6 +9,8 @@
   <ItemGroup>
     <PackageReference Include="Autofac" Version="6.1.0" />
     <PackageReference Include="Dapper" Version="2.0.78" />
+    <PackageReference Include="HtmlAgilityPack" Version="1.11.29" />
+    <PackageReference Include="Selenium.WebDriver.ChromeDriver" Version="87.0.4280.8800" />
     <PackageReference Include="System.Data.SqlClient" Version="4.8.2" />
   </ItemGroup>
 

RepositoryPattern/frmInitial.Designer.cs

@@ -35,6 +35,10 @@ namespace RepositoryPattern
             this.btnReload = new System.Windows.Forms.Button();
             this.rdbStock = new System.Windows.Forms.RadioButton();
             this.rdbNorth = new System.Windows.Forms.RadioButton();
+            this.btnTestScrapFunction = new System.Windows.Forms.Button();
+            this.lblStockRows = new System.Windows.Forms.Label();
+            this.button1 = new System.Windows.Forms.Button();
+            this.chbShowBrowser = new System.Windows.Forms.CheckBox();
             ((System.ComponentModel.ISupportInitialize)(this.dataGridView)).BeginInit();
             this.gB1.SuspendLayout();
             this.SuspendLayout();
@@ -102,11 +106,55 @@ namespace RepositoryPattern
             this.rdbNorth.Text = "NorthWind";
             this.rdbNorth.UseVisualStyleBackColor = true;
             // 
+            // btnTestScrapFunction
+            // 
+            this.btnTestScrapFunction.Location = new System.Drawing.Point(12, 452);
+            this.btnTestScrapFunction.Name = "btnTestScrapFunction";
+            this.btnTestScrapFunction.Size = new System.Drawing.Size(75, 23);
+            this.btnTestScrapFunction.TabIndex = 4;
+            this.btnTestScrapFunction.Text = "LoadScrap";
+            this.btnTestScrapFunction.UseVisualStyleBackColor = true;
+            this.btnTestScrapFunction.Click += new System.EventHandler(this.btnTestScrapFunction_Click);
+            // 
+            // lblStockRows
+            // 
+            this.lblStockRows.AutoSize = true;
+            this.lblStockRows.Location = new System.Drawing.Point(93, 456);
+            this.lblStockRows.Name = "lblStockRows";
+            this.lblStockRows.Size = new System.Drawing.Size(14, 15);
+            this.lblStockRows.TabIndex = 5;
+            this.lblStockRows.Text = "#";
+            // 
+            // button1
+            // 
+            this.button1.Location = new System.Drawing.Point(157, 452);
+            this.button1.Name = "button1";
+            this.button1.Size = new System.Drawing.Size(75, 23);
+            this.button1.TabIndex = 6;
+            this.button1.Text = "ReLoad";
+            this.button1.UseVisualStyleBackColor = true;
+            this.button1.Click += new System.EventHandler(this.button1_Click);
+            // 
+            // chbShowBrowser
+            // 
+            this.chbShowBrowser.AutoSize = true;
+            this.chbShowBrowser.Location = new System.Drawing.Point(12, 478);
+            this.chbShowBrowser.Name = "chbShowBrowser";
+            this.chbShowBrowser.Size = new System.Drawing.Size(100, 22);
+            this.chbShowBrowser.TabIndex = 7;
+            this.chbShowBrowser.Text = "Show Browser";
+            this.chbShowBrowser.UseCompatibleTextRendering = true;
+            this.chbShowBrowser.UseVisualStyleBackColor = true;
+            // 
             // frmInitial
             // 
             this.AutoScaleDimensions = new System.Drawing.SizeF(7F, 15F);
             this.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font;
             this.ClientSize = new System.Drawing.Size(800, 509);
+            this.Controls.Add(this.chbShowBrowser);
+            this.Controls.Add(this.button1);
+            this.Controls.Add(this.lblStockRows);
+            this.Controls.Add(this.btnTestScrapFunction);
             this.Controls.Add(this.gB1);
             this.Controls.Add(this.lblTotalRecords);
             this.Controls.Add(this.dataGridView);
@@ -114,6 +162,7 @@ namespace RepositoryPattern
             this.Name = "frmInitial";
             this.StartPosition = System.Windows.Forms.FormStartPosition.CenterScreen;
             this.Text = "Repository Pattern";
+            this.FormClosing += new System.Windows.Forms.FormClosingEventHandler(this.frmInitial_FormClosing);
             this.Load += new System.EventHandler(this.Form1_Load);
             ((System.ComponentModel.ISupportInitialize)(this.dataGridView)).EndInit();
             this.gB1.ResumeLayout(false);
@@ -131,6 +180,10 @@ namespace RepositoryPattern
         private System.Windows.Forms.RadioButton rdbStock;
         private System.Windows.Forms.RadioButton rdbNorth;
         private System.Windows.Forms.Button btnReload;
+        private System.Windows.Forms.Button btnTestScrapFunction;
+        private System.Windows.Forms.Label lblStockRows;
+        private System.Windows.Forms.Button button1;
+        private System.Windows.Forms.CheckBox chbShowBrowser;
     }
 }
 

RepositoryPattern/frmInitial.cs

@@ -15,12 +15,14 @@ namespace RepositoryPattern
     {
         IProductRepository _productRepository;
         private readonly IStockMemberRepository _stockMemberRepository;
+        private readonly IStockMarketRepository _stockMarketRepository;
 
-        public frmInitial(IProductRepository productRepository,IStockMemberRepository stockMemberRepository)
+        public frmInitial(IProductRepository productRepository,IStockMemberRepository stockMemberRepository,IStockMarketRepository stockMarketRepository)
         {
             InitializeComponent();
             _productRepository = productRepository;
             _stockMemberRepository = stockMemberRepository;
+            _stockMarketRepository = stockMarketRepository;
         }
 
         private void Form1_Load(object sender, EventArgs e)
@@ -44,5 +46,31 @@ namespace RepositoryPattern
         {
             ReloadData();
         }
+
+        private void btnTestScrapFunction_Click(object sender, EventArgs e)
+        {
+            if (rdbStock.Checked)
+            {
+                _stockMarketRepository.LoadStockMarketList(chbShowBrowser.Checked);
+                var stocklist = _stockMarketRepository.StockMarketList;
+                lblStockRows.Text = stocklist.Count().ToString();
+            }
+        }
+
+        private void frmInitial_FormClosing(object sender, FormClosingEventArgs e)
+        {
+            _stockMarketRepository.Clean();
+        }
+
+        private void button1_Click(object sender, EventArgs e)
+        {
+            if (rdbStock.Checked)
+            {
+                lblStockRows.Text = "";
+                _stockMarketRepository.RefreshMarketList();
+                var stocklist = _stockMarketRepository.StockMarketList;
+                lblStockRows.Text = stocklist.Count().ToString();
+            }
+        }
     }
 }

StockDal.Interface/IStockMarketRepository.cs

@@ -0,0 +1,20 @@
+using StockDomain;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace StockDal.Interface
+{
+    public interface IStockMarketRepository
+    {
+        Dictionary<string, DiTraderStockRow> StockMarketList { get; set; }
+        bool ViewBrowser { get; set; }
+
+        void Clean();
+        void LoadStockMarketList();
+        void LoadStockMarketList(bool viewBrowser);
+        void RefreshMarketList();
+    }
+}

StockDal.Interface/StockDal.Interface.csproj

@@ -6,6 +6,7 @@
 
   <ItemGroup>
     <PackageReference Include="Dapper" Version="2.0.78" />
+    <PackageReference Include="HtmlAgilityPack" Version="1.11.29" />
   </ItemGroup>
 
   <ItemGroup>

StockDal/StockDal.csproj

@@ -6,6 +6,8 @@
 
   <ItemGroup>
     <PackageReference Include="Dapper" Version="2.0.78" />
+    <PackageReference Include="HtmlAgilityPack" Version="1.11.29" />
+    <PackageReference Include="Selenium.WebDriverBackedSelenium" Version="3.141.0" />
     <PackageReference Include="System.Configuration.ConfigurationManager" Version="5.0.0" />
     <PackageReference Include="System.Data.SqlClient" Version="4.8.2" />
   </ItemGroup>

StockDal/StockMarketRepository.cs

@@ -0,0 +1,192 @@
+using OpenQA.Selenium;
+using OpenQA.Selenium.Chrome;
+using StockDal.Interface;
+using StockDomain;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace StockDal
+{
+    public class StockMarketRepository : IStockMarketRepository
+    {
+        public Dictionary<string, DiTraderStockRow> StockMarketList { get; set; }
+        public DiTraderStockRow SaveRow { get; set; }
+        public StringBuilder TextResults { get; set; }
+        public bool ViewBrowser { get; set; }
+
+        public IWebDriver driver;
+
+        public StockMarketRepository()
+        {
+            ViewBrowser = false;
+        }
+
+        private void Find_Data()
+        {
+            TextResults = new StringBuilder();
+            StockMarketList = new Dictionary<string, DiTraderStockRow>();
+            IList<IWebElement> searchElements = driver.FindElements(By.TagName("tbody"));
+            foreach (IWebElement i in searchElements)
+            {
+
+                HtmlAgilityPack.HtmlDocument htmlDocument = new HtmlAgilityPack.HtmlDocument();
+                var text = i.GetAttribute("innerHTML");
+                htmlDocument.LoadHtml(text);
+                var inputs = htmlDocument.DocumentNode.Descendants("tr").ToList();
+                foreach (var items in inputs)
+                {
+                    HtmlAgilityPack.HtmlDocument htmlDocument1 = new HtmlAgilityPack.HtmlDocument();
+                    htmlDocument1.LoadHtml(items.InnerHtml);
+                    var tds = htmlDocument1.DocumentNode.Descendants("td").ToList();
+                    var appendText = "";
+                    var fNr = 0;
+                    foreach (var item in tds)
+                    {
+                        var intext = item.InnerText.Replace("\r\n", "");
+                        appendText += appendText.Length == 0 ? intext : " " + intext;
+                        if (tds.Count == 10)
+                        {
+                            AddValueToListRow(fNr++, intext);
+                        }
+                    }
+                    if (!string.IsNullOrEmpty(appendText))
+                    {
+                        TextResults.Append(appendText + "\r\n");
+                    }
+                }
+
+                TextResults.Append("\r\n");
+            }
+            var oxe = StockMarketList;
+        }
+        private void AddValueToListRow(int pos, string value)
+        {
+            switch (pos)
+            {
+                case 0:
+                    {
+                        SaveRow = new DiTraderStockRow();
+                        SaveRow.StockName = value;
+                        break;
+                    }
+                case 1:
+                    {
+                        SaveRow.ProcChange = string.IsNullOrWhiteSpace(value) ? 0 : decimal.Parse(value);
+                        break;
+                    }
+                case 2:
+                    {
+                        SaveRow.RealChange = string.IsNullOrWhiteSpace(value) ? 0 : decimal.Parse(value);
+                        break;
+                    }
+                case 3:
+                    {
+                        SaveRow.BuyPrice = string.IsNullOrWhiteSpace(value) ? 0 : decimal.Parse(value);
+                        break;
+                    }
+                case 4:
+                    {
+                        SaveRow.SellPrice = string.IsNullOrWhiteSpace(value) ? 0 : decimal.Parse(value);
+                        break;
+                    }
+                case 5:
+                    {
+                        SaveRow.LatestPrice = string.IsNullOrWhiteSpace(value) ? 0 : decimal.Parse(value);
+                        break;
+                    }
+                case 6:
+                    {
+                        SaveRow.HighestPrice = string.IsNullOrWhiteSpace(value) ? 0 : decimal.Parse(value);
+                        break;
+                    }
+                case 7:
+                    {
+                        SaveRow.LowestPrice = string.IsNullOrWhiteSpace(value) ? 0 : decimal.Parse(value);
+                        break;
+                    }
+                case 8:
+                    {
+
+                        SaveRow.Volume = string.IsNullOrWhiteSpace(value) ? 0 : long.Parse(value.Replace(" ", ""));
+                        break;
+                    }
+                case 9:
+                    {
+                        SaveRow.TimeOfDay = TimeSpan.Parse(value);
+                        //StockMarketList.Add(SaveRow.StockName, SaveRow);
+                        try
+                        {
+                            StockMarketList.Add(SaveRow.StockName, SaveRow);
+                        }
+                        catch (ArgumentException ae)
+                        {
+                            StockMarketList.Add(SaveRow.StockName + "-2", SaveRow);
+                        }
+                        break;
+                    }
+                default:
+                    break;
+            }
+        }
+
+        private void OpenBrowser()
+        {
+            var driverService = ChromeDriverService.CreateDefaultService();
+            driverService.HideCommandPromptWindow = true;
+
+            if (ViewBrowser)
+            {
+                driver = new ChromeDriver(driverService);
+            }
+            else
+            {
+                var options = new ChromeOptions();
+                options.AddArgument("headless");
+                driver = new ChromeDriver(driverService, options);
+            }
+
+            try
+            {
+                //var url0 = "https://money.cnn.com/data/hotstocks/index.html";
+                var url = "https://trader.di.se/index.php/stocklist/index/2055?list=7126";
+                //var url1 = "https://www.finansportalen.se/aktiekurser/";
+
+                driver.Navigate().GoToUrl(url);
+            }
+            catch
+            {
+                throw;
+            }
+
+        }
+
+        public void LoadStockMarketList()
+        {
+            OpenBrowser();
+            Find_Data();
+        }
+
+
+        public void LoadStockMarketList(bool viewBrowser)
+        {
+            ViewBrowser = viewBrowser;
+            OpenBrowser();
+            Find_Data();
+        }
+
+        public void RefreshMarketList()
+        {
+            driver.Navigate().Refresh();
+            Find_Data();
+        }
+
+        public void Clean()
+        {
+            driver.Quit();
+        }
+
+    }
+}