dags att checka in
This commit is contained in:
12
AngleSharpTest/AngleSharpTest.csproj
Normal file
12
AngleSharpTest/AngleSharpTest.csproj
Normal file
@ -0,0 +1,12 @@
|
|||||||
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
|
<PropertyGroup>
|
||||||
|
<OutputType>Exe</OutputType>
|
||||||
|
<TargetFramework>net5.0</TargetFramework>
|
||||||
|
</PropertyGroup>
|
||||||
|
|
||||||
|
<ItemGroup>
|
||||||
|
<PackageReference Include="AngleSharp" Version="0.16.0" />
|
||||||
|
</ItemGroup>
|
||||||
|
|
||||||
|
</Project>
|
||||||
25
AngleSharpTest/Program.cs
Normal file
25
AngleSharpTest/Program.cs
Normal file
@ -0,0 +1,25 @@
|
|||||||
|
using AngleSharp;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
using System;
|
||||||
|
using System.Linq;
|
||||||
|
|
||||||
|
namespace AngleSharpTest
|
||||||
|
{
|
||||||
|
class Program
|
||||||
|
{
|
||||||
|
static async Task Main(string[] args)
|
||||||
|
{
|
||||||
|
var config = Configuration.Default.WithDefaultLoader();
|
||||||
|
var address = "https://en.wikipedia.org/wiki/List_of_The_Big_Bang_Theory_episodes";
|
||||||
|
var context = BrowsingContext.New(config);
|
||||||
|
var document = await context.OpenAsync(address);
|
||||||
|
var cellSelector = "tr.vevent td:nth-child(3)";
|
||||||
|
var cells = document.QuerySelectorAll(cellSelector);
|
||||||
|
var titles = cells.Select(m => m.TextContent);
|
||||||
|
foreach (var title in titles)
|
||||||
|
{
|
||||||
|
Console.WriteLine(title.ToString()); ;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -9,9 +9,15 @@ namespace CsharpCorner
|
|||||||
static void Main(string[] args)
|
static void Main(string[] args)
|
||||||
{
|
{
|
||||||
var web = new HtmlWeb();
|
var web = new HtmlWeb();
|
||||||
var doc = web.Load("https://www.avanza.se/aktier/lista.html");
|
// var doc = web.Load("https://www.avanza.se/aktier/lista.html");
|
||||||
|
var doc = web.Load("https://www.finansportalen.se/aktiekurser/");
|
||||||
|
// var doc = web.Load("http://www.yellowpages.com/search?search_terms=Software&geo_location_terms=Sydney2C+ND");
|
||||||
|
|
||||||
|
// //*[@id="shareslist"]/table/tbody
|
||||||
|
// //*[@id="shareslist"]/table/tbody/tr[4]/td[1]/a
|
||||||
var headerNames = doc.DocumentNode
|
var headerNames = doc.DocumentNode
|
||||||
.SelectNodes("//a[@class='ellipsis']").ToList();
|
.SelectNodes("//*[@id='shareslist']/table/tbody/tr[1]/td[1]/a")
|
||||||
|
.ToList();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
8
HttpWebRequestType/HttpWebRequestType.csproj
Normal file
8
HttpWebRequestType/HttpWebRequestType.csproj
Normal file
@ -0,0 +1,8 @@
|
|||||||
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
|
<PropertyGroup>
|
||||||
|
<OutputType>Exe</OutputType>
|
||||||
|
<TargetFramework>net5.0</TargetFramework>
|
||||||
|
</PropertyGroup>
|
||||||
|
|
||||||
|
</Project>
|
||||||
29
HttpWebRequestType/Program.cs
Normal file
29
HttpWebRequestType/Program.cs
Normal file
@ -0,0 +1,29 @@
|
|||||||
|
using System;
|
||||||
|
using System.Net;
|
||||||
|
|
||||||
|
namespace HttpWebRequestType
|
||||||
|
{
|
||||||
|
class Program
|
||||||
|
{
|
||||||
|
static void Main(string[] args)
|
||||||
|
{
|
||||||
|
var url = "https://www.di.se/bors/aktier/";
|
||||||
|
|
||||||
|
Uri ourUri = new Uri(url);
|
||||||
|
|
||||||
|
// Create a 'WebRequest' object with the specified url.
|
||||||
|
WebRequest myWebRequest = WebRequest.Create( url);
|
||||||
|
|
||||||
|
// Send the 'WebRequest' and wait for response.
|
||||||
|
WebResponse myWebResponse = myWebRequest.GetResponse();
|
||||||
|
|
||||||
|
// Use "ResponseUri" property to get the actual Uri from where the response was attained.
|
||||||
|
if (ourUri.Equals(myWebResponse.ResponseUri))
|
||||||
|
Console.WriteLine("\nRequest Url : {0} was not redirected", url);
|
||||||
|
else
|
||||||
|
Console.WriteLine("\nRequest Url : {0} was redirected to {1}", url, myWebResponse.ResponseUri);
|
||||||
|
// Release resources of response object.
|
||||||
|
myWebResponse.Close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
16
MailGunTest/MailGunTest.csproj
Normal file
16
MailGunTest/MailGunTest.csproj
Normal file
@ -0,0 +1,16 @@
|
|||||||
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
|
<PropertyGroup>
|
||||||
|
<OutputType>Exe</OutputType>
|
||||||
|
<TargetFramework>net5.0</TargetFramework>
|
||||||
|
</PropertyGroup>
|
||||||
|
|
||||||
|
<ItemGroup>
|
||||||
|
<Compile Remove="Program.cs" />
|
||||||
|
</ItemGroup>
|
||||||
|
|
||||||
|
<ItemGroup>
|
||||||
|
<PackageReference Include="RestSharp" Version="106.12.0" />
|
||||||
|
</ItemGroup>
|
||||||
|
|
||||||
|
</Project>
|
||||||
12
MailGunTest/Program.cs
Normal file
12
MailGunTest/Program.cs
Normal file
@ -0,0 +1,12 @@
|
|||||||
|
using System;
|
||||||
|
|
||||||
|
namespace MailGunTest
|
||||||
|
{
|
||||||
|
class Program
|
||||||
|
{
|
||||||
|
static void Main(string[] args)
|
||||||
|
{
|
||||||
|
Console.WriteLine("Hello World!");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
39
MailGunTest/SendSimpleMailChunk.cs
Normal file
39
MailGunTest/SendSimpleMailChunk.cs
Normal file
@ -0,0 +1,39 @@
|
|||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Text;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
using System.IO;
|
||||||
|
using RestSharp;
|
||||||
|
using RestSharp.Authenticators;
|
||||||
|
|
||||||
|
namespace MailGunTest
|
||||||
|
{
|
||||||
|
public class SendSimpleMailChunk
|
||||||
|
{
|
||||||
|
public static void Main(string[] args)
|
||||||
|
{
|
||||||
|
Console.WriteLine(SendSimpleMessage().Content.ToString());
|
||||||
|
}
|
||||||
|
public static IRestResponse SendSimpleMessage()
|
||||||
|
{
|
||||||
|
RestClient client = new RestClient();
|
||||||
|
client.BaseUrl = new Uri("https://api.eu.mailgun.net/v3");
|
||||||
|
client.Authenticator = new HttpBasicAuthenticator("api", "c416cb13b5937e7c2274f9f3038101cd-c485922e-70545ed4");
|
||||||
|
|
||||||
|
RestRequest request = new RestRequest();
|
||||||
|
request.AddParameter("domain", "mg.tfoman.me", ParameterType.UrlSegment);
|
||||||
|
request.Resource = "{domain}/messages";
|
||||||
|
request.AddParameter("from", "Excited User <mailgun@mg.tfoman.me>");
|
||||||
|
request.AddParameter("to", "tommy@oeman.se");
|
||||||
|
request.AddParameter("to", "tommy.oman@mg.tfoman.me");
|
||||||
|
request.AddParameter("to", "tommy.oman@gmail.com");
|
||||||
|
request.AddParameter("subject", "Api-sent message TEST");
|
||||||
|
request.AddParameter("text", "Let's see how it works!");
|
||||||
|
request.Method = Method.POST;
|
||||||
|
return client.Execute(request);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
73
OceanNetWorks/Form1.Designer.cs
generated
Normal file
73
OceanNetWorks/Form1.Designer.cs
generated
Normal file
@ -0,0 +1,73 @@
|
|||||||
|
|
||||||
|
namespace OceanNetWorks
|
||||||
|
{
|
||||||
|
partial class Form1
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Required designer variable.
|
||||||
|
/// </summary>
|
||||||
|
private System.ComponentModel.IContainer components = null;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Clean up any resources being used.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="disposing">true if managed resources should be disposed; otherwise, false.</param>
|
||||||
|
protected override void Dispose(bool disposing)
|
||||||
|
{
|
||||||
|
if (disposing && (components != null))
|
||||||
|
{
|
||||||
|
components.Dispose();
|
||||||
|
}
|
||||||
|
base.Dispose(disposing);
|
||||||
|
}
|
||||||
|
|
||||||
|
#region Windows Form Designer generated code
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Required method for Designer support - do not modify
|
||||||
|
/// the contents of this method with the code editor.
|
||||||
|
/// </summary>
|
||||||
|
private void InitializeComponent()
|
||||||
|
{
|
||||||
|
this.scraperBox = new System.Windows.Forms.RichTextBox();
|
||||||
|
this.btnScrape = new System.Windows.Forms.Button();
|
||||||
|
this.SuspendLayout();
|
||||||
|
//
|
||||||
|
// scraperBox
|
||||||
|
//
|
||||||
|
this.scraperBox.Location = new System.Drawing.Point(12, 12);
|
||||||
|
this.scraperBox.Name = "scraperBox";
|
||||||
|
this.scraperBox.Size = new System.Drawing.Size(768, 680);
|
||||||
|
this.scraperBox.TabIndex = 0;
|
||||||
|
this.scraperBox.Text = "";
|
||||||
|
//
|
||||||
|
// btnScrape
|
||||||
|
//
|
||||||
|
this.btnScrape.Location = new System.Drawing.Point(364, 711);
|
||||||
|
this.btnScrape.Name = "btnScrape";
|
||||||
|
this.btnScrape.Size = new System.Drawing.Size(75, 23);
|
||||||
|
this.btnScrape.TabIndex = 1;
|
||||||
|
this.btnScrape.Text = "Start";
|
||||||
|
this.btnScrape.UseVisualStyleBackColor = true;
|
||||||
|
this.btnScrape.Click += new System.EventHandler(this.btnScrape_Click);
|
||||||
|
//
|
||||||
|
// Form1
|
||||||
|
//
|
||||||
|
this.AutoScaleDimensions = new System.Drawing.SizeF(7F, 15F);
|
||||||
|
this.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font;
|
||||||
|
this.ClientSize = new System.Drawing.Size(792, 758);
|
||||||
|
this.Controls.Add(this.btnScrape);
|
||||||
|
this.Controls.Add(this.scraperBox);
|
||||||
|
this.Name = "Form1";
|
||||||
|
this.Text = "WebScraper";
|
||||||
|
this.ResumeLayout(false);
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
#endregion
|
||||||
|
|
||||||
|
private System.Windows.Forms.RichTextBox scraperBox;
|
||||||
|
private System.Windows.Forms.Button btnScrape;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
107
OceanNetWorks/Form1.cs
Normal file
107
OceanNetWorks/Form1.cs
Normal file
@ -0,0 +1,107 @@
|
|||||||
|
using AngleSharp;
|
||||||
|
using AngleSharp.Dom;
|
||||||
|
using AngleSharp.Html.Dom;
|
||||||
|
using AngleSharp.Html.Parser;
|
||||||
|
using AngleSharp.Text;
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.ComponentModel;
|
||||||
|
using System.Data;
|
||||||
|
using System.Drawing;
|
||||||
|
using System.IO;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Net.Http;
|
||||||
|
using System.Text;
|
||||||
|
using System.Threading;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
using System.Windows.Forms;
|
||||||
|
|
||||||
|
namespace OceanNetWorks
|
||||||
|
{
|
||||||
|
public partial class Form1 : Form
|
||||||
|
{
|
||||||
|
public Form1()
|
||||||
|
{
|
||||||
|
InitializeComponent();
|
||||||
|
}
|
||||||
|
|
||||||
|
private string Title { get; set; }
|
||||||
|
private string Url { get; set; }
|
||||||
|
private string siteUrl = "https://www.oceannetworks.ca/news/stories";
|
||||||
|
//private string siteUrlx = "https://www.finansportalen.se/aktiekurser/";
|
||||||
|
public string[] QueryTerms { get; } = { "Ocean", "Nature", "Pollution" };
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
internal async void ScrapeWebsite()
|
||||||
|
{
|
||||||
|
//var config = Configuration.Default
|
||||||
|
// .WithJs(); // from AngleSharp.Js
|
||||||
|
//var context = BrowsingContext.New(config);
|
||||||
|
|
||||||
|
CancellationTokenSource cancellationToken = new CancellationTokenSource();
|
||||||
|
HttpClient httpClient = new HttpClient();
|
||||||
|
HttpResponseMessage request = await httpClient.GetAsync(siteUrl);
|
||||||
|
|
||||||
|
cancellationToken.Token.ThrowIfCancellationRequested();
|
||||||
|
|
||||||
|
Stream response = await request.Content.ReadAsStreamAsync();
|
||||||
|
cancellationToken.Token.ThrowIfCancellationRequested();
|
||||||
|
|
||||||
|
HtmlParser parser = new HtmlParser();
|
||||||
|
|
||||||
|
IHtmlDocument document = parser.ParseDocument(response);
|
||||||
|
|
||||||
|
GetScrapeResults(document);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void GetScrapeResults(IHtmlDocument document)
|
||||||
|
{
|
||||||
|
IEnumerable<IElement> articleLink = null;
|
||||||
|
|
||||||
|
foreach (var term in QueryTerms)
|
||||||
|
{
|
||||||
|
articleLink = document.All.Where(x =>
|
||||||
|
x.ClassName == "views-field views-field-nothing" &&
|
||||||
|
(x.ParentElement.InnerHtml.Contains(term) || x.ParentElement.InnerHtml.Contains(term.ToLower())));
|
||||||
|
}
|
||||||
|
|
||||||
|
if (articleLink.Any())
|
||||||
|
{
|
||||||
|
PrintResults(articleLink);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void PrintResults(IEnumerable<IElement> articleLink)
|
||||||
|
{
|
||||||
|
// Clean Up Results: See Next Step
|
||||||
|
foreach (var element in articleLink)
|
||||||
|
{
|
||||||
|
CleanUpResults(element);
|
||||||
|
scraperBox.Text = $"{Title} - {Url}{Environment.NewLine}";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void CleanUpResults(IElement result)
|
||||||
|
{
|
||||||
|
string htmlResult = result.InnerHtml.ReplaceFirst(" <span class=\"field-content\"><div><a href=\"", "https://www.oceannetworks.ca");
|
||||||
|
htmlResult = htmlResult.ReplaceFirst("\">", "*");
|
||||||
|
htmlResult = htmlResult.ReplaceFirst("</a></div>\n<div class=\"article-title-top\">", "-");
|
||||||
|
htmlResult = htmlResult.ReplaceFirst("</div>\n<hr></span> ", "");
|
||||||
|
|
||||||
|
SplitResults(htmlResult);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void SplitResults(string htmlResult)
|
||||||
|
{
|
||||||
|
string[] splitResults = htmlResult.Split('*');
|
||||||
|
Url = splitResults[0];
|
||||||
|
Title = splitResults[1];
|
||||||
|
}
|
||||||
|
|
||||||
|
private void btnScrape_Click(object sender, EventArgs e)
|
||||||
|
{
|
||||||
|
ScrapeWebsite();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
60
OceanNetWorks/Form1.resx
Normal file
60
OceanNetWorks/Form1.resx
Normal file
@ -0,0 +1,60 @@
|
|||||||
|
<root>
|
||||||
|
<xsd:schema id="root" xmlns="" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:msdata="urn:schemas-microsoft-com:xml-msdata">
|
||||||
|
<xsd:import namespace="http://www.w3.org/XML/1998/namespace" />
|
||||||
|
<xsd:element name="root" msdata:IsDataSet="true">
|
||||||
|
<xsd:complexType>
|
||||||
|
<xsd:choice maxOccurs="unbounded">
|
||||||
|
<xsd:element name="metadata">
|
||||||
|
<xsd:complexType>
|
||||||
|
<xsd:sequence>
|
||||||
|
<xsd:element name="value" type="xsd:string" minOccurs="0" />
|
||||||
|
</xsd:sequence>
|
||||||
|
<xsd:attribute name="name" use="required" type="xsd:string" />
|
||||||
|
<xsd:attribute name="type" type="xsd:string" />
|
||||||
|
<xsd:attribute name="mimetype" type="xsd:string" />
|
||||||
|
<xsd:attribute ref="xml:space" />
|
||||||
|
</xsd:complexType>
|
||||||
|
</xsd:element>
|
||||||
|
<xsd:element name="assembly">
|
||||||
|
<xsd:complexType>
|
||||||
|
<xsd:attribute name="alias" type="xsd:string" />
|
||||||
|
<xsd:attribute name="name" type="xsd:string" />
|
||||||
|
</xsd:complexType>
|
||||||
|
</xsd:element>
|
||||||
|
<xsd:element name="data">
|
||||||
|
<xsd:complexType>
|
||||||
|
<xsd:sequence>
|
||||||
|
<xsd:element name="value" type="xsd:string" minOccurs="0" msdata:Ordinal="1" />
|
||||||
|
<xsd:element name="comment" type="xsd:string" minOccurs="0" msdata:Ordinal="2" />
|
||||||
|
</xsd:sequence>
|
||||||
|
<xsd:attribute name="name" type="xsd:string" use="required" msdata:Ordinal="1" />
|
||||||
|
<xsd:attribute name="type" type="xsd:string" msdata:Ordinal="3" />
|
||||||
|
<xsd:attribute name="mimetype" type="xsd:string" msdata:Ordinal="4" />
|
||||||
|
<xsd:attribute ref="xml:space" />
|
||||||
|
</xsd:complexType>
|
||||||
|
</xsd:element>
|
||||||
|
<xsd:element name="resheader">
|
||||||
|
<xsd:complexType>
|
||||||
|
<xsd:sequence>
|
||||||
|
<xsd:element name="value" type="xsd:string" minOccurs="0" msdata:Ordinal="1" />
|
||||||
|
</xsd:sequence>
|
||||||
|
<xsd:attribute name="name" type="xsd:string" use="required" />
|
||||||
|
</xsd:complexType>
|
||||||
|
</xsd:element>
|
||||||
|
</xsd:choice>
|
||||||
|
</xsd:complexType>
|
||||||
|
</xsd:element>
|
||||||
|
</xsd:schema>
|
||||||
|
<resheader name="resmimetype">
|
||||||
|
<value>text/microsoft-resx</value>
|
||||||
|
</resheader>
|
||||||
|
<resheader name="version">
|
||||||
|
<value>2.0</value>
|
||||||
|
</resheader>
|
||||||
|
<resheader name="reader">
|
||||||
|
<value>System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
|
||||||
|
</resheader>
|
||||||
|
<resheader name="writer">
|
||||||
|
<value>System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
|
||||||
|
</resheader>
|
||||||
|
</root>
|
||||||
13
OceanNetWorks/OceanNetWorks.csproj
Normal file
13
OceanNetWorks/OceanNetWorks.csproj
Normal file
@ -0,0 +1,13 @@
|
|||||||
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
|
<PropertyGroup>
|
||||||
|
<OutputType>WinExe</OutputType>
|
||||||
|
<TargetFramework>net5.0-windows</TargetFramework>
|
||||||
|
<UseWindowsForms>true</UseWindowsForms>
|
||||||
|
</PropertyGroup>
|
||||||
|
|
||||||
|
<ItemGroup>
|
||||||
|
<PackageReference Include="AngleSharp" Version="0.12.1" />
|
||||||
|
</ItemGroup>
|
||||||
|
|
||||||
|
</Project>
|
||||||
23
OceanNetWorks/Program.cs
Normal file
23
OceanNetWorks/Program.cs
Normal file
@ -0,0 +1,23 @@
|
|||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
using System.Windows.Forms;
|
||||||
|
|
||||||
|
namespace OceanNetWorks
|
||||||
|
{
|
||||||
|
static class Program
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// The main entry point for the application.
|
||||||
|
/// </summary>
|
||||||
|
[STAThread]
|
||||||
|
static void Main()
|
||||||
|
{
|
||||||
|
Application.SetHighDpiMode(HighDpiMode.SystemAware);
|
||||||
|
Application.EnableVisualStyles();
|
||||||
|
Application.SetCompatibleTextRenderingDefault(false);
|
||||||
|
Application.Run(new Form1());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
14
PuppeteerSharpTest/DtoEst.cs
Normal file
14
PuppeteerSharpTest/DtoEst.cs
Normal file
@ -0,0 +1,14 @@
|
|||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Text;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
|
||||||
|
namespace PuppeteerSharpTest
|
||||||
|
{
|
||||||
|
public class DtoEst
|
||||||
|
{
|
||||||
|
public decimal EstBetrag { get; set; }
|
||||||
|
public decimal EstProzentsatz { get; set; }
|
||||||
|
}
|
||||||
|
}
|
||||||
87
PuppeteerSharpTest/Form1.Designer.cs
generated
Normal file
87
PuppeteerSharpTest/Form1.Designer.cs
generated
Normal file
@ -0,0 +1,87 @@
|
|||||||
|
|
||||||
|
namespace PuppeteerSharpTest
|
||||||
|
{
|
||||||
|
partial class Form1
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Required designer variable.
|
||||||
|
/// </summary>
|
||||||
|
private System.ComponentModel.IContainer components = null;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Clean up any resources being used.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="disposing">true if managed resources should be disposed; otherwise, false.</param>
|
||||||
|
protected override void Dispose(bool disposing)
|
||||||
|
{
|
||||||
|
if (disposing && (components != null))
|
||||||
|
{
|
||||||
|
components.Dispose();
|
||||||
|
}
|
||||||
|
base.Dispose(disposing);
|
||||||
|
}
|
||||||
|
|
||||||
|
#region Windows Form Designer generated code
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Required method for Designer support - do not modify
|
||||||
|
/// the contents of this method with the code editor.
|
||||||
|
/// </summary>
|
||||||
|
private void InitializeComponent()
|
||||||
|
{
|
||||||
|
this.rtbWatcher = new System.Windows.Forms.RichTextBox();
|
||||||
|
this.btnScrape = new System.Windows.Forms.Button();
|
||||||
|
this.txtNettoUmsatz = new System.Windows.Forms.TextBox();
|
||||||
|
this.SuspendLayout();
|
||||||
|
//
|
||||||
|
// rtbWatcher
|
||||||
|
//
|
||||||
|
this.rtbWatcher.Anchor = ((System.Windows.Forms.AnchorStyles)((((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Bottom)
|
||||||
|
| System.Windows.Forms.AnchorStyles.Left)
|
||||||
|
| System.Windows.Forms.AnchorStyles.Right)));
|
||||||
|
this.rtbWatcher.Location = new System.Drawing.Point(12, 12);
|
||||||
|
this.rtbWatcher.Name = "rtbWatcher";
|
||||||
|
this.rtbWatcher.Size = new System.Drawing.Size(776, 494);
|
||||||
|
this.rtbWatcher.TabIndex = 0;
|
||||||
|
this.rtbWatcher.Text = "";
|
||||||
|
//
|
||||||
|
// btnScrape
|
||||||
|
//
|
||||||
|
this.btnScrape.Location = new System.Drawing.Point(377, 507);
|
||||||
|
this.btnScrape.Name = "btnScrape";
|
||||||
|
this.btnScrape.Size = new System.Drawing.Size(75, 23);
|
||||||
|
this.btnScrape.TabIndex = 1;
|
||||||
|
this.btnScrape.Text = "Start";
|
||||||
|
this.btnScrape.UseVisualStyleBackColor = true;
|
||||||
|
this.btnScrape.Click += new System.EventHandler(this.btnScrape_Click);
|
||||||
|
//
|
||||||
|
// txtNettoUmsatz
|
||||||
|
//
|
||||||
|
this.txtNettoUmsatz.Location = new System.Drawing.Point(84, 513);
|
||||||
|
this.txtNettoUmsatz.Name = "txtNettoUmsatz";
|
||||||
|
this.txtNettoUmsatz.Size = new System.Drawing.Size(100, 23);
|
||||||
|
this.txtNettoUmsatz.TabIndex = 2;
|
||||||
|
//
|
||||||
|
// Form1
|
||||||
|
//
|
||||||
|
this.AutoScaleDimensions = new System.Drawing.SizeF(7F, 15F);
|
||||||
|
this.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font;
|
||||||
|
this.ClientSize = new System.Drawing.Size(800, 534);
|
||||||
|
this.Controls.Add(this.txtNettoUmsatz);
|
||||||
|
this.Controls.Add(this.btnScrape);
|
||||||
|
this.Controls.Add(this.rtbWatcher);
|
||||||
|
this.Name = "Form1";
|
||||||
|
this.Text = "Puppeteer";
|
||||||
|
this.ResumeLayout(false);
|
||||||
|
this.PerformLayout();
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
#endregion
|
||||||
|
|
||||||
|
private System.Windows.Forms.RichTextBox rtbWatcher;
|
||||||
|
private System.Windows.Forms.Button btnScrape;
|
||||||
|
private System.Windows.Forms.TextBox txtNettoUmsatz;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
87
PuppeteerSharpTest/Form1.cs
Normal file
87
PuppeteerSharpTest/Form1.cs
Normal file
@ -0,0 +1,87 @@
|
|||||||
|
using PuppeteerSharp;
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.ComponentModel;
|
||||||
|
using System.Data;
|
||||||
|
using System.Drawing;
|
||||||
|
using System.Globalization;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Text;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
using System.Windows.Forms;
|
||||||
|
|
||||||
|
namespace PuppeteerSharpTest
|
||||||
|
{
|
||||||
|
public partial class Form1 : Form
|
||||||
|
{
|
||||||
|
public Form1()
|
||||||
|
{
|
||||||
|
InitializeComponent();
|
||||||
|
}
|
||||||
|
|
||||||
|
private async void btnScrape_Click(object sender, EventArgs e)
|
||||||
|
{
|
||||||
|
var result = await LoadXXXAsync();
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task<DtoEst> LoadXXXAsync()
|
||||||
|
{
|
||||||
|
// downloads chromium to the local project, needed by Puppeteer Sharp for execution, takes about 2 min!
|
||||||
|
await new BrowserFetcher().DownloadAsync();
|
||||||
|
// await new BrowserFetcher().DownloadAsync(DefaultChromiumRevision);
|
||||||
|
|
||||||
|
DtoEst dto = new DtoEst();
|
||||||
|
//var url = "https://www.xxxxxx.xhtml";
|
||||||
|
var url = "https://www.bmf-steuerrechner.de/ekst/eingabeformekst.xhtml";
|
||||||
|
|
||||||
|
#region SELECTORS
|
||||||
|
// https://stackoverflow.com/questions/45110893/select-elements-by-attributes-with-colon
|
||||||
|
// use css escapes: https://mothereff.in/css-escapes
|
||||||
|
string id_checkbox_IsMarried = "bmf_form_ekst:ekst_pv:0";
|
||||||
|
string id_input_Umsatz = "bmf_form_ekst:ekst_zve";
|
||||||
|
string id_button_Submit2 = @"#bmf_form_ekst\3A income_ekst";
|
||||||
|
string NettoUmsatz = txtNettoUmsatz.Text;
|
||||||
|
#endregion
|
||||||
|
|
||||||
|
//// This statement downloads and installs chromium
|
||||||
|
//await new BrowserFetcher().DownloadAsync(BrowserFetcher.DefaultRevision);
|
||||||
|
var browser = await Puppeteer.LaunchAsync(new LaunchOptions
|
||||||
|
{
|
||||||
|
Headless = true
|
||||||
|
});
|
||||||
|
|
||||||
|
using (var page = await browser.NewPageAsync())
|
||||||
|
{
|
||||||
|
// load page
|
||||||
|
await page.GoToAsync(url);
|
||||||
|
// wait for element to be there
|
||||||
|
await page.WaitForExpressionAsync($"document.getElementById('{id_input_Umsatz}')!=null");
|
||||||
|
await page.EvaluateExpressionAsync($"document.getElementById('{id_input_Umsatz}').value = '{NettoUmsatz}';");
|
||||||
|
// get element value
|
||||||
|
var val = await page.EvaluateFunctionAsync<string>($"()=>document.getElementById('{id_input_Umsatz}').value");
|
||||||
|
// wait for element to be there
|
||||||
|
await page.WaitForExpressionAsync($"document.getElementById('{id_checkbox_IsMarried}')!=null");
|
||||||
|
await page.EvaluateExpressionAsync($"document.getElementById('{id_checkbox_IsMarried}').checked = false;");
|
||||||
|
|
||||||
|
await page.ClickAsync(id_button_Submit2);
|
||||||
|
// wait for redirect
|
||||||
|
await page.WaitForNavigationAsync();
|
||||||
|
// get value from element
|
||||||
|
var strValueEst = await page.EvaluateFunctionAsync<string>("()=>document.querySelector('#ui-id-4 > div.ekst_ergebnis > div:nth-child(6) > table > tbody > tr:nth-child(3) > td:nth-child(2) > strong').textContent");
|
||||||
|
// get value from element
|
||||||
|
var strValueEstPercent = await page.EvaluateFunctionAsync<string>("()=>document.querySelector('#ui-id-4 > div.ekst_ergebnis > div:nth-child(6) > table > tbody > tr:nth-child(3) > td:nth-child(3) > strong').textContent");
|
||||||
|
|
||||||
|
var deDE = new CultureInfo("de-DE");
|
||||||
|
|
||||||
|
Decimal.TryParse(strValueEst.TrimEnd(" Euro".ToCharArray()), NumberStyles.AllowThousands | NumberStyles.AllowDecimalPoint, deDE, out var decValueEst);
|
||||||
|
Decimal.TryParse(strValueEstPercent.TrimEnd(" %".ToCharArray()), NumberStyles.AllowThousands | NumberStyles.AllowDecimalPoint, deDE, out var decValueEstPercent);
|
||||||
|
|
||||||
|
dto.EstBetrag = decValueEst;
|
||||||
|
dto.EstProzentsatz = decValueEstPercent;
|
||||||
|
}
|
||||||
|
|
||||||
|
return await Task.FromResult(dto);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
60
PuppeteerSharpTest/Form1.resx
Normal file
60
PuppeteerSharpTest/Form1.resx
Normal file
@ -0,0 +1,60 @@
|
|||||||
|
<root>
|
||||||
|
<xsd:schema id="root" xmlns="" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:msdata="urn:schemas-microsoft-com:xml-msdata">
|
||||||
|
<xsd:import namespace="http://www.w3.org/XML/1998/namespace" />
|
||||||
|
<xsd:element name="root" msdata:IsDataSet="true">
|
||||||
|
<xsd:complexType>
|
||||||
|
<xsd:choice maxOccurs="unbounded">
|
||||||
|
<xsd:element name="metadata">
|
||||||
|
<xsd:complexType>
|
||||||
|
<xsd:sequence>
|
||||||
|
<xsd:element name="value" type="xsd:string" minOccurs="0" />
|
||||||
|
</xsd:sequence>
|
||||||
|
<xsd:attribute name="name" use="required" type="xsd:string" />
|
||||||
|
<xsd:attribute name="type" type="xsd:string" />
|
||||||
|
<xsd:attribute name="mimetype" type="xsd:string" />
|
||||||
|
<xsd:attribute ref="xml:space" />
|
||||||
|
</xsd:complexType>
|
||||||
|
</xsd:element>
|
||||||
|
<xsd:element name="assembly">
|
||||||
|
<xsd:complexType>
|
||||||
|
<xsd:attribute name="alias" type="xsd:string" />
|
||||||
|
<xsd:attribute name="name" type="xsd:string" />
|
||||||
|
</xsd:complexType>
|
||||||
|
</xsd:element>
|
||||||
|
<xsd:element name="data">
|
||||||
|
<xsd:complexType>
|
||||||
|
<xsd:sequence>
|
||||||
|
<xsd:element name="value" type="xsd:string" minOccurs="0" msdata:Ordinal="1" />
|
||||||
|
<xsd:element name="comment" type="xsd:string" minOccurs="0" msdata:Ordinal="2" />
|
||||||
|
</xsd:sequence>
|
||||||
|
<xsd:attribute name="name" type="xsd:string" use="required" msdata:Ordinal="1" />
|
||||||
|
<xsd:attribute name="type" type="xsd:string" msdata:Ordinal="3" />
|
||||||
|
<xsd:attribute name="mimetype" type="xsd:string" msdata:Ordinal="4" />
|
||||||
|
<xsd:attribute ref="xml:space" />
|
||||||
|
</xsd:complexType>
|
||||||
|
</xsd:element>
|
||||||
|
<xsd:element name="resheader">
|
||||||
|
<xsd:complexType>
|
||||||
|
<xsd:sequence>
|
||||||
|
<xsd:element name="value" type="xsd:string" minOccurs="0" msdata:Ordinal="1" />
|
||||||
|
</xsd:sequence>
|
||||||
|
<xsd:attribute name="name" type="xsd:string" use="required" />
|
||||||
|
</xsd:complexType>
|
||||||
|
</xsd:element>
|
||||||
|
</xsd:choice>
|
||||||
|
</xsd:complexType>
|
||||||
|
</xsd:element>
|
||||||
|
</xsd:schema>
|
||||||
|
<resheader name="resmimetype">
|
||||||
|
<value>text/microsoft-resx</value>
|
||||||
|
</resheader>
|
||||||
|
<resheader name="version">
|
||||||
|
<value>2.0</value>
|
||||||
|
</resheader>
|
||||||
|
<resheader name="reader">
|
||||||
|
<value>System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
|
||||||
|
</resheader>
|
||||||
|
<resheader name="writer">
|
||||||
|
<value>System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
|
||||||
|
</resheader>
|
||||||
|
</root>
|
||||||
23
PuppeteerSharpTest/Program.cs
Normal file
23
PuppeteerSharpTest/Program.cs
Normal file
@ -0,0 +1,23 @@
|
|||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
using System.Windows.Forms;
|
||||||
|
|
||||||
|
namespace PuppeteerSharpTest
|
||||||
|
{
|
||||||
|
static class Program
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// The main entry point for the application.
|
||||||
|
/// </summary>
|
||||||
|
[STAThread]
|
||||||
|
static void Main()
|
||||||
|
{
|
||||||
|
Application.SetHighDpiMode(HighDpiMode.SystemAware);
|
||||||
|
Application.EnableVisualStyles();
|
||||||
|
Application.SetCompatibleTextRenderingDefault(false);
|
||||||
|
Application.Run(new Form1());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
13
PuppeteerSharpTest/PuppeteerSharpTest.csproj
Normal file
13
PuppeteerSharpTest/PuppeteerSharpTest.csproj
Normal file
@ -0,0 +1,13 @@
|
|||||||
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
|
<PropertyGroup>
|
||||||
|
<OutputType>WinExe</OutputType>
|
||||||
|
<TargetFramework>net5.0-windows</TargetFramework>
|
||||||
|
<UseWindowsForms>true</UseWindowsForms>
|
||||||
|
</PropertyGroup>
|
||||||
|
|
||||||
|
<ItemGroup>
|
||||||
|
<PackageReference Include="PuppeteerSharp" Version="5.0.0" />
|
||||||
|
</ItemGroup>
|
||||||
|
|
||||||
|
</Project>
|
||||||
41
Selenium/Program.cs
Normal file
41
Selenium/Program.cs
Normal file
@ -0,0 +1,41 @@
|
|||||||
|
using OpenQA.Selenium;
|
||||||
|
using OpenQA.Selenium.Chrome;
|
||||||
|
using OpenQA.Selenium.Support.UI;
|
||||||
|
using System;
|
||||||
|
using System.Drawing.Imaging;
|
||||||
|
using System.IO;
|
||||||
|
|
||||||
|
namespace Selenium
|
||||||
|
{
|
||||||
|
class Program
|
||||||
|
{
|
||||||
|
static void Main(string[] args)
|
||||||
|
{
|
||||||
|
// Initialize the Chrome Driver
|
||||||
|
using (var driver = new ChromeDriver())
|
||||||
|
{
|
||||||
|
// Go to the home page
|
||||||
|
driver.Navigate().GoToUrl("http://testing-ground.webscraping.pro/login");
|
||||||
|
|
||||||
|
// Get the page elements
|
||||||
|
var userNameField = driver.FindElementById("usr");
|
||||||
|
var userPasswordField = driver.FindElementById("pwd");
|
||||||
|
var loginButton = driver.FindElementByXPath("//input[@value='Login']");
|
||||||
|
|
||||||
|
// Type user name and password
|
||||||
|
userNameField.SendKeys("admin");
|
||||||
|
userPasswordField.SendKeys("12345");
|
||||||
|
|
||||||
|
// and click the login button
|
||||||
|
loginButton.Click();
|
||||||
|
|
||||||
|
// Extract the text and save it into result.txt
|
||||||
|
var result = driver.FindElementByXPath("//div[@id='case_login']/h3").Text;
|
||||||
|
File.WriteAllText("result.txt", result);
|
||||||
|
|
||||||
|
// Take a screenshot and save it into screen.png
|
||||||
|
driver.GetScreenshot().SaveAsFile(@"screen.png", ScreenshotImageFormat.Png);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
17
Selenium/Selenium.csproj
Normal file
17
Selenium/Selenium.csproj
Normal file
@ -0,0 +1,17 @@
|
|||||||
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
|
<PropertyGroup>
|
||||||
|
<OutputType>Exe</OutputType>
|
||||||
|
<TargetFramework>net5.0</TargetFramework>
|
||||||
|
</PropertyGroup>
|
||||||
|
|
||||||
|
<ItemGroup>
|
||||||
|
<PackageReference Include="Selenium.Chrome.WebDriver" Version="85.0.0" />
|
||||||
|
<PackageReference Include="Selenium.Support" Version="3.141.0" />
|
||||||
|
<PackageReference Include="Selenium.WebDriver" Version="3.141.0" />
|
||||||
|
<PackageReference Include="Selenium.WebDriver.ChromeDriver" Version="92.0.4515.4300" />
|
||||||
|
<PackageReference Include="Selenium.WebDriverBackedSelenium" Version="3.141.0" />
|
||||||
|
<PackageReference Include="System.Drawing.Common" Version="5.0.2" />
|
||||||
|
</ItemGroup>
|
||||||
|
|
||||||
|
</Project>
|
||||||
24
WebClientScraper/Program.cs
Normal file
24
WebClientScraper/Program.cs
Normal file
@ -0,0 +1,24 @@
|
|||||||
|
using System;
|
||||||
|
using System.Net;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
|
||||||
|
namespace WebClientScraper
|
||||||
|
{
|
||||||
|
class Program
|
||||||
|
{
|
||||||
|
static void Main(string[] args)
|
||||||
|
{
|
||||||
|
using var client = new WebClient();
|
||||||
|
|
||||||
|
client.DownloadStringCompleted += (sender, e) =>
|
||||||
|
{
|
||||||
|
Console.WriteLine(e.Result);
|
||||||
|
};
|
||||||
|
|
||||||
|
string url = "https://finansportalen.millistream.com/shares.php";
|
||||||
|
client.DownloadStringAsync(new Uri(url));
|
||||||
|
|
||||||
|
Console.ReadLine();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
8
WebClientScraper/WebClientScraper.csproj
Normal file
8
WebClientScraper/WebClientScraper.csproj
Normal file
@ -0,0 +1,8 @@
|
|||||||
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
|
<PropertyGroup>
|
||||||
|
<OutputType>Exe</OutputType>
|
||||||
|
<TargetFramework>net5.0</TargetFramework>
|
||||||
|
</PropertyGroup>
|
||||||
|
|
||||||
|
</Project>
|
||||||
15
WebFlurl/Program.cs
Normal file
15
WebFlurl/Program.cs
Normal file
@ -0,0 +1,15 @@
|
|||||||
|
using Flurl.Http;
|
||||||
|
using System;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
|
||||||
|
namespace WebFlurl
|
||||||
|
{
|
||||||
|
class Program
|
||||||
|
{
|
||||||
|
static async Task Main(string[] args)
|
||||||
|
{
|
||||||
|
string result = await "https://www.di.se/bors/aktier/".GetStringAsync();
|
||||||
|
Console.WriteLine(result);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
12
WebFlurl/WebFlurl.csproj
Normal file
12
WebFlurl/WebFlurl.csproj
Normal file
@ -0,0 +1,12 @@
|
|||||||
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
|
<PropertyGroup>
|
||||||
|
<OutputType>Exe</OutputType>
|
||||||
|
<TargetFramework>net5.0</TargetFramework>
|
||||||
|
</PropertyGroup>
|
||||||
|
|
||||||
|
<ItemGroup>
|
||||||
|
<PackageReference Include="Flurl.Http" Version="3.2.0" />
|
||||||
|
</ItemGroup>
|
||||||
|
|
||||||
|
</Project>
|
||||||
@ -3,11 +3,29 @@ Microsoft Visual Studio Solution File, Format Version 12.00
|
|||||||
# Visual Studio Version 16
|
# Visual Studio Version 16
|
||||||
VisualStudioVersion = 16.0.31313.79
|
VisualStudioVersion = 16.0.31313.79
|
||||||
MinimumVisualStudioVersion = 10.0.40219.1
|
MinimumVisualStudioVersion = 10.0.40219.1
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "WebScrape", "WebScrape\WebScrape.csproj", "{DF64E211-B223-48C5-83F8-60792D8DFB04}"
|
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "WebScrape", "WebScrape\WebScrape.csproj", "{DF64E211-B223-48C5-83F8-60792D8DFB04}"
|
||||||
EndProject
|
EndProject
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CsharpCorner", "CsharpCorne\CsharpCorner.csproj", "{BF147D4E-3654-4FF7-9B71-7E1B67CE7B0D}"
|
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CsharpCorner", "CsharpCorne\CsharpCorner.csproj", "{BF147D4E-3654-4FF7-9B71-7E1B67CE7B0D}"
|
||||||
EndProject
|
EndProject
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HeadlessBrowser", "HeadlessBrowser\HeadlessBrowser.csproj", "{F0E685F6-9B05-42D1-B2F2-B14C861668A3}"
|
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "HeadlessBrowser", "HeadlessBrowser\HeadlessBrowser.csproj", "{F0E685F6-9B05-42D1-B2F2-B14C861668A3}"
|
||||||
|
EndProject
|
||||||
|
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "WpfScraper", "WpfScraper\WpfScraper.csproj", "{C7CA27DE-F697-4001-A4A6-1AFCF373155B}"
|
||||||
|
EndProject
|
||||||
|
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "WebFlurl", "WebFlurl\WebFlurl.csproj", "{7437BA76-15CC-4CF2-AB72-4E6AD3DF6254}"
|
||||||
|
EndProject
|
||||||
|
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "WebClientScraper", "WebClientScraper\WebClientScraper.csproj", "{2D12E885-2779-4D30-9C14-AAF9B5D6C821}"
|
||||||
|
EndProject
|
||||||
|
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "HttpWebRequestType", "HttpWebRequestType\HttpWebRequestType.csproj", "{9C3F1EC8-0B6A-4B6F-AEF7-503EA9EADE07}"
|
||||||
|
EndProject
|
||||||
|
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Selenium", "Selenium\Selenium.csproj", "{36450C69-8A59-496B-A483-E697CA1F25B0}"
|
||||||
|
EndProject
|
||||||
|
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AngleSharpTest", "AngleSharpTest\AngleSharpTest.csproj", "{3CE0F660-46F5-4779-9C68-33634C2145E4}"
|
||||||
|
EndProject
|
||||||
|
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MailGunTest", "MailGunTest\MailGunTest.csproj", "{1549E3EC-FF09-4DC9-813D-96ECBC2B02CF}"
|
||||||
|
EndProject
|
||||||
|
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OceanNetWorks", "OceanNetWorks\OceanNetWorks.csproj", "{E02B7AA4-B7C5-49FB-8B74-7AEADA041194}"
|
||||||
|
EndProject
|
||||||
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PuppeteerSharpTest", "PuppeteerSharpTest\PuppeteerSharpTest.csproj", "{A3A9F192-0881-40D0-9389-53C9AF7018F5}"
|
||||||
EndProject
|
EndProject
|
||||||
Global
|
Global
|
||||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||||
@ -27,6 +45,42 @@ Global
|
|||||||
{F0E685F6-9B05-42D1-B2F2-B14C861668A3}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
{F0E685F6-9B05-42D1-B2F2-B14C861668A3}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||||
{F0E685F6-9B05-42D1-B2F2-B14C861668A3}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
{F0E685F6-9B05-42D1-B2F2-B14C861668A3}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||||
{F0E685F6-9B05-42D1-B2F2-B14C861668A3}.Release|Any CPU.Build.0 = Release|Any CPU
|
{F0E685F6-9B05-42D1-B2F2-B14C861668A3}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||||
|
{C7CA27DE-F697-4001-A4A6-1AFCF373155B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||||
|
{C7CA27DE-F697-4001-A4A6-1AFCF373155B}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||||
|
{C7CA27DE-F697-4001-A4A6-1AFCF373155B}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||||
|
{C7CA27DE-F697-4001-A4A6-1AFCF373155B}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||||
|
{7437BA76-15CC-4CF2-AB72-4E6AD3DF6254}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||||
|
{7437BA76-15CC-4CF2-AB72-4E6AD3DF6254}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||||
|
{7437BA76-15CC-4CF2-AB72-4E6AD3DF6254}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||||
|
{7437BA76-15CC-4CF2-AB72-4E6AD3DF6254}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||||
|
{2D12E885-2779-4D30-9C14-AAF9B5D6C821}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||||
|
{2D12E885-2779-4D30-9C14-AAF9B5D6C821}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||||
|
{2D12E885-2779-4D30-9C14-AAF9B5D6C821}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||||
|
{2D12E885-2779-4D30-9C14-AAF9B5D6C821}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||||
|
{9C3F1EC8-0B6A-4B6F-AEF7-503EA9EADE07}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||||
|
{9C3F1EC8-0B6A-4B6F-AEF7-503EA9EADE07}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||||
|
{9C3F1EC8-0B6A-4B6F-AEF7-503EA9EADE07}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||||
|
{9C3F1EC8-0B6A-4B6F-AEF7-503EA9EADE07}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||||
|
{36450C69-8A59-496B-A483-E697CA1F25B0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||||
|
{36450C69-8A59-496B-A483-E697CA1F25B0}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||||
|
{36450C69-8A59-496B-A483-E697CA1F25B0}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||||
|
{36450C69-8A59-496B-A483-E697CA1F25B0}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||||
|
{3CE0F660-46F5-4779-9C68-33634C2145E4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||||
|
{3CE0F660-46F5-4779-9C68-33634C2145E4}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||||
|
{3CE0F660-46F5-4779-9C68-33634C2145E4}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||||
|
{3CE0F660-46F5-4779-9C68-33634C2145E4}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||||
|
{1549E3EC-FF09-4DC9-813D-96ECBC2B02CF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||||
|
{1549E3EC-FF09-4DC9-813D-96ECBC2B02CF}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||||
|
{1549E3EC-FF09-4DC9-813D-96ECBC2B02CF}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||||
|
{1549E3EC-FF09-4DC9-813D-96ECBC2B02CF}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||||
|
{E02B7AA4-B7C5-49FB-8B74-7AEADA041194}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||||
|
{E02B7AA4-B7C5-49FB-8B74-7AEADA041194}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||||
|
{E02B7AA4-B7C5-49FB-8B74-7AEADA041194}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||||
|
{E02B7AA4-B7C5-49FB-8B74-7AEADA041194}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||||
|
{A3A9F192-0881-40D0-9389-53C9AF7018F5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||||
|
{A3A9F192-0881-40D0-9389-53C9AF7018F5}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||||
|
{A3A9F192-0881-40D0-9389-53C9AF7018F5}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||||
|
{A3A9F192-0881-40D0-9389-53C9AF7018F5}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||||
EndGlobalSection
|
EndGlobalSection
|
||||||
GlobalSection(SolutionProperties) = preSolution
|
GlobalSection(SolutionProperties) = preSolution
|
||||||
HideSolutionNode = FALSE
|
HideSolutionNode = FALSE
|
||||||
|
|||||||
9
WpfScraper/App.xaml
Normal file
9
WpfScraper/App.xaml
Normal file
@ -0,0 +1,9 @@
|
|||||||
|
<Application x:Class="WpfScraper.App"
|
||||||
|
xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
|
||||||
|
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
|
||||||
|
xmlns:local="clr-namespace:WpfScraper"
|
||||||
|
StartupUri="MainWindow.xaml">
|
||||||
|
<Application.Resources>
|
||||||
|
|
||||||
|
</Application.Resources>
|
||||||
|
</Application>
|
||||||
17
WpfScraper/App.xaml.cs
Normal file
17
WpfScraper/App.xaml.cs
Normal file
@ -0,0 +1,17 @@
|
|||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Configuration;
|
||||||
|
using System.Data;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
using System.Windows;
|
||||||
|
|
||||||
|
namespace WpfScraper
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Interaction logic for App.xaml
|
||||||
|
/// </summary>
|
||||||
|
public partial class App : Application
|
||||||
|
{
|
||||||
|
}
|
||||||
|
}
|
||||||
10
WpfScraper/AssemblyInfo.cs
Normal file
10
WpfScraper/AssemblyInfo.cs
Normal file
@ -0,0 +1,10 @@
|
|||||||
|
using System.Windows;
|
||||||
|
|
||||||
|
[assembly: ThemeInfo(
|
||||||
|
ResourceDictionaryLocation.None, //where theme specific resource dictionaries are located
|
||||||
|
//(used if a resource is not found in the page,
|
||||||
|
// or application resource dictionaries)
|
||||||
|
ResourceDictionaryLocation.SourceAssembly //where the generic resource dictionary is located
|
||||||
|
//(used if a resource is not found in the page,
|
||||||
|
// app, or any theme specific resource dictionaries)
|
||||||
|
)]
|
||||||
14
WpfScraper/EntryModel.cs
Normal file
14
WpfScraper/EntryModel.cs
Normal file
@ -0,0 +1,14 @@
|
|||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Text;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
|
||||||
|
namespace WpfScraper
|
||||||
|
{
|
||||||
|
public class EntryModel
|
||||||
|
{
|
||||||
|
public string Title { get; set; }
|
||||||
|
public string Description { get; set; }
|
||||||
|
}
|
||||||
|
}
|
||||||
43
WpfScraper/MainWindow.xaml
Normal file
43
WpfScraper/MainWindow.xaml
Normal file
@ -0,0 +1,43 @@
|
|||||||
|
<Window x:Class="WpfScraper.MainWindow"
|
||||||
|
xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
|
||||||
|
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
|
||||||
|
xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
|
||||||
|
xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
|
||||||
|
xmlns:local="clr-namespace:WpfScraper"
|
||||||
|
mc:Ignorable="d"
|
||||||
|
Title="My awesome webscraper" Height="450" Width="800">
|
||||||
|
<Grid>
|
||||||
|
<Grid.RowDefinitions>
|
||||||
|
<RowDefinition Height="50"/>
|
||||||
|
<RowDefinition Height="*" />
|
||||||
|
</Grid.RowDefinitions>
|
||||||
|
<Grid Grid.Row="0">
|
||||||
|
<DockPanel>
|
||||||
|
<Menu>
|
||||||
|
<MenuItem Header="File">
|
||||||
|
<MenuItem Header="Export"
|
||||||
|
x:Name="ItemExport"
|
||||||
|
Click="ItemExport_Click"/>
|
||||||
|
</MenuItem>
|
||||||
|
</Menu>
|
||||||
|
<Button x:Name="BtnScraper"
|
||||||
|
Content="Scrape"
|
||||||
|
DockPanel.Dock="Right"
|
||||||
|
Height="25"
|
||||||
|
Width="50"
|
||||||
|
Click="BtnScraper_Click"
|
||||||
|
Margin="0,0,10,0"/>
|
||||||
|
|
||||||
|
<TextBox x:Name="TbPage"
|
||||||
|
DockPanel.Dock="Bottom"
|
||||||
|
Text="WebPage..."
|
||||||
|
Height="20"
|
||||||
|
Margin="0,0,10,0"/>
|
||||||
|
|
||||||
|
</DockPanel>
|
||||||
|
</Grid>
|
||||||
|
<Grid Grid.Row="1">
|
||||||
|
<DataGrid ItemsSource="{Binding Entries}"/>
|
||||||
|
</Grid>
|
||||||
|
</Grid>
|
||||||
|
</Window>
|
||||||
41
WpfScraper/MainWindow.xaml.cs
Normal file
41
WpfScraper/MainWindow.xaml.cs
Normal file
@ -0,0 +1,41 @@
|
|||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Text;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
using System.Windows;
|
||||||
|
using System.Windows.Controls;
|
||||||
|
using System.Windows.Data;
|
||||||
|
using System.Windows.Documents;
|
||||||
|
using System.Windows.Input;
|
||||||
|
using System.Windows.Media;
|
||||||
|
using System.Windows.Media.Imaging;
|
||||||
|
using System.Windows.Navigation;
|
||||||
|
using System.Windows.Shapes;
|
||||||
|
|
||||||
|
namespace WpfScraper
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Interaction logic for MainWindow.xaml
|
||||||
|
/// </summary>
|
||||||
|
public partial class MainWindow : Window
|
||||||
|
{
|
||||||
|
Scraper scraper;
|
||||||
|
public MainWindow()
|
||||||
|
{
|
||||||
|
InitializeComponent();
|
||||||
|
scraper = new Scraper();
|
||||||
|
DataContext = scraper;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void BtnScraper_Click(object sender, RoutedEventArgs e)
|
||||||
|
{
|
||||||
|
scraper.ScrapeData(TbPage.Text);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void ItemExport_Click(object sender, RoutedEventArgs e)
|
||||||
|
{
|
||||||
|
scraper.Export();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
61
WpfScraper/Scraper.cs
Normal file
61
WpfScraper/Scraper.cs
Normal file
@ -0,0 +1,61 @@
|
|||||||
|
using CsvHelper;
|
||||||
|
using HtmlAgilityPack;
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Collections.ObjectModel;
|
||||||
|
using System.Diagnostics;
|
||||||
|
using System.Globalization;
|
||||||
|
using System.IO;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Text;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
using System.Web;
|
||||||
|
|
||||||
|
namespace WpfScraper
|
||||||
|
{
|
||||||
|
public class Scraper
|
||||||
|
{
|
||||||
|
private ObservableCollection<EntryModel> _entries = new ObservableCollection<EntryModel>();
|
||||||
|
|
||||||
|
public ObservableCollection<EntryModel> Entries
|
||||||
|
{
|
||||||
|
get { return _entries; }
|
||||||
|
set { _entries = value; }
|
||||||
|
}
|
||||||
|
|
||||||
|
public void ScrapeData(string page)
|
||||||
|
{
|
||||||
|
var web = new HtmlWeb();
|
||||||
|
var doc = web.Load(page);
|
||||||
|
|
||||||
|
//var Articles = doc.DocumentNode.SelectNodes("//*[@class = 'article-single']");
|
||||||
|
var Articles = doc.DocumentNode.SelectNodes("//*[@id='shareslist']/table/tbody/tr[1]");
|
||||||
|
|
||||||
|
foreach (var article in Articles)
|
||||||
|
{
|
||||||
|
//var header = HttpUtility.HtmlDecode(article.SelectSingleNode(".//li[@class = 'article-header']").InnerText);
|
||||||
|
//var description = HttpUtility.HtmlDecode(article.SelectSingleNode(".//li[@class = 'article-copy']").InnerText);
|
||||||
|
var header = HttpUtility.HtmlDecode(article.SelectSingleNode(".//li[@class = 'millistream-link']").InnerText);
|
||||||
|
var description = HttpUtility.HtmlDecode(article.SelectSingleNode(".//li[@class = 'millistream-numeric millistream-lastprice millistream-numeric-positive']").InnerText);
|
||||||
|
Debug.Print($"Title: {header} \n"+
|
||||||
|
$" Description: {description}");
|
||||||
|
_entries.Add(new EntryModel { Title = header, Description = description });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void Export()
|
||||||
|
{
|
||||||
|
using(TextWriter tw = File.CreateText("SampleData.csv"))
|
||||||
|
{
|
||||||
|
using(var cw = new CsvWriter(tw,CultureInfo.InvariantCulture))
|
||||||
|
{
|
||||||
|
foreach(var entry in Entries)
|
||||||
|
{
|
||||||
|
cw.WriteRecord(entry);
|
||||||
|
cw.NextRecord();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
14
WpfScraper/WpfScraper.csproj
Normal file
14
WpfScraper/WpfScraper.csproj
Normal file
@ -0,0 +1,14 @@
|
|||||||
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
|
<PropertyGroup>
|
||||||
|
<OutputType>WinExe</OutputType>
|
||||||
|
<TargetFramework>net5.0-windows</TargetFramework>
|
||||||
|
<UseWPF>true</UseWPF>
|
||||||
|
</PropertyGroup>
|
||||||
|
|
||||||
|
<ItemGroup>
|
||||||
|
<PackageReference Include="CsvHelper" Version="27.1.1" />
|
||||||
|
<PackageReference Include="HtmlAgilityPack" Version="1.11.34" />
|
||||||
|
</ItemGroup>
|
||||||
|
|
||||||
|
</Project>
|
||||||
Reference in New Issue
Block a user