I have a web crawler designed using web browser control. Sometimes it works fine - it goes to the login page ο on the home page ο on the catalog menu page ο on the product details page ο returns to the catalog menu page. But at a certain time, it just gets to the home page, and then gives a blank page. It does not receive the required item identifiers.
I tried various suggestions as below ... They did not help. An interesting fact is that this question is periodic. Any thoughts on what needs to be done to overcome this recurring problem?
Note. I will not be able to share the site URL, as this is an intranet site.
LITERATURE
CODE
public partial class Form1 : Form { private System.Windows.Forms.WebBrowser wb = null; private ListBox listBox1 = null; List<string> visitedUrls = new List<string>(); List<string> visitedProducts = new List<string>(); bool isFirstPage = true; string clickType = String.Empty; bool isUnvisitedProductExist = true; private void ExerciseApp(object sender, EventArgs e) { #region Listbox Data Filling if (listBox1.Items.Count == 0) { listBox1.Items.Add("Start--" + DateTime.Now.ToString()); } else { if (listBox1.Items.Count == 2) { listBox1.Items.RemoveAt(1); } listBox1.Items.Add("Now--" + DateTime.Now.ToString()); } #endregion WriteLogFunction(" -----------------------------------------------"); #region Login //Check whether login page if (isFirstPage) { HtmlElement logonId = this.wb.Document.GetElementById("logonId"); HtmlElement password = this.wb.Document.GetElementById("logonPassword"); HtmlElement btnLogin = this.wb.Document.GetElementById("WC_AccountDisplay_links_2"); if (logonId != null && password != null && btnLogin != null) { logonId.InnerText = ConfigValues.userName; password.InnerText = ConfigValues.passwordText; isFirstPage = false; //Call click for login btnLogin.InvokeMember("click"); } } #endregion bool isClickCalled = false; #region Specific Product Details int catalogElementIterationCounter = 0; var elementsToConsider = wb.Document.All; bool isMenuPage = false; foreach (HtmlElement e1 in elementsToConsider) { catalogElementIterationCounter++; string x = e1.TagName; String idStr = e1.GetAttribute("id"); if (!String.IsNullOrWhiteSpace(idStr)) { //Each Product Navigation if (idStr.Contains("catalogEntry_img")) { isMenuPage = true; string productUrl = e1.GetAttribute("href"); if (!visitedProducts.Contains(productUrl)) { WriteLogFunction("p__" + productUrl); isUnvisitedProductExist = true; visitedProducts.Add(productUrl); isClickCalled = true; clickType = "Product"; e1.InvokeMember("Click"); break; } } } if (isMenuPage) { //Even after traversing the page, there is no unvisited product pending. //So good to go for next page if (catalogElementIterationCounter == elementsToConsider.Count - 1) { isUnvisitedProductExist = false; } } } #endregion #region Menu Page if (!isClickCalled) { #region Time Delay try { DateTime start = DateTime.Now; if (!wb.IsDisposed) { while (wb.ReadyState != WebBrowserReadyState.Complete) { System.Windows.Forms.Application.DoEvents(); if (wb.IsDisposed || DateTime.Now.Subtract(start).TotalSeconds > 2) { // Time limit break and dispose break break; } } } } catch (Exception ex) { WriteLogFunction(ex.Message); //Supress the exception } #endregion int menuPageIterationCounter = 0; bool isMatchFound = false; WriteLogFunction("Count-" + wb.Document.All.Count); var elementsInMenuPage = wb.Document.All; foreach (HtmlElement e1 in elementsInMenuPage) { menuPageIterationCounter++; string x = e1.TagName; String idStr = e1.GetAttribute("id"); WriteLogFunction("Before--"+idStr); #region time Delay try { DateTime start = DateTime.Now; if (!wb.IsDisposed) { while (wb.ReadyState != WebBrowserReadyState.Complete) { System.Windows.Forms.Application.DoEvents(); if (wb.IsDisposed || DateTime.Now.Subtract(start).TotalSeconds > 50) { // Time limit break and dispose break break; } } } } catch (Exception ex) { WriteLogFunction(ex.Message); //Supress the exception } #endregion WriteLogFunction("After--" + idStr); //Main Menu Item Navigation if (idStr.Contains("WC_CachedHeaderDisplay_links")) { WriteLogFunction("*******INSIDE"); string url = e1.GetAttribute("href"); string latestUrl = String.Empty; if (visitedUrls.Count > 0) { latestUrl = visitedUrls[visitedUrls.Count - 1]; } WriteLogFunction("L__" + latestUrl); WriteLogFunction("isUnvisitedProductExist__" + isUnvisitedProductExist.ToString()); if (visitedUrls.Contains(url) && isUnvisitedProductExist) { if (latestUrl == url) { isMatchFound = true; clickType = "Menu"; WriteLogFunction("u1__" + url); e1.InvokeMember("Click"); break; } } else if (!visitedUrls.Contains(url)) { isMatchFound = true; //Reset visited Products visitedProducts = new List<string>(); visitedUrls.Add(url); clickType = "Menu"; WriteLogFunction("u2__" + url); e1.InvokeMember("Click"); break; } if (!isMatchFound && (menuPageIterationCounter == elementsInMenuPage.Count - 1)) { //wb.Navigate(websiteUrl); //Application.Exit(); //Environment.Exit(0); } } } } #endregion } public Form1() { // listBox1 listBox1 = new ListBox(); listBox1.Location = new Point(10, 10); listBox1.Size = new Size(500, 50); this.Controls.Add(listBox1); // Web Browser wb = new WebBrowser(); wb.Location = new Point(10, 80); wb.Size = new Size(900, 900); //wb.DocumentCompleted += new WebBrowserDocumentCompletedEventHandler(ExerciseApp); wb.ScriptErrorsSuppressed = true; wb.Url = new Uri(ConfigValues.websiteUrl); // Form1 this.Text = "Web Browser Test"; this.Size = new Size(950, 950); this.Controls.Add(wb); this.Load += Form1_Load; } private void Form1_Load(object sender, EventArgs e) { this.wb.DocumentCompleted += delegate { // DocumentCompleted is fired before window.onload and body.onload this.wb.Document.Window.AttachEventHandler("onload", delegate { // Defer this to make sure all possible onload event handlers got fired System.Threading.SynchronizationContext.Current.Post(delegate { MessageBox.Show("window.onload was fired, can access DOM!"); ExerciseApp(null, null); }, null); }); }; this.wb.Navigate(ConfigValues.websiteUrl); } private void WriteLogFunction(string strMessage) { using (StreamWriter w = File.AppendText("log.txt")) { w.WriteLine("\r\n{0} {1} ", DateTime.Now.ToLongTimeString(), strMessage); } } }
Lijo
source share