From 21e1f2e3c0db7bdfe857ede9ca014424eec579c0 Mon Sep 17 00:00:00 2001 From: Lynne Date: Tue, 15 Jun 2021 11:57:36 +1000 Subject: [PATCH] handle 404'd pages, more cleanup i also added some comments =u= --- MainWindow.cs | 153 ++++++++++++++++++++++--------------------- YahooAuctionsItem.cs | 32 ++++++--- 2 files changed, 101 insertions(+), 84 deletions(-) diff --git a/MainWindow.cs b/MainWindow.cs index 916f3c4..bc6251f 100755 --- a/MainWindow.cs +++ b/MainWindow.cs @@ -1,6 +1,6 @@ /* buypeeb - a program to track yahoo jp auctions of peebus. -Copyright (C) 2020 lynnesbian +Copyright (C) 2021 lynnesbian This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -39,13 +39,13 @@ using Timeout = GLib.Timeout; namespace Buypeeb { [SuppressMessage("ReSharper", "UnusedMember.Local")] internal class MainWindow : Window { - private string location; - private JsonSerializerOptions jsonOptions; + private readonly string location; + private readonly JsonSerializerOptions jsonOptions; - private ListStore items; + private readonly ListStore items; private Settings settings; - private TreeView itemTreeView; - private Builder builder; + private readonly TreeView itemTreeView; + private readonly Builder builder; // TODO: whenever we get something from the builder, cache it for later // that way we don't need to constantly do "builder.GetObject"s @@ -59,7 +59,7 @@ namespace Buypeeb { // ...to here. - private static SemaphoreSlim taskLimit = new SemaphoreSlim(6); + private static readonly SemaphoreSlim TaskLimit = new SemaphoreSlim(6); private readonly Queue updateQueue = new Queue(); private IEnumerable filterQuery => @@ -82,8 +82,9 @@ namespace Buypeeb { // only returns items that meet all of the following: // - marked as "ready", as in, they aren't in the process of updating // - not updated since the interval + // - hasn't already ended from item in settings.watchlist.Values.ToList() - where item.Ready && settings.ItemNotUpdatedSinceInterval(item) + where item.Ready && settings.ItemNotUpdatedSinceInterval(item) && item.endDate.CompareTo(DateTime.UtcNow) > 0 select item; private YahooAuctionsItem selectedItem { @@ -103,13 +104,14 @@ namespace Buypeeb { private MainWindow(Builder builder) : base(builder.GetObject("wndMain").Handle) { jsonOptions = new JsonSerializerOptions { + // enable full unicode support to ensure that japanese characters are saved properly Encoder = JavaScriptEncoder.Create(UnicodeRanges.All), }; if (Environment.OSVersion.Platform == PlatformID.Win32NT) { // C:\Users\Beebus\AppData\Roaming\Lynnear Software\buypeeb - location = System.IO.Path.Combine(Environment.ExpandEnvironmentVariables("%APPDATA%"), "Lynnear Software", - "buypeeb"); + location = System.IO.Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), + "Lynnear Software", "buypeeb"); } else { // ~/.config/Lynnear Software/buypeeb location = System.IO.Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".config", @@ -133,6 +135,7 @@ namespace Buypeeb { SaveSettings(); Title = "Buypeeb"; + // build main window UI this.builder = builder; builder.Autoconnect(this); @@ -144,9 +147,11 @@ namespace Buypeeb { selectionViewBox = (Box) builder.GetObject("SelectionViewBox"); endingLabel = (Label) builder.GetObject("LabelSelectedEnding"); searchEntry = (SearchEntry) builder.GetObject("FilterSearchEntry"); + + // construct the list of filters, and ensure they're all disabled foreach (var name in new List { "Favourites", "NonFavourites", "Active", "Ended", "EndingToday", "EndingAfterToday", "WithWinPrice", - "WithNoWinPrice" + "WithNoWinPrice", }) { filterChecks.Add(name, (CheckButton) builder.GetObject($"CheckButtonFilter{name}")); filterChecks[name].Active = false; @@ -158,6 +163,7 @@ namespace Buypeeb { var filteredItems = new TreeModelFilter(items, null) {VisibleFunc = ItemFilter}; itemTreeView.Model = filteredItems; + // functions for rendering text to the associated columns TreeCellDataFunc[] funcs = { RenderColumnFavourite, RenderColumnName, @@ -193,8 +199,7 @@ namespace Buypeeb { /// a tuple of (TreePath, TreeIter) private (TreePath path, TreeIter iter) GetRow(string id) { // TODO: surely there's a better way to do this - TreeIter iter; - itemTreeView.Model.GetIterFirst(out iter); + itemTreeView.Model.GetIterFirst(out var iter); var m = (TreeModelFilter) itemTreeView.Model; for (var i = 0; i < itemTreeView.Model.IterNChildren(); i++) { @@ -221,9 +226,8 @@ namespace Buypeeb { Directory.CreateDirectory(location); } - using (var fs = File.CreateText(p)) { - fs.Write(j); - } + using var fs = File.CreateText(p); + fs.Write(j); } /// @@ -234,28 +238,37 @@ namespace Buypeeb { var item = settings.watchlist[id]; // Console.WriteLine($"Updating {id}..."); // set item.ready to false to show that it's still being updated - // this changes a few behaviours, such as displaying the price as "..." instead of whatever's currently stored + // this changes a few behaviours, such as displaying the price as "..." instead of whatever is currently stored item.Ready = false; Application.Invoke(delegate { - // TODO: find a way to not have to do this. i think we need to avoid actually modifying the items outside of the main thread :/ - var pathAndIter = GetRow(id); - if (pathAndIter.path != null) { - items.EmitRowChanged(pathAndIter.path, pathAndIter.iter); + // TODO: find a way to not have to do this. i think we need to avoid actually modifying the items outside of the + // main thread :/ + var (path, iter) = GetRow(id); + if (path != null) { + items.EmitRowChanged(path, iter); } }); using (var client = new WebClient()) { // TODO: download should have timeout - item.Update(client.DownloadString(item.url)); - // Thread.Sleep(5000); - // item.Update(File.ReadAllText("yahoo.html")); + try { + // item.Update(client.DownloadString("http://10.0.0.10/poop")); + item.Update(client.DownloadString(item.url)); + } catch (WebException e) { + if (((HttpWebResponse) e.Response).StatusCode == HttpStatusCode.NotFound) { + // the auction has ended (or otherwise been removed) + item.AuctionEnded(); + } else { + Console.WriteLine($"Failed to update item ${id}!"); + } + } } Application.Invoke(delegate { - var pathAndIter = GetRow(id); - if (pathAndIter.path != null) { - items.EmitRowChanged(pathAndIter.path, pathAndIter.iter); + var (path, iter) = GetRow(id); + if (path != null) { + items.EmitRowChanged(path, iter); } if (item == selectedItem) { @@ -270,15 +283,13 @@ namespace Buypeeb { } /// - /// recursively processes the update queue. this is a blocking function. + /// processes the update queue. this is a blocking function. /// private void ProcessUpdateQueue() { queueActive = true; - UpdateItem(updateQueue.Dequeue()); - if (updateQueue.TryPeek(out var _)) { - ProcessUpdateQueue(); - } else { - queueActive = false; + while (queueActive) { + UpdateItem(updateQueue.Dequeue()); + queueActive = updateQueue.TryPeek(out _); } } @@ -288,17 +299,16 @@ namespace Buypeeb { /// the id of the task to update /// whether or not to call this.RenderList() after updating the item private void UpdateItem(string id, bool renderListWhenDone = false) { - var item = settings.watchlist[id]; - if (item.updatedRecently) { - // the item has been updated recently, and force is not true + if (settings.watchlist[id].updatedRecently) { return; } // don't start a new task if there are more than `tasklimit` tasks currently running - // this makes sure we don't make 1000 simultaneous requests to yahoo auctions if there are 1000 items on the watchlist - taskLimit.Wait(); + // this makes sure we don't make 1000 simultaneous requests to yahoo auctions if there are 1000 items on the + // watchlist + TaskLimit.Wait(); Task.Factory.StartNew(() => { UpdateThread(id); }).ContinueWith(task => { - taskLimit.Release(); + TaskLimit.Release(); if (renderListWhenDone) { Application.Invoke(delegate { RenderList(); }); } @@ -306,7 +316,8 @@ namespace Buypeeb { } /// - /// add every item in the watchlist to the update queue. if the update queue is already being processed (this.queueActive), this will do nothing. + /// add every item in the watchlist to the update queue. if the update queue is already being processed + /// (this.queueActive), this will do nothing. /// private void UpdateItems() { if (queueActive) { @@ -329,7 +340,7 @@ namespace Buypeeb { } itemTreeView.QueueDraw(); - Task.Factory.StartNew(() => { ProcessUpdateQueue(); }); + Task.Factory.StartNew(ProcessUpdateQueue); } /// @@ -350,6 +361,7 @@ namespace Buypeeb { selectionViewBox.Sensitive = item.Ready; infobox.Visible = true; + var lastUpdated = item.LastUpdated.ToLocalTime().ToString("MMM dd, HH:mm:ss"); var info = new Dictionary { {"Name", item.name}, {"YahooName", item.originalName}, @@ -358,12 +370,16 @@ namespace Buypeeb { {"Ending", "Please wait..."}, {"Bids", $"{item.Bids}"}, {"BuyItNow", item.WinPrice == 0 ? "No" : $"{item.winPriceJpy} ({item.winPriceAud})"}, - {"AutoExtension", item.AutoExtension ? "Yes" : "No"}, - {"LastUpdated", $"Last updated: {(item.Ready ? item.LastUpdated.ToString("MMM dd, HH:mm:ss") : "Right now!")}"}, + {"AutoExtension", item.AutoExtension ? "Yes" : "No"}, { + "LastUpdated", + item.UpdateFailed + ? $"Failed to update item on {lastUpdated}" + : $"Last updated: {(item.Ready ? lastUpdated : "Right now!")}" + }, }; - foreach (var row in info) { - ((Label) builder.GetObject($"LabelSelected{row.Key}")).Text = row.Value; + foreach (var (key, value) in info) { + ((Label) builder.GetObject($"LabelSelected{key}")).Text = value; } UpdateSelectionEndTime(); @@ -381,18 +397,12 @@ namespace Buypeeb { /// opens a URL in the user's browser. /// /// the url to open - private void OpenUrl(string url) { - // https://github.com/dotnet/runtime/issues/17938 - if (Environment.OSVersion.Platform == PlatformID.Win32NT) { - var psi = new ProcessStartInfo { - FileName = url, - UseShellExecute = true, - }; - Process.Start(psi); - } else { - // let's hope you have xdg-open installed - Process.Start("xdg-open", url); - } + private static void OpenUrl(string url) { + var psi = new ProcessStartInfo { + FileName = url, + UseShellExecute = true, + }; + Process.Start(psi); } /// @@ -462,7 +472,7 @@ namespace Buypeeb { /// the id of the radiobutton without the "Sort" prefix private string GetSortType() { foreach (var name in new List { - "NameDescending", "NameAscending", "PriceDescending", "PriceAscending", "EndingDescending", "EndingAscending" + "NameDescending", "NameAscending", "PriceDescending", "PriceAscending", "EndingDescending", "EndingAscending", }) { var radio = (RadioMenuItem) builder.GetObject($"Sort{name}"); if (radio.Active) { @@ -474,7 +484,8 @@ namespace Buypeeb { } /// - /// clears the treeview's liststore and adds everything in the watchlist to it, obeying sort order. tries to reselect the item that the user had selected, if possible. + /// clears the treeview's liststore and adds everything in the watchlist to it, obeying sort order. tries to + /// reselect the item that the user had selected, if possible. /// private void RenderList() { string id = null; @@ -595,8 +606,7 @@ namespace Buypeeb { "Cancel", ResponseType.Cancel, "Open", ResponseType.Accept ); - var odf = new FileFilter(); - odf.Name = "JSON files"; + var odf = new FileFilter {Name = "JSON files"}; odf.AddMimeType("application/json"); odf.AddPattern("*.json"); od.AddFilter(odf); @@ -628,8 +638,7 @@ namespace Buypeeb { parent: this, action: FileChooserAction.Save, "Cancel", ResponseType.Cancel, "Save", ResponseType.Accept - ); - sd.CurrentName = "userdata.json"; + ) {CurrentName = "userdata.json"}; var sdf = new FileFilter {Name = "JSON files"}; sdf.AddMimeType("application/json"); @@ -639,9 +648,8 @@ namespace Buypeeb { if (sd.Run() == (int) ResponseType.Accept) { try { if (!File.Exists(sd.Filename)) { - using (var fs = File.CreateText(sd.Filename)) { - fs.Write(JsonSerializer.Serialize(settings, jsonOptions)); - } + using var fs = File.CreateText(sd.Filename); + fs.Write(JsonSerializer.Serialize(settings, jsonOptions)); } } catch (Exception e) { Console.WriteLine(e); @@ -669,21 +677,18 @@ namespace Buypeeb { parent: this, action: FileChooserAction.Save, "Cancel", ResponseType.Cancel, "Save", ResponseType.Accept - ); - sd.CurrentName = "buypeeb.csv"; + ) {CurrentName = "buypeeb.csv"}; - var sdf = new FileFilter(); - sdf.Name = "CSV files"; + var sdf = new FileFilter {Name = "CSV files"}; sdf.AddMimeType("text/csv"); sdf.AddPattern("*.csv"); sd.AddFilter(sdf); if (sd.Run() == (int) ResponseType.Accept) { try { - using (var writer = new StreamWriter(sd.Filename)) - using (var csv = new CsvWriter(writer, CultureInfo.InvariantCulture)) { - csv.WriteRecords(settings.watchlist); - } + using var writer = new StreamWriter(sd.Filename); + using var csv = new CsvWriter(writer, CultureInfo.InvariantCulture); + csv.WriteRecords(settings.watchlist); } catch (Exception e) { Console.WriteLine(e); var md = MsgBox($"Failed to write {sd.Filename}!\n{e.Message}.", ButtonsType.Ok); diff --git a/YahooAuctionsItem.cs b/YahooAuctionsItem.cs index a6d1a8e..4e89726 100644 --- a/YahooAuctionsItem.cs +++ b/YahooAuctionsItem.cs @@ -17,8 +17,11 @@ namespace Buypeeb { // the id *must* be saved! // i'm also saving the original name to make it easier to tell what the items are in the userdata.json // there's not really a need for it i guess but it's my program and i can do what i want - // anything with the attribute [Ignore] won't be put in the CSV, and things with [JsonIgnore] won't be put in userdata.json - [Ignore] public string id { get; } + // anything with the attribute [Ignore] won't be put in the CSV, and things with [JsonIgnore] won't be put in + // userdata.json + // ReSharper disable once MemberCanBePrivate.Global + // don't remove ID's set attribute - JSON deserialisation will silently fail to assign IDs to the auction items! + [Ignore] public string id { get; set; } public string name { get; set; } public int Price; public int WinPrice; @@ -32,6 +35,7 @@ namespace Buypeeb { public bool AutoExtension; public bool Ready; public bool Available; + public bool UpdateFailed; [Ignore, JsonIgnore] public bool updatedRecently { @@ -66,6 +70,13 @@ namespace Buypeeb { // parameterless constructor for deserialisation } + public void AuctionEnded() { + // the page 404'd. this probably means that the auction has ended, and the page has been removed. + Available = false; + LastUpdated = DateTime.UtcNow; + UpdateFailed = true; + } + public void Update(string html) { // TODO: handle all the parsing errors and weird interpretation that could possibly happen here var rx = new Regex(@"var pageData ?= ?(\{.+?\});", @@ -83,7 +94,7 @@ namespace Buypeeb { } var jst = TimeZoneInfo.CreateCustomTimeZone("JST", new TimeSpan(9, 0, 0), "Japan Standard Time", - "Japen Standard Time"); + "Japan Standard Time"); var j = jFull["items"]; originalName = j["productName"]; @@ -102,15 +113,16 @@ namespace Buypeeb { name = originalName; } - // as far as i can tell, neither the `pageData` nor the `conf` variables in the html seem to store whether or not the auction uses automatic extension - // the `conf` variable *does* store whether or not the auction has the "early end" feature enabled, in the key `earlyed`. - // unfortunately, it seems like the only way to get the auto extension info is to scrape the page for the info column that displays the auto ext status - // and check whether or not it's equal to "ari" (japanese for "yes"). + // as far as i can tell, neither the `pageData` nor the `conf` variables in the html seem to store whether or not + // the auction uses automatic extension, the `conf` variable *does*, however, store whether or not the auction + // has the "early end" feature enabled, in the key `earlyed`. unfortunately, it seems like the only way to get the + // auto extension info is to scrape the page for the info column that displays the auto ext status and check + // whether or not it's equal to "ari" (japanese for "yes"). rx = new Regex(@"自動延長.+\n.+>(.+)<"); m = rx.Match(html); - if (m.Groups[1].Value != null) { - AutoExtension = (m.Groups[1].Value == "あり"); - } + AutoExtension = (m.Groups[1].Value == "あり"); + + UpdateFailed = false; } public override string ToString() {