lynnesbian/buypeeb-cs
lynnesbian/buypeeb-cs
2
1
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

handle 404'd pages, more cleanup

Browse source 
i also added some comments =u=
This commit is contained in:
Lynne Megido 2021-06-15 11:57:36 +10:00
parent 34657b9b78
commit 21e1f2e3c0
Signed by: lynnesbian
GPG key ID: F0A184B5213D9F90
2 changed files with 101 additions and 84 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

153
MainWindow.cs
View file

@ -1,6 +1,6 @@
/* /*
buypeeb - a program to track yahoo jp auctions of peebus. buypeeb - a program to track yahoo jp auctions of peebus.
Copyright (C) 2020 lynnesbian Copyright (C) 2021 lynnesbian
This program is free software: you can redistribute it and/or modify This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by it under the terms of the GNU General Public License as published by
@ -39,13 +39,13 @@ using Timeout = GLib.Timeout;
namespace Buypeeb { namespace Buypeeb {
[SuppressMessage("ReSharper", "UnusedMember.Local")] [SuppressMessage("ReSharper", "UnusedMember.Local")]
internal class MainWindow : Window { internal class MainWindow : Window {
private string location; private readonly string location;
private JsonSerializerOptions jsonOptions; private readonly JsonSerializerOptions jsonOptions;
private ListStore items; private readonly ListStore items;
private Settings settings; private Settings settings;
private TreeView itemTreeView; private readonly TreeView itemTreeView;
private Builder builder; private readonly Builder builder;
// TODO: whenever we get something from the builder, cache it for later // TODO: whenever we get something from the builder, cache it for later
// that way we don't need to constantly do "builder.GetObject"s // that way we don't need to constantly do "builder.GetObject"s
@ -59,7 +59,7 @@ namespace Buypeeb {
// ...to here. // ...to here.
private static SemaphoreSlim taskLimit = new SemaphoreSlim(6); private static readonly SemaphoreSlim TaskLimit = new SemaphoreSlim(6);
private readonly Queue<string> updateQueue = new Queue<string>(); private readonly Queue<string> updateQueue = new Queue<string>();
private IEnumerable<YahooAuctionsItem> filterQuery => private IEnumerable<YahooAuctionsItem> filterQuery =>
@ -82,8 +82,9 @@ namespace Buypeeb {
// only returns items that meet all of the following: // only returns items that meet all of the following:
// - marked as "ready", as in, they aren't in the process of updating // - marked as "ready", as in, they aren't in the process of updating
// - not updated since the interval // - not updated since the interval
// - hasn't already ended
from item in settings.watchlist.Values.ToList() from item in settings.watchlist.Values.ToList()
where item.Ready && settings.ItemNotUpdatedSinceInterval(item) where item.Ready && settings.ItemNotUpdatedSinceInterval(item) && item.endDate.CompareTo(DateTime.UtcNow) > 0
select item; select item;
private YahooAuctionsItem selectedItem { private YahooAuctionsItem selectedItem {
@ -103,13 +104,14 @@ namespace Buypeeb {
private MainWindow(Builder builder) : base(builder.GetObject("wndMain").Handle) { private MainWindow(Builder builder) : base(builder.GetObject("wndMain").Handle) {
jsonOptions = new JsonSerializerOptions { jsonOptions = new JsonSerializerOptions {
// enable full unicode support to ensure that japanese characters are saved properly
Encoder = JavaScriptEncoder.Create(UnicodeRanges.All), Encoder = JavaScriptEncoder.Create(UnicodeRanges.All),
}; };
if (Environment.OSVersion.Platform == PlatformID.Win32NT) { if (Environment.OSVersion.Platform == PlatformID.Win32NT) {
// C:\Users\Beebus\AppData\Roaming\Lynnear Software\buypeeb // C:\Users\Beebus\AppData\Roaming\Lynnear Software\buypeeb
location = System.IO.Path.Combine(Environment.ExpandEnvironmentVariables("%APPDATA%"), "Lynnear Software", location = System.IO.Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData),
"buypeeb"); "Lynnear Software", "buypeeb");
} else { } else {
// ~/.config/Lynnear Software/buypeeb // ~/.config/Lynnear Software/buypeeb
location = System.IO.Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".config", location = System.IO.Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".config",
@ -133,6 +135,7 @@ namespace Buypeeb {
SaveSettings(); SaveSettings();
Title = "Buypeeb"; Title = "Buypeeb";
// build main window UI
this.builder = builder; this.builder = builder;
builder.Autoconnect(this); builder.Autoconnect(this);
@ -144,9 +147,11 @@ namespace Buypeeb {
selectionViewBox = (Box) builder.GetObject("SelectionViewBox"); selectionViewBox = (Box) builder.GetObject("SelectionViewBox");
endingLabel = (Label) builder.GetObject("LabelSelectedEnding"); endingLabel = (Label) builder.GetObject("LabelSelectedEnding");
searchEntry = (SearchEntry) builder.GetObject("FilterSearchEntry"); searchEntry = (SearchEntry) builder.GetObject("FilterSearchEntry");
// construct the list of filters, and ensure they're all disabled
foreach (var name in new List<string> { foreach (var name in new List<string> {
"Favourites", "NonFavourites", "Active", "Ended", "EndingToday", "EndingAfterToday", "WithWinPrice", "Favourites", "NonFavourites", "Active", "Ended", "EndingToday", "EndingAfterToday", "WithWinPrice",
"WithNoWinPrice" "WithNoWinPrice",
}) { }) {
filterChecks.Add(name, (CheckButton) builder.GetObject($"CheckButtonFilter{name}")); filterChecks.Add(name, (CheckButton) builder.GetObject($"CheckButtonFilter{name}"));
filterChecks[name].Active = false; filterChecks[name].Active = false;
@ -158,6 +163,7 @@ namespace Buypeeb {
var filteredItems = new TreeModelFilter(items, null) {VisibleFunc = ItemFilter}; var filteredItems = new TreeModelFilter(items, null) {VisibleFunc = ItemFilter};
itemTreeView.Model = filteredItems; itemTreeView.Model = filteredItems;
// functions for rendering text to the associated columns
TreeCellDataFunc[] funcs = { TreeCellDataFunc[] funcs = {
RenderColumnFavourite, RenderColumnFavourite,
RenderColumnName, RenderColumnName,
@ -193,8 +199,7 @@ namespace Buypeeb {
/// <returns>a tuple of (TreePath, TreeIter)</returns> /// <returns>a tuple of (TreePath, TreeIter)</returns>
private (TreePath path, TreeIter iter) GetRow(string id) { private (TreePath path, TreeIter iter) GetRow(string id) {
// TODO: surely there's a better way to do this // TODO: surely there's a better way to do this
TreeIter iter; itemTreeView.Model.GetIterFirst(out var iter);
itemTreeView.Model.GetIterFirst(out iter);
var m = (TreeModelFilter) itemTreeView.Model; var m = (TreeModelFilter) itemTreeView.Model;
for (var i = 0; i < itemTreeView.Model.IterNChildren(); i++) { for (var i = 0; i < itemTreeView.Model.IterNChildren(); i++) {
@ -221,9 +226,8 @@ namespace Buypeeb {
Directory.CreateDirectory(location); Directory.CreateDirectory(location);
} }
using (var fs = File.CreateText(p)) { using var fs = File.CreateText(p);
fs.Write(j); fs.Write(j);
}
} }
/// <summary> /// <summary>
@ -234,28 +238,37 @@ namespace Buypeeb {
var item = settings.watchlist[id]; var item = settings.watchlist[id];
// Console.WriteLine($"Updating {id}..."); // Console.WriteLine($"Updating {id}...");
// set item.ready to false to show that it's still being updated // set item.ready to false to show that it's still being updated
// this changes a few behaviours, such as displaying the price as "..." instead of whatever's currently stored // this changes a few behaviours, such as displaying the price as "..." instead of whatever is currently stored
item.Ready = false; item.Ready = false;
Application.Invoke(delegate { Application.Invoke(delegate {
// TODO: find a way to not have to do this. i think we need to avoid actually modifying the items outside of the main thread :/ // TODO: find a way to not have to do this. i think we need to avoid actually modifying the items outside of the
var pathAndIter = GetRow(id); // main thread :/
if (pathAndIter.path != null) { var (path, iter) = GetRow(id);
items.EmitRowChanged(pathAndIter.path, pathAndIter.iter); if (path != null) {
items.EmitRowChanged(path, iter);
} }
}); });
using (var client = new WebClient()) { using (var client = new WebClient()) {
// TODO: download should have timeout // TODO: download should have timeout
item.Update(client.DownloadString(item.url)); try {
// Thread.Sleep(5000); // item.Update(client.DownloadString("http://10.0.0.10/poop"));
// item.Update(File.ReadAllText("yahoo.html")); item.Update(client.DownloadString(item.url));
} catch (WebException e) {
if (((HttpWebResponse) e.Response).StatusCode == HttpStatusCode.NotFound) {
// the auction has ended (or otherwise been removed)
item.AuctionEnded();
} else {
Console.WriteLine($"Failed to update item ${id}!");
}
}
} }
Application.Invoke(delegate { Application.Invoke(delegate {
var pathAndIter = GetRow(id); var (path, iter) = GetRow(id);
if (pathAndIter.path != null) { if (path != null) {
items.EmitRowChanged(pathAndIter.path, pathAndIter.iter); items.EmitRowChanged(path, iter);
} }
if (item == selectedItem) { if (item == selectedItem) {
@ -270,15 +283,13 @@ namespace Buypeeb {
} }
/// <summary> /// <summary>
/// recursively processes the update queue. this is a blocking function. /// processes the update queue. this is a blocking function.
/// </summary> /// </summary>
private void ProcessUpdateQueue() { private void ProcessUpdateQueue() {
queueActive = true; queueActive = true;
UpdateItem(updateQueue.Dequeue()); while (queueActive) {
if (updateQueue.TryPeek(out var _)) { UpdateItem(updateQueue.Dequeue());
ProcessUpdateQueue(); queueActive = updateQueue.TryPeek(out _);
} else {
queueActive = false;
} }
} }
@ -288,17 +299,16 @@ namespace Buypeeb {
/// <param name="id">the id of the task to update</param> /// <param name="id">the id of the task to update</param>
/// <param name="renderListWhenDone">whether or not to call this.RenderList() after updating the item</param> /// <param name="renderListWhenDone">whether or not to call this.RenderList() after updating the item</param>
private void UpdateItem(string id, bool renderListWhenDone = false) { private void UpdateItem(string id, bool renderListWhenDone = false) {
var item = settings.watchlist[id]; if (settings.watchlist[id].updatedRecently) {
if (item.updatedRecently) {
// the item has been updated recently, and force is not true
return; return;
} }
// don't start a new task if there are more than `tasklimit` tasks currently running // don't start a new task if there are more than `tasklimit` tasks currently running
// this makes sure we don't make 1000 simultaneous requests to yahoo auctions if there are 1000 items on the watchlist // this makes sure we don't make 1000 simultaneous requests to yahoo auctions if there are 1000 items on the
taskLimit.Wait(); // watchlist
TaskLimit.Wait();
Task.Factory.StartNew(() => { UpdateThread(id); }).ContinueWith(task => { Task.Factory.StartNew(() => { UpdateThread(id); }).ContinueWith(task => {
taskLimit.Release(); TaskLimit.Release();
if (renderListWhenDone) { if (renderListWhenDone) {
Application.Invoke(delegate { RenderList(); }); Application.Invoke(delegate { RenderList(); });
} }
@ -306,7 +316,8 @@ namespace Buypeeb {
} }
/// <summary> /// <summary>
/// add every item in the watchlist to the update queue. if the update queue is already being processed (this.queueActive), this will do nothing. /// add every item in the watchlist to the update queue. if the update queue is already being processed
/// (this.queueActive), this will do nothing.
/// </summary> /// </summary>
private void UpdateItems() { private void UpdateItems() {
if (queueActive) { if (queueActive) {
@ -329,7 +340,7 @@ namespace Buypeeb {
} }
itemTreeView.QueueDraw(); itemTreeView.QueueDraw();
Task.Factory.StartNew(() => { ProcessUpdateQueue(); }); Task.Factory.StartNew(ProcessUpdateQueue);
} }
/// <summary> /// <summary>
@ -350,6 +361,7 @@ namespace Buypeeb {
selectionViewBox.Sensitive = item.Ready; selectionViewBox.Sensitive = item.Ready;
infobox.Visible = true; infobox.Visible = true;
var lastUpdated = item.LastUpdated.ToLocalTime().ToString("MMM dd, HH:mm:ss");
var info = new Dictionary<string, string> { var info = new Dictionary<string, string> {
{"Name", item.name}, {"Name", item.name},
{"YahooName", item.originalName}, {"YahooName", item.originalName},
@ -358,12 +370,16 @@ namespace Buypeeb {
{"Ending", "Please wait..."}, {"Ending", "Please wait..."},
{"Bids", $"{item.Bids}"}, {"Bids", $"{item.Bids}"},
{"BuyItNow", item.WinPrice == 0 ? "No" : $"{item.winPriceJpy} ({item.winPriceAud})"}, {"BuyItNow", item.WinPrice == 0 ? "No" : $"{item.winPriceJpy} ({item.winPriceAud})"},
{"AutoExtension", item.AutoExtension ? "Yes" : "No"}, {"AutoExtension", item.AutoExtension ? "Yes" : "No"}, {
{"LastUpdated", $"Last updated: {(item.Ready ? item.LastUpdated.ToString("MMM dd, HH:mm:ss") : "Right now!")}"}, "LastUpdated",
item.UpdateFailed
? $"Failed to update item on {lastUpdated}"
: $"Last updated: {(item.Ready ? lastUpdated : "Right now!")}"
},
}; };
foreach (var row in info) { foreach (var (key, value) in info) {
((Label) builder.GetObject($"LabelSelected{row.Key}")).Text = row.Value; ((Label) builder.GetObject($"LabelSelected{key}")).Text = value;
} }
UpdateSelectionEndTime(); UpdateSelectionEndTime();
@ -381,18 +397,12 @@ namespace Buypeeb {
/// opens a URL in the user's browser. /// opens a URL in the user's browser.
/// </summary> /// </summary>
/// <param name="url">the url to open</param> /// <param name="url">the url to open</param>
private void OpenUrl(string url) { private static void OpenUrl(string url) {
// https://github.com/dotnet/runtime/issues/17938 var psi = new ProcessStartInfo {
if (Environment.OSVersion.Platform == PlatformID.Win32NT) { FileName = url,
var psi = new ProcessStartInfo { UseShellExecute = true,
FileName = url, };
UseShellExecute = true, Process.Start(psi);
};
Process.Start(psi);
} else {
// let's hope you have xdg-open installed
Process.Start("xdg-open", url);
}
} }
/// <summary> /// <summary>
@ -462,7 +472,7 @@ namespace Buypeeb {
/// <returns>the id of the radiobutton without the "Sort" prefix</returns> /// <returns>the id of the radiobutton without the "Sort" prefix</returns>
private string GetSortType() { private string GetSortType() {
foreach (var name in new List<string> { foreach (var name in new List<string> {
"NameDescending", "NameAscending", "PriceDescending", "PriceAscending", "EndingDescending", "EndingAscending" "NameDescending", "NameAscending", "PriceDescending", "PriceAscending", "EndingDescending", "EndingAscending",
}) { }) {
var radio = (RadioMenuItem) builder.GetObject($"Sort{name}"); var radio = (RadioMenuItem) builder.GetObject($"Sort{name}");
if (radio.Active) { if (radio.Active) {
@ -474,7 +484,8 @@ namespace Buypeeb {
} }
/// <summary> /// <summary>
/// clears the treeview's liststore and adds everything in the watchlist to it, obeying sort order. tries to reselect the item that the user had selected, if possible. /// clears the treeview's liststore and adds everything in the watchlist to it, obeying sort order. tries to
/// reselect the item that the user had selected, if possible.
/// </summary> /// </summary>
private void RenderList() { private void RenderList() {
string id = null; string id = null;
@ -595,8 +606,7 @@ namespace Buypeeb {
"Cancel", ResponseType.Cancel, "Open", ResponseType.Accept "Cancel", ResponseType.Cancel, "Open", ResponseType.Accept
); );
var odf = new FileFilter(); var odf = new FileFilter {Name = "JSON files"};
odf.Name = "JSON files";
odf.AddMimeType("application/json"); odf.AddMimeType("application/json");
odf.AddPattern("*.json"); odf.AddPattern("*.json");
od.AddFilter(odf); od.AddFilter(odf);
@ -628,8 +638,7 @@ namespace Buypeeb {
parent: this, parent: this,
action: FileChooserAction.Save, action: FileChooserAction.Save,
"Cancel", ResponseType.Cancel, "Save", ResponseType.Accept "Cancel", ResponseType.Cancel, "Save", ResponseType.Accept
); ) {CurrentName = "userdata.json"};
sd.CurrentName = "userdata.json";
var sdf = new FileFilter {Name = "JSON files"}; var sdf = new FileFilter {Name = "JSON files"};
sdf.AddMimeType("application/json"); sdf.AddMimeType("application/json");
@ -639,9 +648,8 @@ namespace Buypeeb {
if (sd.Run() == (int) ResponseType.Accept) { if (sd.Run() == (int) ResponseType.Accept) {
try { try {
if (!File.Exists(sd.Filename)) { if (!File.Exists(sd.Filename)) {
using (var fs = File.CreateText(sd.Filename)) { using var fs = File.CreateText(sd.Filename);
fs.Write(JsonSerializer.Serialize(settings, jsonOptions)); fs.Write(JsonSerializer.Serialize(settings, jsonOptions));
}
} }
} catch (Exception e) { } catch (Exception e) {
Console.WriteLine(e); Console.WriteLine(e);
@ -669,21 +677,18 @@ namespace Buypeeb {
parent: this, parent: this,
action: FileChooserAction.Save, action: FileChooserAction.Save,
"Cancel", ResponseType.Cancel, "Save", ResponseType.Accept "Cancel", ResponseType.Cancel, "Save", ResponseType.Accept
); ) {CurrentName = "buypeeb.csv"};
sd.CurrentName = "buypeeb.csv";
var sdf = new FileFilter(); var sdf = new FileFilter {Name = "CSV files"};
sdf.Name = "CSV files";
sdf.AddMimeType("text/csv"); sdf.AddMimeType("text/csv");
sdf.AddPattern("*.csv"); sdf.AddPattern("*.csv");
sd.AddFilter(sdf); sd.AddFilter(sdf);
if (sd.Run() == (int) ResponseType.Accept) { if (sd.Run() == (int) ResponseType.Accept) {
try { try {
using (var writer = new StreamWriter(sd.Filename)) using var writer = new StreamWriter(sd.Filename);
using (var csv = new CsvWriter(writer, CultureInfo.InvariantCulture)) { using var csv = new CsvWriter(writer, CultureInfo.InvariantCulture);
csv.WriteRecords(settings.watchlist); csv.WriteRecords(settings.watchlist);
}
} catch (Exception e) { } catch (Exception e) {
Console.WriteLine(e); Console.WriteLine(e);
var md = MsgBox($"Failed to write {sd.Filename}!\n{e.Message}.", ButtonsType.Ok); var md = MsgBox($"Failed to write {sd.Filename}!\n{e.Message}.", ButtonsType.Ok);

32
YahooAuctionsItem.cs
View file

@ -17,8 +17,11 @@ namespace Buypeeb {
// the id *must* be saved! // the id *must* be saved!
// i'm also saving the original name to make it easier to tell what the items are in the userdata.json // i'm also saving the original name to make it easier to tell what the items are in the userdata.json
// there's not really a need for it i guess but it's my program and i can do what i want // there's not really a need for it i guess but it's my program and i can do what i want
// anything with the attribute [Ignore] won't be put in the CSV, and things with [JsonIgnore] won't be put in userdata.json // anything with the attribute [Ignore] won't be put in the CSV, and things with [JsonIgnore] won't be put in
[Ignore] public string id { get; } // userdata.json
// ReSharper disable once MemberCanBePrivate.Global
// don't remove ID's set attribute - JSON deserialisation will silently fail to assign IDs to the auction items!
[Ignore] public string id { get; set; }
public string name { get; set; } public string name { get; set; }
public int Price; public int Price;
public int WinPrice; public int WinPrice;
@ -32,6 +35,7 @@ namespace Buypeeb {
public bool AutoExtension; public bool AutoExtension;
public bool Ready; public bool Ready;
public bool Available; public bool Available;
public bool UpdateFailed;
[Ignore, JsonIgnore] [Ignore, JsonIgnore]
public bool updatedRecently { public bool updatedRecently {
@ -66,6 +70,13 @@ namespace Buypeeb {
// parameterless constructor for deserialisation // parameterless constructor for deserialisation
} }
public void AuctionEnded() {
// the page 404'd. this probably means that the auction has ended, and the page has been removed.
Available = false;
LastUpdated = DateTime.UtcNow;
UpdateFailed = true;
}
public void Update(string html) { public void Update(string html) {
// TODO: handle all the parsing errors and weird interpretation that could possibly happen here // TODO: handle all the parsing errors and weird interpretation that could possibly happen here
var rx = new Regex(@"var pageData ?= ?(\{.+?\});", var rx = new Regex(@"var pageData ?= ?(\{.+?\});",
@ -83,7 +94,7 @@ namespace Buypeeb {
} }
var jst = TimeZoneInfo.CreateCustomTimeZone("JST", new TimeSpan(9, 0, 0), "Japan Standard Time", var jst = TimeZoneInfo.CreateCustomTimeZone("JST", new TimeSpan(9, 0, 0), "Japan Standard Time",
"Japen Standard Time"); "Japan Standard Time");
var j = jFull["items"]; var j = jFull["items"];
originalName = j["productName"]; originalName = j["productName"];
@ -102,15 +113,16 @@ namespace Buypeeb {
name = originalName; name = originalName;
} }
// as far as i can tell, neither the `pageData` nor the `conf` variables in the html seem to store whether or not the auction uses automatic extension // as far as i can tell, neither the `pageData` nor the `conf` variables in the html seem to store whether or not
// the `conf` variable *does* store whether or not the auction has the "early end" feature enabled, in the key `earlyed`. // the auction uses automatic extension, the `conf` variable *does*, however, store whether or not the auction
// unfortunately, it seems like the only way to get the auto extension info is to scrape the page for the info column that displays the auto ext status // has the "early end" feature enabled, in the key `earlyed`. unfortunately, it seems like the only way to get the
// and check whether or not it's equal to "ari" (japanese for "yes"). // auto extension info is to scrape the page for the info column that displays the auto ext status and check
// whether or not it's equal to "ari" (japanese for "yes").
rx = new Regex(@"自動延長.+\n.+>(.+)<"); rx = new Regex(@"自動延長.+\n.+>(.+)<");
m = rx.Match(html); m = rx.Match(html);
if (m.Groups[1].Value != null) { AutoExtension = (m.Groups[1].Value == "あり");
AutoExtension = (m.Groups[1].Value == "あり");
} UpdateFailed = false;
} }
public override string ToString() { public override string ToString() {