lynnesbian/buypeeb-cs
lynnesbian/buypeeb-cs
2
1
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

handle 404'd pages, more cleanup

Browse source 
i also added some comments =u=
This commit is contained in:
Lynne Megido 2021-06-15 11:57:36 +10:00
parent 34657b9b78
commit 21e1f2e3c0
Signed by: lynnesbian
GPG key ID: F0A184B5213D9F90
2 changed files with 101 additions and 84 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

153
MainWindow.cs
View file

@ -1,6 +1,6 @@
/*
buypeeb - a program to track yahoo jp auctions of peebus.
Copyright (C) 2020 lynnesbian
Copyright (C) 2021 lynnesbian
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@ -39,13 +39,13 @@ using Timeout = GLib.Timeout;
namespace Buypeeb {
[SuppressMessage("ReSharper", "UnusedMember.Local")]
internal class MainWindow : Window {
private string location;
private JsonSerializerOptions jsonOptions;
private readonly string location;
private readonly JsonSerializerOptions jsonOptions;
private ListStore items;
private readonly ListStore items;
private Settings settings;
private TreeView itemTreeView;
private Builder builder;
private readonly TreeView itemTreeView;
private readonly Builder builder;
// TODO: whenever we get something from the builder, cache it for later
// that way we don't need to constantly do "builder.GetObject"s
@ -59,7 +59,7 @@ namespace Buypeeb {
// ...to here.
private static SemaphoreSlim taskLimit = new SemaphoreSlim(6);
private static readonly SemaphoreSlim TaskLimit = new SemaphoreSlim(6);
private readonly Queue<string> updateQueue = new Queue<string>();
private IEnumerable<YahooAuctionsItem> filterQuery =>
@ -82,8 +82,9 @@ namespace Buypeeb {
// only returns items that meet all of the following:
// - marked as "ready", as in, they aren't in the process of updating
// - not updated since the interval
// - hasn't already ended
from item in settings.watchlist.Values.ToList()
where item.Ready && settings.ItemNotUpdatedSinceInterval(item)
where item.Ready && settings.ItemNotUpdatedSinceInterval(item) && item.endDate.CompareTo(DateTime.UtcNow) > 0
select item;
private YahooAuctionsItem selectedItem {
@ -103,13 +104,14 @@ namespace Buypeeb {
private MainWindow(Builder builder) : base(builder.GetObject("wndMain").Handle) {
jsonOptions = new JsonSerializerOptions {
// enable full unicode support to ensure that japanese characters are saved properly
Encoder = JavaScriptEncoder.Create(UnicodeRanges.All),
};
if (Environment.OSVersion.Platform == PlatformID.Win32NT) {
// C:\Users\Beebus\AppData\Roaming\Lynnear Software\buypeeb
location = System.IO.Path.Combine(Environment.ExpandEnvironmentVariables("%APPDATA%"), "Lynnear Software",
"buypeeb");
location = System.IO.Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData),
"Lynnear Software", "buypeeb");
} else {
// ~/.config/Lynnear Software/buypeeb
location = System.IO.Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".config",
@ -133,6 +135,7 @@ namespace Buypeeb {
SaveSettings();
Title = "Buypeeb";
// build main window UI
this.builder = builder;
builder.Autoconnect(this);
@ -144,9 +147,11 @@ namespace Buypeeb {
selectionViewBox = (Box) builder.GetObject("SelectionViewBox");
endingLabel = (Label) builder.GetObject("LabelSelectedEnding");
searchEntry = (SearchEntry) builder.GetObject("FilterSearchEntry");
// construct the list of filters, and ensure they're all disabled
foreach (var name in new List<string> {
"Favourites", "NonFavourites", "Active", "Ended", "EndingToday", "EndingAfterToday", "WithWinPrice",
"WithNoWinPrice"
"WithNoWinPrice",
}) {
filterChecks.Add(name, (CheckButton) builder.GetObject($"CheckButtonFilter{name}"));
filterChecks[name].Active = false;
@ -158,6 +163,7 @@ namespace Buypeeb {
var filteredItems = new TreeModelFilter(items, null) {VisibleFunc = ItemFilter};
itemTreeView.Model = filteredItems;
// functions for rendering text to the associated columns
TreeCellDataFunc[] funcs = {
RenderColumnFavourite,
RenderColumnName,
@ -193,8 +199,7 @@ namespace Buypeeb {
/// <returns>a tuple of (TreePath, TreeIter)</returns>
private (TreePath path, TreeIter iter) GetRow(string id) {
// TODO: surely there's a better way to do this
TreeIter iter;
itemTreeView.Model.GetIterFirst(out iter);
itemTreeView.Model.GetIterFirst(out var iter);
var m = (TreeModelFilter) itemTreeView.Model;
for (var i = 0; i < itemTreeView.Model.IterNChildren(); i++) {
@ -221,9 +226,8 @@ namespace Buypeeb {
Directory.CreateDirectory(location);
}
using (var fs = File.CreateText(p)) {
fs.Write(j);
}
using var fs = File.CreateText(p);
fs.Write(j);
}
/// <summary>
@ -234,28 +238,37 @@ namespace Buypeeb {
var item = settings.watchlist[id];
// Console.WriteLine($"Updating {id}...");
// set item.ready to false to show that it's still being updated
// this changes a few behaviours, such as displaying the price as "..." instead of whatever's currently stored
// this changes a few behaviours, such as displaying the price as "..." instead of whatever is currently stored
item.Ready = false;
Application.Invoke(delegate {
// TODO: find a way to not have to do this. i think we need to avoid actually modifying the items outside of the main thread :/
var pathAndIter = GetRow(id);
if (pathAndIter.path != null) {
items.EmitRowChanged(pathAndIter.path, pathAndIter.iter);
// TODO: find a way to not have to do this. i think we need to avoid actually modifying the items outside of the
// main thread :/
var (path, iter) = GetRow(id);
if (path != null) {
items.EmitRowChanged(path, iter);
}
});
using (var client = new WebClient()) {
// TODO: download should have timeout
item.Update(client.DownloadString(item.url));
// Thread.Sleep(5000);
// item.Update(File.ReadAllText("yahoo.html"));
try {
// item.Update(client.DownloadString("http://10.0.0.10/poop"));
item.Update(client.DownloadString(item.url));
} catch (WebException e) {
if (((HttpWebResponse) e.Response).StatusCode == HttpStatusCode.NotFound) {
// the auction has ended (or otherwise been removed)
item.AuctionEnded();
} else {
Console.WriteLine($"Failed to update item ${id}!");
}
}
}
Application.Invoke(delegate {
var pathAndIter = GetRow(id);
if (pathAndIter.path != null) {
items.EmitRowChanged(pathAndIter.path, pathAndIter.iter);
var (path, iter) = GetRow(id);
if (path != null) {
items.EmitRowChanged(path, iter);
}
if (item == selectedItem) {
@ -270,15 +283,13 @@ namespace Buypeeb {
}
/// <summary>
/// recursively processes the update queue. this is a blocking function.
/// processes the update queue. this is a blocking function.
/// </summary>
private void ProcessUpdateQueue() {
queueActive = true;
UpdateItem(updateQueue.Dequeue());
if (updateQueue.TryPeek(out var _)) {
ProcessUpdateQueue();
} else {
queueActive = false;
while (queueActive) {
UpdateItem(updateQueue.Dequeue());
queueActive = updateQueue.TryPeek(out _);
}
}
@ -288,17 +299,16 @@ namespace Buypeeb {
/// <param name="id">the id of the task to update</param>
/// <param name="renderListWhenDone">whether or not to call this.RenderList() after updating the item</param>
private void UpdateItem(string id, bool renderListWhenDone = false) {
var item = settings.watchlist[id];
if (item.updatedRecently) {
// the item has been updated recently, and force is not true
if (settings.watchlist[id].updatedRecently) {
return;
}
// don't start a new task if there are more than `tasklimit` tasks currently running
// this makes sure we don't make 1000 simultaneous requests to yahoo auctions if there are 1000 items on the watchlist
taskLimit.Wait();
// this makes sure we don't make 1000 simultaneous requests to yahoo auctions if there are 1000 items on the
// watchlist
TaskLimit.Wait();
Task.Factory.StartNew(() => { UpdateThread(id); }).ContinueWith(task => {
taskLimit.Release();
TaskLimit.Release();
if (renderListWhenDone) {
Application.Invoke(delegate { RenderList(); });
}
@ -306,7 +316,8 @@ namespace Buypeeb {
}
/// <summary>
/// add every item in the watchlist to the update queue. if the update queue is already being processed (this.queueActive), this will do nothing.
/// add every item in the watchlist to the update queue. if the update queue is already being processed
/// (this.queueActive), this will do nothing.
/// </summary>
private void UpdateItems() {
if (queueActive) {
@ -329,7 +340,7 @@ namespace Buypeeb {
}
itemTreeView.QueueDraw();
Task.Factory.StartNew(() => { ProcessUpdateQueue(); });
Task.Factory.StartNew(ProcessUpdateQueue);
}
/// <summary>
@ -350,6 +361,7 @@ namespace Buypeeb {
selectionViewBox.Sensitive = item.Ready;
infobox.Visible = true;
var lastUpdated = item.LastUpdated.ToLocalTime().ToString("MMM dd, HH:mm:ss");
var info = new Dictionary<string, string> {
{"Name", item.name},
{"YahooName", item.originalName},
@ -358,12 +370,16 @@ namespace Buypeeb {
{"Ending", "Please wait..."},
{"Bids", $"{item.Bids}"},
{"BuyItNow", item.WinPrice == 0 ? "No" : $"{item.winPriceJpy} ({item.winPriceAud})"},
{"AutoExtension", item.AutoExtension ? "Yes" : "No"},
{"LastUpdated", $"Last updated: {(item.Ready ? item.LastUpdated.ToString("MMM dd, HH:mm:ss") : "Right now!")}"},
{"AutoExtension", item.AutoExtension ? "Yes" : "No"}, {
"LastUpdated",
item.UpdateFailed
? $"Failed to update item on {lastUpdated}"
: $"Last updated: {(item.Ready ? lastUpdated : "Right now!")}"
},
};
foreach (var row in info) {
((Label) builder.GetObject($"LabelSelected{row.Key}")).Text = row.Value;
foreach (var (key, value) in info) {
((Label) builder.GetObject($"LabelSelected{key}")).Text = value;
}
UpdateSelectionEndTime();
@ -381,18 +397,12 @@ namespace Buypeeb {
/// opens a URL in the user's browser.
/// </summary>
/// <param name="url">the url to open</param>
private void OpenUrl(string url) {
// https://github.com/dotnet/runtime/issues/17938
if (Environment.OSVersion.Platform == PlatformID.Win32NT) {
var psi = new ProcessStartInfo {
FileName = url,
UseShellExecute = true,
};
Process.Start(psi);
} else {
// let's hope you have xdg-open installed
Process.Start("xdg-open", url);
}
private static void OpenUrl(string url) {
var psi = new ProcessStartInfo {
FileName = url,
UseShellExecute = true,
};
Process.Start(psi);
}
/// <summary>
@ -462,7 +472,7 @@ namespace Buypeeb {
/// <returns>the id of the radiobutton without the "Sort" prefix</returns>
private string GetSortType() {
foreach (var name in new List<string> {
"NameDescending", "NameAscending", "PriceDescending", "PriceAscending", "EndingDescending", "EndingAscending"
"NameDescending", "NameAscending", "PriceDescending", "PriceAscending", "EndingDescending", "EndingAscending",
}) {
var radio = (RadioMenuItem) builder.GetObject($"Sort{name}");
if (radio.Active) {
@ -474,7 +484,8 @@ namespace Buypeeb {
}
/// <summary>
/// clears the treeview's liststore and adds everything in the watchlist to it, obeying sort order. tries to reselect the item that the user had selected, if possible.
/// clears the treeview's liststore and adds everything in the watchlist to it, obeying sort order. tries to
/// reselect the item that the user had selected, if possible.
/// </summary>
private void RenderList() {
string id = null;
@ -595,8 +606,7 @@ namespace Buypeeb {
"Cancel", ResponseType.Cancel, "Open", ResponseType.Accept
);
var odf = new FileFilter();
odf.Name = "JSON files";
var odf = new FileFilter {Name = "JSON files"};
odf.AddMimeType("application/json");
odf.AddPattern("*.json");
od.AddFilter(odf);
@ -628,8 +638,7 @@ namespace Buypeeb {
parent: this,
action: FileChooserAction.Save,
"Cancel", ResponseType.Cancel, "Save", ResponseType.Accept
);
sd.CurrentName = "userdata.json";
) {CurrentName = "userdata.json"};
var sdf = new FileFilter {Name = "JSON files"};
sdf.AddMimeType("application/json");
@ -639,9 +648,8 @@ namespace Buypeeb {
if (sd.Run() == (int) ResponseType.Accept) {
try {
if (!File.Exists(sd.Filename)) {
using (var fs = File.CreateText(sd.Filename)) {
fs.Write(JsonSerializer.Serialize(settings, jsonOptions));
}
using var fs = File.CreateText(sd.Filename);
fs.Write(JsonSerializer.Serialize(settings, jsonOptions));
}
} catch (Exception e) {
Console.WriteLine(e);
@ -669,21 +677,18 @@ namespace Buypeeb {
parent: this,
action: FileChooserAction.Save,
"Cancel", ResponseType.Cancel, "Save", ResponseType.Accept
);
sd.CurrentName = "buypeeb.csv";
) {CurrentName = "buypeeb.csv"};
var sdf = new FileFilter();
sdf.Name = "CSV files";
var sdf = new FileFilter {Name = "CSV files"};
sdf.AddMimeType("text/csv");
sdf.AddPattern("*.csv");
sd.AddFilter(sdf);
if (sd.Run() == (int) ResponseType.Accept) {
try {
using (var writer = new StreamWriter(sd.Filename))
using (var csv = new CsvWriter(writer, CultureInfo.InvariantCulture)) {
csv.WriteRecords(settings.watchlist);
}
using var writer = new StreamWriter(sd.Filename);
using var csv = new CsvWriter(writer, CultureInfo.InvariantCulture);
csv.WriteRecords(settings.watchlist);
} catch (Exception e) {
Console.WriteLine(e);
var md = MsgBox($"Failed to write {sd.Filename}!\n{e.Message}.", ButtonsType.Ok);

32
YahooAuctionsItem.cs
View file

@ -17,8 +17,11 @@ namespace Buypeeb {
// the id *must* be saved!
// i'm also saving the original name to make it easier to tell what the items are in the userdata.json
// there's not really a need for it i guess but it's my program and i can do what i want
// anything with the attribute [Ignore] won't be put in the CSV, and things with [JsonIgnore] won't be put in userdata.json
[Ignore] public string id { get; }
// anything with the attribute [Ignore] won't be put in the CSV, and things with [JsonIgnore] won't be put in
// userdata.json
// ReSharper disable once MemberCanBePrivate.Global
// don't remove ID's set attribute - JSON deserialisation will silently fail to assign IDs to the auction items!
[Ignore] public string id { get; set; }
public string name { get; set; }
public int Price;
public int WinPrice;
@ -32,6 +35,7 @@ namespace Buypeeb {
public bool AutoExtension;
public bool Ready;
public bool Available;
public bool UpdateFailed;
[Ignore, JsonIgnore]
public bool updatedRecently {
@ -66,6 +70,13 @@ namespace Buypeeb {
// parameterless constructor for deserialisation
}
public void AuctionEnded() {
// the page 404'd. this probably means that the auction has ended, and the page has been removed.
Available = false;
LastUpdated = DateTime.UtcNow;
UpdateFailed = true;
}
public void Update(string html) {
// TODO: handle all the parsing errors and weird interpretation that could possibly happen here
var rx = new Regex(@"var pageData ?= ?(\{.+?\});",
@ -83,7 +94,7 @@ namespace Buypeeb {
}
var jst = TimeZoneInfo.CreateCustomTimeZone("JST", new TimeSpan(9, 0, 0), "Japan Standard Time",
"Japen Standard Time");
"Japan Standard Time");
var j = jFull["items"];
originalName = j["productName"];
@ -102,15 +113,16 @@ namespace Buypeeb {
name = originalName;
}
// as far as i can tell, neither the `pageData` nor the `conf` variables in the html seem to store whether or not the auction uses automatic extension
// the `conf` variable *does* store whether or not the auction has the "early end" feature enabled, in the key `earlyed`.
// unfortunately, it seems like the only way to get the auto extension info is to scrape the page for the info column that displays the auto ext status
// and check whether or not it's equal to "ari" (japanese for "yes").
// as far as i can tell, neither the `pageData` nor the `conf` variables in the html seem to store whether or not
// the auction uses automatic extension, the `conf` variable *does*, however, store whether or not the auction
// has the "early end" feature enabled, in the key `earlyed`. unfortunately, it seems like the only way to get the
// auto extension info is to scrape the page for the info column that displays the auto ext status and check
// whether or not it's equal to "ari" (japanese for "yes").
rx = new Regex(@"自動延長.+\n.+>(.+)<");
m = rx.Match(html);
if (m.Groups[1].Value != null) {
AutoExtension = (m.Groups[1].Value == "あり");
}
AutoExtension = (m.Groups[1].Value == "あり");
UpdateFailed = false;
}
public override string ToString() {