From 277fbd6311722fc3104d6924ba9dee331a65ee1e Mon Sep 17 00:00:00 2001 From: Lynne Date: Sun, 23 Aug 2020 21:18:11 +1000 Subject: [PATCH] retrieve data in a separate thread from the UI --- buypeeb.py | 49 +++++++++++++++++++++++++++++++++++++++++++------ listing.py | 13 ++++++++++--- settings.py | 19 +++++++++++++++++++ ui/main.glade | 3 +-- 4 files changed, 73 insertions(+), 11 deletions(-) diff --git a/buypeeb.py b/buypeeb.py index 308bdbe..fd47311 100755 --- a/buypeeb.py +++ b/buypeeb.py @@ -20,14 +20,15 @@ import requests, gi gi.require_version('Gtk', '3.0') -from gi.repository import Gtk, Gio, Gdk - -import functions -from listing import YahooAuctionsItem -from settings import BuypeebSettings +from gi.repository import Gtk import json, sys from os import path +from threading import Thread + +import functions +from listing import YahooAuctionsItem, JST +from settings import BuypeebSettings isWindows = sys.platform.startswith('win') @@ -36,6 +37,25 @@ if isWindows: else: settingsLocation = path.expanduser("~/.config/Lynnear Software/buypeeb/") # dotfiles in ~ need to die. begone dot +class WatchlistUpdater(Thread): + def __init__(self, app, update_all: bool = False): + Thread.__init__(self) + self.app = app + self.update_all = update_all + + def run(self): + # TODO: make this happen in parallel + if self.update_all: + for id, item in self.app.settings.watchlist.items(): + if not (item.ready or item.updating): + item.update() + + else: + for item in self.app.settings.outdated_items(): + if not(item.ready or item.updating): + item.update() + + self.app.renderList() class BuypeebApp: # SETUP @@ -69,6 +89,7 @@ class BuypeebApp: def activate(self, app): self.window.show_all() + self.updateItems() def shutdown(self, app): self.settings.save() @@ -123,11 +144,26 @@ class BuypeebApp: def setExchangeRate(self): self.rate = functions.get_exchange_rate() + def updateItems(self, update_all: bool = False): + updater = WatchlistUpdater(self, update_all) + updater.start() + def renderList(self): self.items.clear() print(self.settings.watchlist) for id, item in self.settings.watchlist.items(): - self.items.append([item.name, item.price_aud(self.rate), "heenlo", id]) + if item.ready: + self.items.append([item.name, item.price_aud(self.rate), "heenlo", id]) + else: + self.items.append([item.name, "...", "...", id]) + + self.updateListTimes() + + def updateListTimes(self): + now = datetime.now() + for item in self.items: + id = item[3] + # BUTTON CLICKS @@ -141,6 +177,7 @@ class BuypeebApp: self.msgBox("Invalid URL", "The provided URL was invalid.", Gtk.MessageType.ERROR) self.renderList() + self.updateItems() def btnClearEndedClicked(self, widget): if self.msgBox("Clear ended?", "Are you sure you want to clear all ended items from the watchlist?", buttons = Gtk.ButtonsType.OK_CANCEL): diff --git a/listing.py b/listing.py index 584b551..16677bb 100644 --- a/listing.py +++ b/listing.py @@ -56,6 +56,10 @@ class YahooAuctionsItem: def __init__(self, url: str, id: str, name: str = None, from_json: dict = None): # note - incoming url is not validated in any way! self.name = name + self.price = 0 + self.original_name = None + self.favourite = None + if url == None and from_json != None and id != None: self.id = id self.name = from_json['name'] @@ -74,15 +78,15 @@ class YahooAuctionsItem: self.available = True self.ready = False self.updating = False - self.update() def update(self): + self.updating = True try: # the good news is, yahoo japan returns all the data we need in handy json format # the bad news is that the only way to get that json format is to download the whole auction page and grep it - # r = requests.get(f"https://page.auctions.yahoo.co.jp/jp/auction/{self.id}").text - r = open("yahoo.html").read() + r = requests.get(f"https://page.auctions.yahoo.co.jp/jp/auction/{self.id}").text + # r = open("yahoo.html").read() j = json.loads(re.match(r'.*var pageData ?= ?(\{.*?\});', r, re.DOTALL).group(1)) except: raise @@ -97,6 +101,9 @@ class YahooAuctionsItem: if self.name == None: self.name = j['productName'] + self.ready = True + self.updating = False + def price_jpy(self): return f"¥{self.price:.2f}" diff --git a/settings.py b/settings.py index cd0c023..63e1f3b 100644 --- a/settings.py +++ b/settings.py @@ -1,9 +1,12 @@ import json, os, pickle from os import path +from datetime import datetime, timezone, timedelta import functions from listing import YahooAuctionsItem +JST = timezone(timedelta(hours = 9)) + class BuypeebSettings: def __init__(self, location: str): self.updateInterval = 10 * 60 @@ -50,3 +53,19 @@ class BuypeebSettings: self.watchlist[id] = YahooAuctionsItem(url, id, name) for item in self.watchlist.values(): print(item.name, item.price) + + def outdated_items(self): + out = [] + now = datetime.now(tz = JST).timestamp() + for item in self.watchlist.values(): + time_difference = now - item.last_checked.timestamp() + ending_soon = False # TODO: this + + if \ + (item.favourite and time_difference >= self.favouriteUpdateInterval) or \ + (item.favourite and ending_soon and time_difference >= self.favouriteUpdateIntervalCritical) or \ + (time_difference >= self.updateInterval) or \ + (ending_soon and time_difference >= self.updateIntervalCritical): + out.append(item) + + return out diff --git a/ui/main.glade b/ui/main.glade index 35b9df0..d042ab3 100644 --- a/ui/main.glade +++ b/ui/main.glade @@ -285,7 +285,6 @@ Name True True - True @@ -309,7 +308,7 @@ True - Ending in + Ending at