retrieve data in a separate thread from the UI

buypeeb.py

@@ -20,14 +20,15 @@
 
 import requests, gi
 gi.require_version('Gtk', '3.0')
-from gi.repository import Gtk, Gio, Gdk
-
-import functions
-from listing import YahooAuctionsItem
-from settings import BuypeebSettings
+from gi.repository import Gtk
 
 import json, sys
 from os import path
+from threading import Thread
+
+import functions
+from listing import YahooAuctionsItem, JST
+from settings import BuypeebSettings
 
 isWindows = sys.platform.startswith('win')
 
@@ -36,6 +37,25 @@ if isWindows:
 else:
 	settingsLocation = path.expanduser("~/.config/Lynnear Software/buypeeb/") # dotfiles in ~ need to die. begone dot
 
+class WatchlistUpdater(Thread):
+	def __init__(self, app, update_all: bool = False):
+		Thread.__init__(self)
+		self.app = app
+		self.update_all = update_all
+
+	def run(self):
+		# TODO: make this happen in parallel
+		if self.update_all:
+			for id, item in self.app.settings.watchlist.items():
+				if not (item.ready or item.updating):
+					item.update()
+
+		else:
+			for item in self.app.settings.outdated_items():
+				if not(item.ready or item.updating):
+					item.update()
+
+		self.app.renderList()
 class BuypeebApp:
 
 	# SETUP
@@ -69,6 +89,7 @@ class BuypeebApp:
 
 	def activate(self, app):
 		self.window.show_all()
+		self.updateItems()
 
 	def shutdown(self, app):
 		self.settings.save()
@@ -123,11 +144,26 @@ class BuypeebApp:
 	def setExchangeRate(self):
 		self.rate = functions.get_exchange_rate()
 
+	def updateItems(self, update_all: bool = False):
+		updater = WatchlistUpdater(self, update_all)
+		updater.start()
+
 	def renderList(self):
 		self.items.clear()
 		print(self.settings.watchlist)
 		for id, item in self.settings.watchlist.items():
+			if item.ready:
 				self.items.append([item.name, item.price_aud(self.rate), "heenlo", id])
+			else:
+				self.items.append([item.name, "...", "...", id])
+
+		self.updateListTimes()
+
+	def updateListTimes(self):
+		now = datetime.now()
+		for item in self.items:
+			id = item[3]
+
 
 	# BUTTON CLICKS
 
@@ -141,6 +177,7 @@ class BuypeebApp:
 				self.msgBox("Invalid URL", "The provided URL was invalid.", Gtk.MessageType.ERROR)
 
 		self.renderList()
+		self.updateItems()
 
 	def btnClearEndedClicked(self, widget):
 		if self.msgBox("Clear ended?", "Are you sure you want to clear all ended items from the watchlist?", buttons = Gtk.ButtonsType.OK_CANCEL):

listing.py

@@ -56,6 +56,10 @@ class YahooAuctionsItem:
 	def __init__(self, url: str, id: str, name: str = None, from_json: dict = None):
 		# note - incoming url is not validated in any way!
 		self.name = name
+		self.price = 0
+		self.original_name = None
+		self.favourite = None
+
 		if url == None and from_json != None and id != None:
 			self.id = id
 			self.name = from_json['name']
@@ -74,15 +78,15 @@ class YahooAuctionsItem:
 		self.available = True
 		self.ready = False
 		self.updating = False
-		self.update()
 
 	def update(self):
+		self.updating = True
 		try:
 			# the good news is, yahoo japan returns all the data we need in handy json format
 			# the bad news is that the only way to get that json format is to download the whole auction page and grep it
 
-			# r = requests.get(f"https://page.auctions.yahoo.co.jp/jp/auction/{self.id}").text
-			r = open("yahoo.html").read()
+			r = requests.get(f"https://page.auctions.yahoo.co.jp/jp/auction/{self.id}").text
+			# r = open("yahoo.html").read()
 			j = json.loads(re.match(r'.*var pageData ?= ?(\{.*?\});', r, re.DOTALL).group(1))
 		except:
 			raise
@@ -97,6 +101,9 @@ class YahooAuctionsItem:
 		if self.name == None:
 			self.name = j['productName']
 
+		self.ready = True
+		self.updating = False
+
 	def price_jpy(self):
 		return f"¥{self.price:.2f}"
 

settings.py

@@ -1,9 +1,12 @@
 import json, os, pickle
 from os import path
+from datetime import datetime, timezone, timedelta
 
 import functions
 from listing import YahooAuctionsItem
 
+JST = timezone(timedelta(hours = 9))
+
 class BuypeebSettings:
 	def __init__(self, location: str):
 		self.updateInterval = 10 * 60
@@ -50,3 +53,19 @@ class BuypeebSettings:
 		self.watchlist[id] = YahooAuctionsItem(url, id, name)
 		for item in self.watchlist.values():
 			print(item.name, item.price)
+
+	def outdated_items(self):
+		out = []
+		now = datetime.now(tz = JST).timestamp()
+		for item in self.watchlist.values():
+			time_difference = now - item.last_checked.timestamp()
+			ending_soon = False # TODO: this
+
+			if \
+			(item.favourite and time_difference >= self.favouriteUpdateInterval) or \
+			(item.favourite and ending_soon and time_difference >= self.favouriteUpdateIntervalCritical) or \
+			(time_difference >= self.updateInterval) or \
+			(ending_soon and time_difference >= self.updateIntervalCritical):
+				out.append(item)
+
+		return out

ui/main.glade

@@ -285,7 +285,6 @@
                             <property name="title" translatable="yes">Name</property>
                             <property name="expand">True</property>
                             <property name="clickable">True</property>
-                            <property name="sort_indicator">True</property>
                             <child>
                               <object class="GtkCellRendererText"/>
                               <attributes>
@@ -309,7 +308,7 @@
                         <child>
                           <object class="GtkTreeViewColumn">
                             <property name="resizable">True</property>
-                            <property name="title" translatable="yes">Ending in</property>
+                            <property name="title" translatable="yes">Ending at</property>
                             <child>
                               <object class="GtkCellRendererText"/>
                               <attributes>