implement learn_from_cw setting

service.py

@@ -2,12 +2,17 @@
 from mastodon import Mastodon
 import MySQLdb
 import requests
+import markovify
 from multiprocessing import Pool
 import json, re
 import functions
 
 cfg = json.load(open('config.json'))
 
+class nlt_fixed(markovify.NewlineText): # modified version of NewlineText that never rejects sentences
+	def test_sentence_input(self, sentence):
+		return True # all sentences are valid <3
+
 def scrape_posts(account):
 	handle = account[0]
 	outbox = account[1]
@@ -82,10 +87,67 @@ def scrape_posts(account):
 
 		db.commit()
 		c.close()
-		print("Finished {}".format(handle))
 
-def make_post(bot):
+def make_post(handle):
+	handle = handle[0]
+	print("Generating post for {}".format(handle))
+	c = db.cursor()
+	c.execute("""
+	SELECT 
+			learn_from_cw, client_id, client_secret, secret
+	FROM
+			bots, credentials
+	WHERE
+			bots.credentials_id = (SELECT 
+							credentials_id
+					FROM
+							bots
+					WHERE
+							handle = %s)
+	""", (handle,))
+
+	bot = c.fetchone()
+	client = Mastodon(
+		client_id = bot[1],
+		client_secret = bot[2],
+		access_token = bot[3],
+		api_base_url = "https://{}".format(handle.split("@")[2])
+	)
+
+	# by default, only select posts that don't have CWs.
+	# if learn_from_cw, then also select posts with CWs
+	cw_list = [False]
+	if bot[0]:
+		cw_list = [False, True]
+
+	# select 1000 random posts for the bot to learn from
+	c.execute("SELECT content FROM posts WHERE fedi_id IN (SELECT fedi_id FROM bot_learned_accounts WHERE bot_id = %s) AND cw IN %s ORDER BY RAND() LIMIT 1000", (handle, cw_list))
+
+	# this line is a little gross/optimised but here's what it does
+	# 1. fetch all of the results from the above query
+	# 2. turn (('this',), ('format')) into ('this', 'format')
+	# 3. convert the tuple to a list
+	# 4. join the list into a string separated by newlines
+	posts = "\n".join(list(sum(c.fetchall(), ())))
+
+	model = nlt_fixed(posts)
+	tries = 0
+	sentence = None
+	# even with such a high tries value for markovify, it still sometimes returns none.
+	# so we implement our own tries function as well, and try ten times.
+	while sentence is None and tries < 10:
+		sentence = model.make_short_sentence(500, tries = 10000)
+		tries += 1
+
+	# TODO: mention handling
+
+	if sentence == None:
+		# TODO: send an error email
 		pass
+	else:
+		client.status_post(sentence)
+
+	# TODO: update date of last post
 
 print("Establishing DB connection")
 db = MySQLdb.connect(
@@ -95,24 +157,19 @@ db = MySQLdb.connect(
 	db=cfg['db_name']
 )
 
-print("Downloading posts")
+print("Cleaning up database")
+# delete any fedi accounts we no longer need
 cursor = db.cursor()
+cursor.execute("DELETE FROM fedi_accounts WHERE handle NOT IN (SELECT fedi_id FROM bot_learned_accounts);")
+
+print("Downloading posts")
 cursor.execute("SELECT `handle`, `outbox` FROM `fedi_accounts` ORDER BY RAND()")
 accounts = cursor.fetchall()
-with Pool(8) as p:
+# with Pool(8) as p:
-	p.map(scrape_posts, accounts)
+# 	p.map(scrape_posts, accounts)
 
 print("Generating posts")
-cursor.execute("""
+cursor.execute("SELECT handle FROM bots WHERE enabled = TRUE")
-SELECT
-    bots.handle, credentials.client_id, credentials.client_secret, credentials.secret
-FROM
-    bots,
-    credentials
-WHERE
-    bots.credentials_id = credentials.id
-        AND bots.enabled = TRUE;
-""")
 bots = cursor.fetchall()
 
 with Pool(8) as p:

webui.py

@@ -184,7 +184,7 @@ def bot_accounts_add():
 			if r.status_code == 200:
 				# success!!
 				c = mysql.connection.cursor()
-				c.execute("INSERT INTO `fedi_accounts` (`handle`, `outbox`) VALUES (%s, %s)", (request.form['account'], outbox))
+				c.execute("REPLACE INTO `fedi_accounts` (`handle`, `outbox`) VALUES (%s, %s)", (request.form['account'], outbox))
 				c.execute("INSERT INTO `bot_learned_accounts` (`bot_id`, `fedi_id`) VALUES (%s, %s)", (session['bot'], request.form['account']))
 				c.close()
 				mysql.connection.commit()
@@ -307,7 +307,7 @@ def bot_create():
 
 			# authentication success!!
 			c = mysql.connection.cursor()
-			c.execute("INSERT INTO `credentials` (client_id, client_secret, secret) VALUES (%s, %s, %s)", (session['client_id'], session['client_secret'], session['code']))
+			c.execute("INSERT INTO `credentials` (client_id, client_secret, secret) VALUES (%s, %s, %s)", (session['client_id'], session['client_secret'], session['secret']))
 			credentials_id = c.lastrowid
 			mysql.connection.commit()