print a 'finished' message when done scraping a given user's posts

.gitignore

@@ -1,3 +1,4 @@
 __pycache__
 config.json
 planning.txt
+*.pyc

functions.py

@@ -34,8 +34,13 @@ def extract_post(post):
 	text = text.rstrip("\n") # remove trailing newline(s)
 	return text
 
-def make_post(handle):
-	handle = handle[0]
+def make_post(args):
+	id = None
+	acct = None
+	if len(args) > 1:
+		id = args[1]
+		acct = args[3]
+	handle = args[0]
 	db = MySQLdb.connect(
 		host = cfg['db_host'],
 		user=cfg['db_user'],
@@ -47,7 +52,8 @@ def make_post(handle):
 	c = db.cursor()
 	dc.execute("""
 	SELECT 
-		learn_from_cw, 
+		learn_from_cw,
+		length,
 		fake_mentions,
 		fake_mentions_full,
 		post_privacy,
@@ -89,19 +95,22 @@ def make_post(handle):
 	# 3. convert the tuple to a list
 	# 4. join the list into a string separated by newlines
 	posts = "\n".join(list(sum(c.fetchall(), ())))
+	if len(posts) == 0:
+		print("No posts to learn from.")
+		return
 
 	model = nlt_fixed(posts)
 	tries = 0
 	post = None
-	# even with such a high tries value for markovify, it still sometimes returns none.
-	# so we implement our own tries function as well, and try ten times.
 
 	if bot['fake_mentions'] == 'never':
 		# remove all mentions from the training data before the markov model sees it
 		posts = re.sub(r"@(\w+)@([\w.]+)\s?", "", posts)
 
+	# even with such a high tries value for markovify, it still sometimes returns none.
+	# so we implement our own tries function as well, and try ten times.
 	while post is None and tries < 10:
-		post = model.make_short_sentence(500, tries = 10000)
+		post = model.make_short_sentence(bot['length'], tries = 1000)
 		tries += 1
 
 	if post == None:
@@ -122,7 +131,16 @@ def make_post(handle):
 				post = re.sub(r"@(\w+)@([\w.]+)", r"@{}\1".format(zws), post)		
 
 		print(post)
-		client.status_post(post, visibility = bot['post_privacy'], spoiler_text = bot['content_warning'])
+		visibility = bot['post_privacy'] if len(args) == 1 else args[2]
+		if acct is not None:
+			post = "{} {}".format(acct, post)
 
-	c.execute("UPDATE bots SET last_post = CURRENT_TIMESTAMP() WHERE handle = %s", (handle,))
-	db.commit()
+		# ensure post isn't longer than bot['length']
+		post = post[:bot['length']] 
+		# send toot!!
+		client.status_post(post, id, visibility = visibility, spoiler_text = bot['content_warning'])
+
+	if id == None:
+		# this wasn't a reply, it was a regular post, so update the last post date
+		c.execute("UPDATE bots SET last_post = CURRENT_TIMESTAMP() WHERE handle = %s", (handle,))
+		db.commit()

scrape.py

@@ -13,7 +13,8 @@ def scrape_posts(account):
 	c = db.cursor()
 	last_post = 0
 	c.execute("SELECT COUNT(*) FROM `posts` WHERE `fedi_id` = %s", (handle,))
-	if c.fetchone()[0] > 0:
+	count = c.fetchone()
+	if count is not None and count[0] > 0:
 		# we've downloaded this user's posts before
 		# find out the most recently downloaded post of theirs
 		c.execute("SELECT `post_id` FROM `posts` WHERE `fedi_id` = %s ORDER BY `id` DESC LIMIT 1", (handle,))
@@ -78,8 +79,9 @@ def scrape_posts(account):
 				else:
 					j = r.json()
 
-		db.commit()
+			db.commit()
 		c.close()
+		print("Finished scraping {}".format(handle))
 
 print("Establishing DB connection")
 db = MySQLdb.connect(
@@ -99,4 +101,6 @@ accounts = cursor.fetchall()
 with Pool(cfg['service_threads']) as p:
 	p.map(scrape_posts, accounts)
 
+db.commit()
+
 print("Done!")

service.py

@@ -20,6 +20,7 @@ print("Cleaning up database")
 # delete any fedi accounts we no longer need
 cursor = db.cursor()
 cursor.execute("DELETE FROM fedi_accounts WHERE handle NOT IN (SELECT fedi_id FROM bot_learned_accounts)")
+db.commit()
 
 print("Generating posts")
 cursor.execute("SELECT handle FROM bots WHERE enabled = TRUE AND TIMESTAMPDIFF(MINUTE, last_post, CURRENT_TIMESTAMP()) > post_frequency")
@@ -29,3 +30,5 @@ with Pool(cfg['service_threads']) as p:
 	p.map(functions.make_post, bots)
 
 #TODO: other cron tasks should be done here, like updating profile pictures
+
+db.commit()

webui.py

@@ -5,6 +5,7 @@ import requests
 import MySQLdb
 import bcrypt
 import json, hashlib, re
+import functions
 
 cfg = json.load(open("config.json"))
 
@@ -22,7 +23,7 @@ scopes = ['write:statuses', 'write:accounts', 'read:accounts', 'read:notificatio
 
 @app.before_request
 def login_check():
-	if request.path not in ['/', '/about', '/welcome', '/login', '/signup', '/do/login', '/do/signup', '/static/style.css']:
+	if request.path not in ['/', '/about', '/welcome', '/login', '/signup', '/do/login', '/do/signup', '/static/style.css'] and not request.path.startswith("/push"):
 		# page requires authentication
 		if 'user_id' not in session:
 			return redirect(url_for('home'))
@@ -489,6 +490,52 @@ def do_authenticate_bot():
 	session['step'] = 4
 	return redirect(url_for("bot_create"), 303)
 
+@app.route("/push/<id>", methods = ['POST'])
+def push(id):
+	c = mysql.connection.cursor()
+	c.execute("SELECT client_id, client_secret, secret FROM credentials WHERE id = (SELECT credentials_id FROM bots WHERE handle = %s)", (id,))
+	login = c.fetchone()
+	client = Mastodon(
+		client_id = login[0],
+		client_secret = login[1],
+		access_token = login[2],
+		api_base_url = "https://{}".format(id.split("@")[2])
+	)
+
+	c.execute("SELECT push_private_key, push_secret, replies_enabled FROM bots WHERE handle = %s", (id,))
+	bot = c.fetchone()
+	if not bot[2]:
+		return "Replies disabled."
+
+	params = {
+		'privkey': int(bot[0].rstrip("\0")),
+		'auth': bot[1]
+	}
+	push_object = client.push_subscription_decrypt_push(request.data, params, request.headers['Encryption'], request.headers['Crypto-Key'])
+	notification = client.notifications(id = push_object['notification_id'])
+	me = client.account_verify_credentials()['id']
+
+	# first, check how many times the bot has posted in this thread.
+	# if it's over 15, don't reply.
+	# this is to stop endless reply chains between two bots.
+	try:
+		context = client.status_context(notification['status']['id'])
+		my_posts = 0
+		for post in context['ancestors']:
+			if post['account']['id'] == me:
+				my_posts += 1
+			if my_posts >= 15:
+				# don't reply
+				return "Didn't reply."
+	except:
+		# failed to fetch context
+		# assume we haven't been participating in this thread
+		pass
+
+	functions.make_post([id, notification['status']['id'], notification['status']['visibility'], "@" + notification['account']['acct']])
+
+	return "Success!"
+
 @app.route("/do/signup", methods=['POST'])
 def do_signup():
 	# email validation is basically impossible without actually sending an email to the address
@@ -554,9 +601,6 @@ def img_bot_generic():
 
 @app.route("/favicon.ico")
 def favicon():
-	# so there's a weird bug where one of my firefox installs wants the favicon, and when it can't find it, it requests /
-	# requesting / resets your session['step'] to 1, which breaks all the multi-step things.
-	# so we need to give it a favicon.
 	return send_file("static/favicon.ico")
 
 def bot_check(bot):