lynnesbian/FediBooks
lynnesbian
/
FediBooks
mirror of https://github.com/Lynnesbian/FediBooks/
1
0
Fork 0
Code Issues Releases Wiki Activity

turns out fedibooks is even buggier than i thought, and i already thought it was pretty shit

This commit is contained in:
Lynne Megido 2020-05-27 22:27:20 +10:00
parent 0a4ddb7687
commit 0997de535b
Signed by: lynnesbian
GPG Key ID: F0A184B5213D9F90
2 changed files with 4 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
app/functions.py
View File

@ -21,12 +21,10 @@ def extract_post(post):
post = html.unescape(post) # convert HTML escape codes to text post = html.unescape(post) # convert HTML escape codes to text
soup = BeautifulSoup(post, "html.parser") soup = BeautifulSoup(post, "html.parser")
for lb in soup.select("br"): # replace <br> with linebreak for lb in soup.select("br"): # replace <br> with linebreak
lb.insert_after("\n") lb.replace_with("\n")
lb.decompose()
for p in soup.select("p"): # ditto for <p> for p in soup.select("p"): # ditto for <p>
p.insert_after("\n") p.replace_with("\n")
p.unwrap()
for ht in soup.select("a.hashtag"): # convert hashtags from links to text for ht in soup.select("a.hashtag"): # convert hashtags from links to text
ht.unwrap() ht.unwrap()
@ -34,8 +32,7 @@ def extract_post(post):
for link in soup.select("a"): #ocnvert <a href='https://example.com>example.com</a> to just https://example.com for link in soup.select("a"): #ocnvert <a href='https://example.com>example.com</a> to just https://example.com
if 'href' in link: if 'href' in link:
# apparently not all a tags have a href, which is understandable if you're doing normal web stuff, but on a social media platform?? # apparently not all a tags have a href, which is understandable if you're doing normal web stuff, but on a social media platform??
link.insert_after(link["href"]) link.replace_with(link["href"])
link.decompose()
text = soup.get_text() text = soup.get_text()
text = re.sub(r"https://([^/]+)/(@[^\s]+)", r"\2@\1", text) # put mastodon-style mentions back in text = re.sub(r"https://([^/]+)/(@[^\s]+)", r"\2@\1", text) # put mastodon-style mentions back in

2
app/scrape.py
View File

@ -50,7 +50,7 @@ def scrape_posts(account):
# here we go! # here we go!
# warning: scraping posts from outbox.json is messy stuff # warning: scraping posts from outbox.json is messy stuff
while not done and len(j['orderedItems']) > 0: while not done and 'orderedItems' in j and len(j['orderedItems']) > 0:
for oi in j['orderedItems']: for oi in j['orderedItems']:
if oi['type'] == "Create": if oi['type'] == "Create":
# this is a status/post/toot/florp/whatever # this is a status/post/toot/florp/whatever