Add /r/all, /r/other, more graceful execution when visiting empty pages

app/app.py

@@ -3,6 +3,7 @@ from urllib.parse import urlparse
 
 import config
 import json
+import re
 import sqlite3
 import subprocess
 
@@ -21,7 +22,14 @@ def hide_post(permalink):
 		permalink = "/" + permalink
 	connection = sqlite3.connect(config.db_file)
 	cursor = connection.cursor()
-	update = "UPDATE post SET hidden = ? WHERE permalink = ?"
+	update = """
+	UPDATE
+		post
+	SET
+		hidden = ?
+	WHERE
+		permalink = ?
+	"""
 	binds = [True, permalink]
 	print(update)
 	print(binds)
@@ -30,33 +38,131 @@ def hide_post(permalink):
 	connection.close()
 	return ""
 
-@app.route('/', methods=['GET', 'POST'])
+@app.route('/')
 def index():
+	return front_page()
+
+@app.route('/r/all')
+def front_page():
+	title = "/r/all"
 	connection = sqlite3.connect(config.db_file)
 	cursor = connection.cursor()
-	select = "SELECT subreddit, count(*) count FROM post WHERE hidden = ? GROUP BY subreddit ORDER BY count desc LIMIT 1"
+	subreddits = get_subreddits(cursor)
-	binds = [False]
+	select = """
-	row = cursor.execute(select, binds).fetchone()
+	SELECT
+		post
+	FROM
+		post
+	WHERE
+		hidden = ?
+	ORDER BY
+		score desc
+	LIMIT ?
+	"""
+	binds = [False, config.posts_per_page_load]
+	posts = get_posts_from_select(cursor, select, binds)
+	add_subreddits_to_posts(posts)
 	connection.close()
-	return get_subreddit(row[0])
+	return render_template('index.html', title=title, posts=posts, subreddits=subreddits)
+
+@app.route('/r/other')
+def other_page():
+	title = "/r/other"
+	connection = sqlite3.connect(config.db_file)
+	cursor = connection.cursor()
+	subreddits = get_subreddits(cursor)
+	select = """
+	SELECT
+		post
+	FROM
+		post
+	WHERE
+		hidden = ? AND
+		subreddit IN 
+			(
+				SELECT
+					subreddit
+				FROM
+					(
+						SELECT
+							subreddit,
+							count(*) count
+						FROM
+							post
+						WHERE
+							hidden = ?
+						GROUP BY
+							subreddit
+					) t
+				WHERE
+					count <= ?
+			)
+	ORDER BY
+		score desc
+	LIMIT ?
+	"""
+	binds = [False, False, config.other_posts_cutoff, config.posts_per_page_load]
+	posts = get_posts_from_select(cursor, select, binds)
+	add_subreddits_to_posts(posts)
+	connection.close()
+	return render_template('index.html', title=title, posts=posts, subreddits=subreddits)
 
 @app.route('/r/<path:subreddit>')
 def get_subreddit(subreddit):
 	title = f"/r/{subreddit}"
 	connection = sqlite3.connect(config.db_file)
 	cursor = connection.cursor()
-	select = "SELECT subreddit, count FROM (SELECT subreddit, count(*) count FROM post WHERE hidden = ? GROUP BY subreddit ORDER BY count desc) t WHERE count > 0"
+	subreddits = get_subreddits(cursor)
-	binds = [False]
+	select = """
+	SELECT
+		post
+	FROM
+		post
+	WHERE
+		subreddit = ? AND
+		hidden = ?
+	ORDER BY
+		score desc
+	LIMIT ?
+	"""
+	binds = [subreddit, False, config.posts_per_page_load]
+	posts = get_posts_from_select(cursor, select, binds)
+	connection.close()
+	return render_template('index.html', title=title, posts=posts, subreddits=subreddits)
+
+def get_subreddits(cursor):
+	select = """
+	SELECT
+		subreddit
+	FROM
+		(
+			SELECT
+				subreddit,
+				count(*) count
+			FROM
+				post
+			WHERE
+				hidden = ?
+			GROUP BY
+				subreddit
+			ORDER BY
+				count desc
+		) t
+	WHERE
+		count > ?
+	"""
+	binds = [False, config.other_posts_cutoff]
 	results = cursor.execute(select, binds).fetchall()
 	subreddits = [f"/r/{sub[0]}" for sub in results]
-	count = results[subreddits.index(title)][1]
+	subreddits.insert(0, "/r/all")
-	select = "SELECT post FROM post WHERE subreddit = ? and hidden = ? ORDER BY score desc LIMIT ?"
+	subreddits.append("/r/other")
-	binds = [subreddit, False, config.posts_per_page_load]
+	return subreddits
+
+def get_posts_from_select(cursor, select, binds):
 	results = cursor.execute(select, binds).fetchall()
 	posts = [json.loads(post[0]) for post in results]
 	add_media_html_to_posts(posts)
-	connection.close()
+	return posts
-	return render_template('index.html', title=title, count=count, posts=posts, subreddits=subreddits)
 
 def add_media_html_to_posts(posts):
 	for post_index, post in enumerate(posts):
@@ -69,6 +175,11 @@ def add_media_html_to_posts(posts):
 			media_html.append(html)
 		post["media_html"] = media_html
 
+def add_subreddits_to_posts(posts):
+	for post in posts:
+		m = re.search(r"\/r\/([a-zA-Z0-9_]+)\/.*", post["permalink"])
+		post["subreddit"] = f"/r/{m.group(1)}"
+			
 
 def get_media_html(file, priority=False):
 	if file.endswith('.jpg') or file.endswith('.jpeg') or file.endswith('.png') or file.endswith('.gif'):
@@ -83,5 +194,4 @@ def get_media_html(file, priority=False):
 
 if __name__ == '__main__':
 	subprocess.run(["python3", "make_db.py"])
-	#subprocess.run(["python3", "scrape_posts.py"])
 	app.run(host='0.0.0.0', port=8000)

app/config.py

@@ -18,10 +18,14 @@ subreddits = [
     ("linux4noobs",           100),
     ("selfhosted",            100),
     ("HomeServer",            100),
-    ("homelab",               100)
+    ("homelab",               100),
+    ("KidsAreFuckingStupid",  100),
+    ("NonPoliticalTwitter",   100),
+    ("all",                  1000)
 ]
 max_age_days = 30
 max_age_seconds = max_age_days * 24 * 60 * 60
+other_posts_cutoff = 1 #subreddits with this many unread posts or fewer are merged to /r/other
 
 # Webpage configuration
 posts_per_page_load = 50

app/templates/index.html

@@ -174,11 +174,14 @@
             {% endfor %}
         </div>
         <div class="content">
-            <h1>{{ title }} ({{ count }})</h1>
+            <h1>{{ title }}</h1>
             {% for post in posts %}
             <div class="post">
-                <h2>{{ post.title }}</h2>
+                <h3>{{ post.title }}</h3>
-                <h4>Score: {{ post.score }}</h4>
+                {% if post.subreddit %}
+                <h5>{{ post.subreddit }}</h5>
+                {% endif %}
+                {% if post.media_html|length > 0 %}
                 <div class="media-div">
                     {% for media in post.media_html %}
                         {{ media|safe }}
@@ -188,6 +191,7 @@
                     </span>
                     {% endif %}
                 </div>
+                {% endif %}
                 {% if post.body %}
                 <div class="text-content" onclick="expand(this)">
                     {{ post.body }}