Add admin page to manage subreddit subscriptions
This commit is contained in:
parent
3d2edfd5cf
commit
9fc267ec6a
|
|
@ -6,7 +6,7 @@ You can change the host port, host volume directories, how often reddit is scann
|
||||||
|
|
||||||
## ./app/config.py
|
## ./app/config.py
|
||||||
|
|
||||||
You can change how much data is pulled, from where, the minimum score to save it to your DB, and how long it is retained.
|
You can change how many posts are displayed per page load and how long data is retained.
|
||||||
|
|
||||||
### Startup
|
### Startup
|
||||||
|
|
||||||
|
|
@ -15,13 +15,13 @@ docker compose build
|
||||||
docker compose up
|
docker compose up
|
||||||
```
|
```
|
||||||
|
|
||||||
The DB is created automatically. You will want to run
|
The DB is created automatically. You will want to visit the /admin endpoint to set up your subreddits, then run
|
||||||
|
|
||||||
```
|
```
|
||||||
docker exec -it reddit-web-1 sh -c "python3 /app/scrape_posts.py"
|
docker exec -it reddit-web-1 sh -c "python3 /app/scrape_posts.py"
|
||||||
```
|
```
|
||||||
|
|
||||||
to populate the DB with initial data, or you will have to wait for the scheduled task to get triggered for the web page to be usable.
|
to populate the DB with initial data, or you will have to wait for the scheduled task to get triggered for posts to start showing.
|
||||||
|
|
||||||
### Thanks
|
### Thanks
|
||||||
|
|
||||||
|
|
|
||||||
63
app/app.py
63
app/app.py
|
|
@ -40,8 +40,70 @@ def hide_post(permalink):
|
||||||
|
|
||||||
@app.route('/')
|
@app.route('/')
|
||||||
def index():
|
def index():
|
||||||
|
connection = sqlite3.connect(config.db_file)
|
||||||
|
cursor = connection.cursor()
|
||||||
|
select = """
|
||||||
|
SELECT
|
||||||
|
count(*)
|
||||||
|
FROM
|
||||||
|
subreddit
|
||||||
|
"""
|
||||||
|
count = cursor.execute(select).fetchone()[0]
|
||||||
|
if count == 0:
|
||||||
|
return admin()
|
||||||
return front_page()
|
return front_page()
|
||||||
|
|
||||||
|
@app.route('/admin', methods=['GET', 'POST', 'DELETE'])
|
||||||
|
def admin():
|
||||||
|
connection = sqlite3.connect(config.db_file)
|
||||||
|
cursor = connection.cursor()
|
||||||
|
if request.method == 'DELETE':
|
||||||
|
delete = """
|
||||||
|
DELETE FROM
|
||||||
|
subreddit
|
||||||
|
WHERE
|
||||||
|
subreddit = ?
|
||||||
|
"""
|
||||||
|
binds = [request.args.get("name")]
|
||||||
|
cursor.execute(delete, binds)
|
||||||
|
connection.commit()
|
||||||
|
connection.close()
|
||||||
|
return ""
|
||||||
|
elif request.method == 'POST':
|
||||||
|
upsert = """
|
||||||
|
INSERT INTO
|
||||||
|
subreddit (subreddit, minimum_score, fetch_by, fetch_max)
|
||||||
|
VALUES
|
||||||
|
(?, ?, ?, ?)
|
||||||
|
ON CONFLICT
|
||||||
|
(subreddit)
|
||||||
|
DO UPDATE SET
|
||||||
|
minimum_score=excluded.minimum_score,
|
||||||
|
fetch_by=excluded.fetch_by,
|
||||||
|
fetch_max=excluded.fetch_max
|
||||||
|
"""
|
||||||
|
binds = [
|
||||||
|
request.form.get("name"),
|
||||||
|
int(request.form.get("score")),
|
||||||
|
request.form.get("by"),
|
||||||
|
int(request.form.get("max"))
|
||||||
|
]
|
||||||
|
cursor.execute(upsert, binds)
|
||||||
|
connection.commit()
|
||||||
|
post_subreddits = get_subreddits(cursor)
|
||||||
|
select = """
|
||||||
|
SELECT
|
||||||
|
subreddit,
|
||||||
|
minimum_score,
|
||||||
|
fetch_by,
|
||||||
|
fetch_max
|
||||||
|
FROM
|
||||||
|
subreddit
|
||||||
|
"""
|
||||||
|
sub_subreddits = cursor.execute(select).fetchall()
|
||||||
|
connection.close()
|
||||||
|
return render_template('admin.html', post_subreddits=post_subreddits, sub_subreddits=sub_subreddits)
|
||||||
|
|
||||||
@app.route('/r/all')
|
@app.route('/r/all')
|
||||||
def front_page():
|
def front_page():
|
||||||
title = "/r/all"
|
title = "/r/all"
|
||||||
|
|
@ -156,6 +218,7 @@ def get_subreddits(cursor):
|
||||||
subreddits = [f"/r/{sub[0]}" for sub in results]
|
subreddits = [f"/r/{sub[0]}" for sub in results]
|
||||||
subreddits.insert(0, "/r/all")
|
subreddits.insert(0, "/r/all")
|
||||||
subreddits.append("/r/other")
|
subreddits.append("/r/other")
|
||||||
|
subreddits.append("/admin")
|
||||||
return subreddits
|
return subreddits
|
||||||
|
|
||||||
def get_posts_from_select(cursor, select, binds):
|
def get_posts_from_select(cursor, select, binds):
|
||||||
|
|
|
||||||
|
|
@ -1,27 +1,4 @@
|
||||||
# Scheduler configuration
|
# Scheduler configuration
|
||||||
max_posts_per_pull = 100
|
|
||||||
pull_by = "day"
|
|
||||||
subreddits = [
|
|
||||||
# name, minimum upvotes
|
|
||||||
("pcgaming", 50),
|
|
||||||
("gadgets", 10),
|
|
||||||
("Nightreign", 100),
|
|
||||||
("CuratedTumblr", 100),
|
|
||||||
("196", 100),
|
|
||||||
("PoliticalCompassMemes", 100),
|
|
||||||
("meirl", 100),
|
|
||||||
("me_irl", 100),
|
|
||||||
("Fauxmoi", 100),
|
|
||||||
("NoFilterNews", 100),
|
|
||||||
("linux", 100),
|
|
||||||
("linux4noobs", 100),
|
|
||||||
("selfhosted", 100),
|
|
||||||
("HomeServer", 100),
|
|
||||||
("homelab", 100),
|
|
||||||
("NonPoliticalTwitter", 100),
|
|
||||||
("comics", 100),
|
|
||||||
("all", 1000)
|
|
||||||
]
|
|
||||||
max_age_days = 30
|
max_age_days = 30
|
||||||
max_age_seconds = max_age_days * 24 * 60 * 60
|
max_age_seconds = max_age_days * 24 * 60 * 60
|
||||||
other_posts_cutoff = 1 #subreddits with this many unread posts or fewer are merged to /r/other
|
other_posts_cutoff = 1 #subreddits with this many unread posts or fewer are merged to /r/other
|
||||||
|
|
|
||||||
|
|
@ -8,5 +8,6 @@ connection = sqlite3.connect(config.db_file)
|
||||||
cursor = connection.cursor()
|
cursor = connection.cursor()
|
||||||
cursor.execute("CREATE TABLE IF NOT EXISTS post(permalink primary key, subreddit, created_utc, score, media_fetched, post, hidden)")
|
cursor.execute("CREATE TABLE IF NOT EXISTS post(permalink primary key, subreddit, created_utc, score, media_fetched, post, hidden)")
|
||||||
cursor.execute("CREATE TABLE IF NOT EXISTS media(permalink, url , local, PRIMARY KEY (permalink, url))")
|
cursor.execute("CREATE TABLE IF NOT EXISTS media(permalink, url , local, PRIMARY KEY (permalink, url))")
|
||||||
|
cursor.execute("CREATE TABLE IF NOT EXISTS subreddit(subreddit primary key, minimum_score, fetch_by, fetch_max)")
|
||||||
connection.commit()
|
connection.commit()
|
||||||
connection.close()
|
connection.close()
|
||||||
|
|
@ -18,20 +18,18 @@ from yars.utils import download_image
|
||||||
miner = YARS()
|
miner = YARS()
|
||||||
|
|
||||||
# Function to scrape subreddit post details and save to JSON
|
# Function to scrape subreddit post details and save to JSON
|
||||||
def scrape_subreddit_data(subreddit, limit=5):
|
def scrape_subreddit_data(subreddit, minimum_score=100, pull_by="day", limit=5):
|
||||||
ret = []
|
ret = []
|
||||||
subreddit_name = subreddit[0]
|
print(f"Starting {subreddit} with min score {minimum_score}, by {pull_by}, limit {limit}")
|
||||||
minimum_score = subreddit[1]
|
|
||||||
print(f"Starting {subreddit_name}")
|
|
||||||
empty = dict()
|
empty = dict()
|
||||||
try:
|
try:
|
||||||
subreddit_posts = miner.fetch_subreddit_posts(
|
subreddit_posts = miner.fetch_subreddit_posts(
|
||||||
subreddit_name, limit=limit, category="top", time_filter=config.pull_by
|
subreddit, limit=limit, category="top", time_filter=pull_by
|
||||||
)
|
)
|
||||||
for i, post in enumerate(subreddit_posts, 1):
|
for i, post in enumerate(subreddit_posts, 1):
|
||||||
score = post.get("score", 0)
|
score = post.get("score", 0)
|
||||||
if score < minimum_score:
|
if score < minimum_score:
|
||||||
continue
|
break
|
||||||
post_data = {
|
post_data = {
|
||||||
"permalink": post.get("permalink"),
|
"permalink": post.get("permalink"),
|
||||||
"title": post.get("title", ""),
|
"title": post.get("title", ""),
|
||||||
|
|
@ -43,7 +41,7 @@ def scrape_subreddit_data(subreddit, limit=5):
|
||||||
"body": post.get("body", None),
|
"body": post.get("body", None),
|
||||||
}
|
}
|
||||||
ret.append(post_data)
|
ret.append(post_data)
|
||||||
print(f"Finished {subreddit_name}")
|
print(f"Finished {subreddit}")
|
||||||
return ret
|
return ret
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"Error occurred while scraping subreddit: {e}")
|
print(f"Error occurred while scraping subreddit: {e}")
|
||||||
|
|
@ -110,8 +108,18 @@ if __name__ == "__main__":
|
||||||
os.makedirs(config.media_dir, exist_ok=True)
|
os.makedirs(config.media_dir, exist_ok=True)
|
||||||
connection = sqlite3.connect(config.db_file)
|
connection = sqlite3.connect(config.db_file)
|
||||||
cursor = connection.cursor()
|
cursor = connection.cursor()
|
||||||
for subreddit in config.subreddits:
|
select = """
|
||||||
post_data = scrape_subreddit_data(subreddit, config.max_posts_per_pull)
|
SELECT
|
||||||
|
subreddit,
|
||||||
|
minimum_score,
|
||||||
|
fetch_by,
|
||||||
|
fetch_max
|
||||||
|
FROM
|
||||||
|
subreddit
|
||||||
|
"""
|
||||||
|
subreddits = cursor.execute(select).fetchall()
|
||||||
|
for subreddit in subreddits:
|
||||||
|
post_data = scrape_subreddit_data(subreddit[0], subreddit[1], subreddit[2], subreddit[3])
|
||||||
save_posts_to_db(post_data, cursor)
|
save_posts_to_db(post_data, cursor)
|
||||||
connection.commit()
|
connection.commit()
|
||||||
download_media(cursor)
|
download_media(cursor)
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,228 @@
|
||||||
|
<!DOCTYPE html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="UTF-8">
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||||
|
<title>Reddit, but better</title>
|
||||||
|
<style>
|
||||||
|
:root {
|
||||||
|
--dark: #2c2c2c;
|
||||||
|
--darker: #171717;
|
||||||
|
--light: #bfbfbf;
|
||||||
|
--confirm: #970000;
|
||||||
|
}
|
||||||
|
html {
|
||||||
|
height: 100%;
|
||||||
|
}
|
||||||
|
body {
|
||||||
|
background-color: var(--darker);
|
||||||
|
color: var(--light);
|
||||||
|
min-height: 100%;
|
||||||
|
margin: 0; /* Removes default browser margin */
|
||||||
|
}
|
||||||
|
img, video {
|
||||||
|
max-width: 100%;
|
||||||
|
width: auto;
|
||||||
|
height: auto;
|
||||||
|
}
|
||||||
|
div.post {
|
||||||
|
background-color: var(--dark);
|
||||||
|
border: 2px solid var(--light);
|
||||||
|
border-radius: 15px;
|
||||||
|
padding: 10px;
|
||||||
|
margin-bottom: 20px;
|
||||||
|
}
|
||||||
|
.sidebar {
|
||||||
|
outline: 2px solid var(--light);
|
||||||
|
position: sticky;
|
||||||
|
top: 0;
|
||||||
|
left: 0;
|
||||||
|
background-color: var(--dark);
|
||||||
|
display: flex;
|
||||||
|
flex-wrap: nowrap;
|
||||||
|
z-index: 1000;
|
||||||
|
}
|
||||||
|
.content {
|
||||||
|
display: flex;
|
||||||
|
flex-grow: 1; /* Takes up remaining space */
|
||||||
|
flex-direction: column;
|
||||||
|
}
|
||||||
|
/* desktop */
|
||||||
|
@media (min-aspect-ratio: 1) {
|
||||||
|
img, video {
|
||||||
|
max-height: 80vh;
|
||||||
|
}
|
||||||
|
div.post {
|
||||||
|
width: 70vw;
|
||||||
|
}
|
||||||
|
.container {
|
||||||
|
display: flex;
|
||||||
|
flex-direction: row;
|
||||||
|
}
|
||||||
|
.sidebar {
|
||||||
|
width: fit-content;
|
||||||
|
height: 100vh;
|
||||||
|
flex-direction: column;
|
||||||
|
overflow-y: auto;
|
||||||
|
padding: 5px;
|
||||||
|
margin-right: 20px;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
/* phone */
|
||||||
|
@media (max-aspect-ratio: 1) {
|
||||||
|
img, video {
|
||||||
|
max-height: 100vh;
|
||||||
|
}
|
||||||
|
div.post {
|
||||||
|
width: calc(100vw - 50px);
|
||||||
|
margin-top: 10px;
|
||||||
|
}
|
||||||
|
.container {
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
}
|
||||||
|
.sidebar {
|
||||||
|
width: 100vw;
|
||||||
|
height: 50px;
|
||||||
|
flex-direction: row;
|
||||||
|
overflow-x: auto;
|
||||||
|
align-items: center;
|
||||||
|
padding-top: 5px;
|
||||||
|
padding-bottom: 5px;
|
||||||
|
margin-bottom: 10px;
|
||||||
|
}
|
||||||
|
.content {
|
||||||
|
align-items: center;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
.sidebar a {
|
||||||
|
display: block;
|
||||||
|
color: var(--light);
|
||||||
|
text-decoration: none;
|
||||||
|
white-space: nowrap;
|
||||||
|
margin: 5px;
|
||||||
|
padding: 5px;
|
||||||
|
}
|
||||||
|
.sidebar a:hover {
|
||||||
|
background-color: var(--darker);
|
||||||
|
color: var(--light);
|
||||||
|
}
|
||||||
|
.invert {
|
||||||
|
filter: invert(1);
|
||||||
|
transition: filter 0.3s;
|
||||||
|
}
|
||||||
|
.button-wrapper {
|
||||||
|
display: flex;
|
||||||
|
width: 100%;
|
||||||
|
gap: 10px;
|
||||||
|
margin-top: 10px;
|
||||||
|
}
|
||||||
|
.button-wrapper.gallery {
|
||||||
|
gap: 5px;
|
||||||
|
}
|
||||||
|
.button-wrapper button {
|
||||||
|
flex: 1;
|
||||||
|
padding: 10px;
|
||||||
|
cursor: pointer;
|
||||||
|
background-color: var(--darker);
|
||||||
|
color: var(--light);
|
||||||
|
border: 2px solid var(--light);
|
||||||
|
border-radius: 10px;
|
||||||
|
font-size: 1.25rem;
|
||||||
|
font-weight: bold;
|
||||||
|
}
|
||||||
|
.button-wrapper button.confirm {
|
||||||
|
background-color: var(--confirm)
|
||||||
|
}
|
||||||
|
.button-wrapper button.gallery {
|
||||||
|
padding: 5px;
|
||||||
|
background-color: var(--darker);
|
||||||
|
border-radius: 5px;
|
||||||
|
border: none;
|
||||||
|
cursor: none;
|
||||||
|
}
|
||||||
|
.button-wrapper button.gallery.selected {
|
||||||
|
background-color: var(--light);
|
||||||
|
}
|
||||||
|
.text-content {
|
||||||
|
overflow: hidden;
|
||||||
|
transition: max-height 0.3s ease-out; /* Smooth transition */
|
||||||
|
max-height: 20vh;
|
||||||
|
position: relative;
|
||||||
|
}
|
||||||
|
.text-content::after {
|
||||||
|
content: "";
|
||||||
|
position: absolute;
|
||||||
|
bottom: 0;
|
||||||
|
left: 0;
|
||||||
|
width: 100%;
|
||||||
|
height: 30px;
|
||||||
|
background: linear-gradient(to bottom, rgba(255,255,255,0), var(--dark));
|
||||||
|
}
|
||||||
|
.text-content.expanded {
|
||||||
|
max-height: 1000vh;
|
||||||
|
}
|
||||||
|
.text-content.expanded::after {
|
||||||
|
display: none;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<div class="container">
|
||||||
|
<div class="sidebar">
|
||||||
|
{% for subreddit in post_subreddits %}
|
||||||
|
<a href="{{ subreddit }}">{{ subreddit }}</a>
|
||||||
|
{% endfor %}
|
||||||
|
</div>
|
||||||
|
<div class="content">
|
||||||
|
<h1>Admin Panel</h1>
|
||||||
|
<div class="post">
|
||||||
|
<h3>Subreddits</h3>
|
||||||
|
<table>
|
||||||
|
<tr>
|
||||||
|
<th>Subreddit</th>
|
||||||
|
<th>Minimum Score</th>
|
||||||
|
<th>Fetch by</th>
|
||||||
|
<th>Fetch max</th>
|
||||||
|
<th>Update</th>
|
||||||
|
</tr>
|
||||||
|
{% for subreddit in sub_subreddits %}
|
||||||
|
<tr>
|
||||||
|
<td>/r/{{ subreddit[0] }}</td>
|
||||||
|
<td>{{ subreddit[1] }}</td>
|
||||||
|
<td>{{ subreddit[2] }}</td>
|
||||||
|
<td>{{ subreddit[3] }}</td>
|
||||||
|
<td><button onclick='deleteSubreddit("{{ subreddit[0] }}")'>Delete</button></td>
|
||||||
|
</tr>
|
||||||
|
{% endfor %}
|
||||||
|
<tr></tr>
|
||||||
|
<tr></tr>
|
||||||
|
<tr>
|
||||||
|
<form method="post">
|
||||||
|
<td>/r/<input name="name" type="text"></td>
|
||||||
|
<td><input name="score" type="text" value="100"></td>
|
||||||
|
<td>
|
||||||
|
<select name="by">
|
||||||
|
<option value="day">Day</option>
|
||||||
|
<option value="week">Week</option>
|
||||||
|
</select>
|
||||||
|
</td>
|
||||||
|
<td><input name="max" type="text" value="100"></td>
|
||||||
|
<td><button type="submit">Add</button></td>
|
||||||
|
</form>
|
||||||
|
</tr>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<script>
|
||||||
|
function deleteSubreddit(name) {
|
||||||
|
fetch('/admin?name='+name, {
|
||||||
|
method: 'DELETE'
|
||||||
|
}).then(() => {
|
||||||
|
window.location.href = window.location.href;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
Loading…
Reference in New Issue