Add admin page to manage subreddit subscriptions
This commit is contained in:
parent
3d2edfd5cf
commit
9fc267ec6a
|
|
@ -6,7 +6,7 @@ You can change the host port, host volume directories, how often reddit is scann
|
|||
|
||||
## ./app/config.py
|
||||
|
||||
You can change how much data is pulled, from where, the minimum score to save it to your DB, and how long it is retained.
|
||||
You can change how many posts are displayed per page load and how long data is retained.
|
||||
|
||||
### Startup
|
||||
|
||||
|
|
@ -15,13 +15,13 @@ docker compose build
|
|||
docker compose up
|
||||
```
|
||||
|
||||
The DB is created automatically. You will want to run
|
||||
The DB is created automatically. You will want to visit the /admin endpoint to set up your subreddits, then run
|
||||
|
||||
```
|
||||
docker exec -it reddit-web-1 sh -c "python3 /app/scrape_posts.py"
|
||||
```
|
||||
|
||||
to populate the DB with initial data, or you will have to wait for the scheduled task to get triggered for the web page to be usable.
|
||||
to populate the DB with initial data, or you will have to wait for the scheduled task to get triggered for posts to start showing.
|
||||
|
||||
### Thanks
|
||||
|
||||
|
|
|
|||
63
app/app.py
63
app/app.py
|
|
@ -40,8 +40,70 @@ def hide_post(permalink):
|
|||
|
||||
@app.route('/')
|
||||
def index():
|
||||
connection = sqlite3.connect(config.db_file)
|
||||
cursor = connection.cursor()
|
||||
select = """
|
||||
SELECT
|
||||
count(*)
|
||||
FROM
|
||||
subreddit
|
||||
"""
|
||||
count = cursor.execute(select).fetchone()[0]
|
||||
if count == 0:
|
||||
return admin()
|
||||
return front_page()
|
||||
|
||||
@app.route('/admin', methods=['GET', 'POST', 'DELETE'])
|
||||
def admin():
|
||||
connection = sqlite3.connect(config.db_file)
|
||||
cursor = connection.cursor()
|
||||
if request.method == 'DELETE':
|
||||
delete = """
|
||||
DELETE FROM
|
||||
subreddit
|
||||
WHERE
|
||||
subreddit = ?
|
||||
"""
|
||||
binds = [request.args.get("name")]
|
||||
cursor.execute(delete, binds)
|
||||
connection.commit()
|
||||
connection.close()
|
||||
return ""
|
||||
elif request.method == 'POST':
|
||||
upsert = """
|
||||
INSERT INTO
|
||||
subreddit (subreddit, minimum_score, fetch_by, fetch_max)
|
||||
VALUES
|
||||
(?, ?, ?, ?)
|
||||
ON CONFLICT
|
||||
(subreddit)
|
||||
DO UPDATE SET
|
||||
minimum_score=excluded.minimum_score,
|
||||
fetch_by=excluded.fetch_by,
|
||||
fetch_max=excluded.fetch_max
|
||||
"""
|
||||
binds = [
|
||||
request.form.get("name"),
|
||||
int(request.form.get("score")),
|
||||
request.form.get("by"),
|
||||
int(request.form.get("max"))
|
||||
]
|
||||
cursor.execute(upsert, binds)
|
||||
connection.commit()
|
||||
post_subreddits = get_subreddits(cursor)
|
||||
select = """
|
||||
SELECT
|
||||
subreddit,
|
||||
minimum_score,
|
||||
fetch_by,
|
||||
fetch_max
|
||||
FROM
|
||||
subreddit
|
||||
"""
|
||||
sub_subreddits = cursor.execute(select).fetchall()
|
||||
connection.close()
|
||||
return render_template('admin.html', post_subreddits=post_subreddits, sub_subreddits=sub_subreddits)
|
||||
|
||||
@app.route('/r/all')
|
||||
def front_page():
|
||||
title = "/r/all"
|
||||
|
|
@ -156,6 +218,7 @@ def get_subreddits(cursor):
|
|||
subreddits = [f"/r/{sub[0]}" for sub in results]
|
||||
subreddits.insert(0, "/r/all")
|
||||
subreddits.append("/r/other")
|
||||
subreddits.append("/admin")
|
||||
return subreddits
|
||||
|
||||
def get_posts_from_select(cursor, select, binds):
|
||||
|
|
|
|||
|
|
@ -1,27 +1,4 @@
|
|||
# Scheduler configuration
|
||||
max_posts_per_pull = 100
|
||||
pull_by = "day"
|
||||
subreddits = [
|
||||
# name, minimum upvotes
|
||||
("pcgaming", 50),
|
||||
("gadgets", 10),
|
||||
("Nightreign", 100),
|
||||
("CuratedTumblr", 100),
|
||||
("196", 100),
|
||||
("PoliticalCompassMemes", 100),
|
||||
("meirl", 100),
|
||||
("me_irl", 100),
|
||||
("Fauxmoi", 100),
|
||||
("NoFilterNews", 100),
|
||||
("linux", 100),
|
||||
("linux4noobs", 100),
|
||||
("selfhosted", 100),
|
||||
("HomeServer", 100),
|
||||
("homelab", 100),
|
||||
("NonPoliticalTwitter", 100),
|
||||
("comics", 100),
|
||||
("all", 1000)
|
||||
]
|
||||
max_age_days = 30
|
||||
max_age_seconds = max_age_days * 24 * 60 * 60
|
||||
other_posts_cutoff = 1 #subreddits with this many unread posts or fewer are merged to /r/other
|
||||
|
|
|
|||
|
|
@ -8,5 +8,6 @@ connection = sqlite3.connect(config.db_file)
|
|||
cursor = connection.cursor()
|
||||
cursor.execute("CREATE TABLE IF NOT EXISTS post(permalink primary key, subreddit, created_utc, score, media_fetched, post, hidden)")
|
||||
cursor.execute("CREATE TABLE IF NOT EXISTS media(permalink, url , local, PRIMARY KEY (permalink, url))")
|
||||
cursor.execute("CREATE TABLE IF NOT EXISTS subreddit(subreddit primary key, minimum_score, fetch_by, fetch_max)")
|
||||
connection.commit()
|
||||
connection.close()
|
||||
|
|
@ -18,20 +18,18 @@ from yars.utils import download_image
|
|||
miner = YARS()
|
||||
|
||||
# Function to scrape subreddit post details and save to JSON
|
||||
def scrape_subreddit_data(subreddit, limit=5):
|
||||
def scrape_subreddit_data(subreddit, minimum_score=100, pull_by="day", limit=5):
|
||||
ret = []
|
||||
subreddit_name = subreddit[0]
|
||||
minimum_score = subreddit[1]
|
||||
print(f"Starting {subreddit_name}")
|
||||
print(f"Starting {subreddit} with min score {minimum_score}, by {pull_by}, limit {limit}")
|
||||
empty = dict()
|
||||
try:
|
||||
subreddit_posts = miner.fetch_subreddit_posts(
|
||||
subreddit_name, limit=limit, category="top", time_filter=config.pull_by
|
||||
subreddit, limit=limit, category="top", time_filter=pull_by
|
||||
)
|
||||
for i, post in enumerate(subreddit_posts, 1):
|
||||
score = post.get("score", 0)
|
||||
if score < minimum_score:
|
||||
continue
|
||||
break
|
||||
post_data = {
|
||||
"permalink": post.get("permalink"),
|
||||
"title": post.get("title", ""),
|
||||
|
|
@ -43,7 +41,7 @@ def scrape_subreddit_data(subreddit, limit=5):
|
|||
"body": post.get("body", None),
|
||||
}
|
||||
ret.append(post_data)
|
||||
print(f"Finished {subreddit_name}")
|
||||
print(f"Finished {subreddit}")
|
||||
return ret
|
||||
except Exception as e:
|
||||
print(f"Error occurred while scraping subreddit: {e}")
|
||||
|
|
@ -110,8 +108,18 @@ if __name__ == "__main__":
|
|||
os.makedirs(config.media_dir, exist_ok=True)
|
||||
connection = sqlite3.connect(config.db_file)
|
||||
cursor = connection.cursor()
|
||||
for subreddit in config.subreddits:
|
||||
post_data = scrape_subreddit_data(subreddit, config.max_posts_per_pull)
|
||||
select = """
|
||||
SELECT
|
||||
subreddit,
|
||||
minimum_score,
|
||||
fetch_by,
|
||||
fetch_max
|
||||
FROM
|
||||
subreddit
|
||||
"""
|
||||
subreddits = cursor.execute(select).fetchall()
|
||||
for subreddit in subreddits:
|
||||
post_data = scrape_subreddit_data(subreddit[0], subreddit[1], subreddit[2], subreddit[3])
|
||||
save_posts_to_db(post_data, cursor)
|
||||
connection.commit()
|
||||
download_media(cursor)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,228 @@
|
|||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Reddit, but better</title>
|
||||
<style>
|
||||
:root {
|
||||
--dark: #2c2c2c;
|
||||
--darker: #171717;
|
||||
--light: #bfbfbf;
|
||||
--confirm: #970000;
|
||||
}
|
||||
html {
|
||||
height: 100%;
|
||||
}
|
||||
body {
|
||||
background-color: var(--darker);
|
||||
color: var(--light);
|
||||
min-height: 100%;
|
||||
margin: 0; /* Removes default browser margin */
|
||||
}
|
||||
img, video {
|
||||
max-width: 100%;
|
||||
width: auto;
|
||||
height: auto;
|
||||
}
|
||||
div.post {
|
||||
background-color: var(--dark);
|
||||
border: 2px solid var(--light);
|
||||
border-radius: 15px;
|
||||
padding: 10px;
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
.sidebar {
|
||||
outline: 2px solid var(--light);
|
||||
position: sticky;
|
||||
top: 0;
|
||||
left: 0;
|
||||
background-color: var(--dark);
|
||||
display: flex;
|
||||
flex-wrap: nowrap;
|
||||
z-index: 1000;
|
||||
}
|
||||
.content {
|
||||
display: flex;
|
||||
flex-grow: 1; /* Takes up remaining space */
|
||||
flex-direction: column;
|
||||
}
|
||||
/* desktop */
|
||||
@media (min-aspect-ratio: 1) {
|
||||
img, video {
|
||||
max-height: 80vh;
|
||||
}
|
||||
div.post {
|
||||
width: 70vw;
|
||||
}
|
||||
.container {
|
||||
display: flex;
|
||||
flex-direction: row;
|
||||
}
|
||||
.sidebar {
|
||||
width: fit-content;
|
||||
height: 100vh;
|
||||
flex-direction: column;
|
||||
overflow-y: auto;
|
||||
padding: 5px;
|
||||
margin-right: 20px;
|
||||
}
|
||||
}
|
||||
/* phone */
|
||||
@media (max-aspect-ratio: 1) {
|
||||
img, video {
|
||||
max-height: 100vh;
|
||||
}
|
||||
div.post {
|
||||
width: calc(100vw - 50px);
|
||||
margin-top: 10px;
|
||||
}
|
||||
.container {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
}
|
||||
.sidebar {
|
||||
width: 100vw;
|
||||
height: 50px;
|
||||
flex-direction: row;
|
||||
overflow-x: auto;
|
||||
align-items: center;
|
||||
padding-top: 5px;
|
||||
padding-bottom: 5px;
|
||||
margin-bottom: 10px;
|
||||
}
|
||||
.content {
|
||||
align-items: center;
|
||||
}
|
||||
}
|
||||
.sidebar a {
|
||||
display: block;
|
||||
color: var(--light);
|
||||
text-decoration: none;
|
||||
white-space: nowrap;
|
||||
margin: 5px;
|
||||
padding: 5px;
|
||||
}
|
||||
.sidebar a:hover {
|
||||
background-color: var(--darker);
|
||||
color: var(--light);
|
||||
}
|
||||
.invert {
|
||||
filter: invert(1);
|
||||
transition: filter 0.3s;
|
||||
}
|
||||
.button-wrapper {
|
||||
display: flex;
|
||||
width: 100%;
|
||||
gap: 10px;
|
||||
margin-top: 10px;
|
||||
}
|
||||
.button-wrapper.gallery {
|
||||
gap: 5px;
|
||||
}
|
||||
.button-wrapper button {
|
||||
flex: 1;
|
||||
padding: 10px;
|
||||
cursor: pointer;
|
||||
background-color: var(--darker);
|
||||
color: var(--light);
|
||||
border: 2px solid var(--light);
|
||||
border-radius: 10px;
|
||||
font-size: 1.25rem;
|
||||
font-weight: bold;
|
||||
}
|
||||
.button-wrapper button.confirm {
|
||||
background-color: var(--confirm)
|
||||
}
|
||||
.button-wrapper button.gallery {
|
||||
padding: 5px;
|
||||
background-color: var(--darker);
|
||||
border-radius: 5px;
|
||||
border: none;
|
||||
cursor: none;
|
||||
}
|
||||
.button-wrapper button.gallery.selected {
|
||||
background-color: var(--light);
|
||||
}
|
||||
.text-content {
|
||||
overflow: hidden;
|
||||
transition: max-height 0.3s ease-out; /* Smooth transition */
|
||||
max-height: 20vh;
|
||||
position: relative;
|
||||
}
|
||||
.text-content::after {
|
||||
content: "";
|
||||
position: absolute;
|
||||
bottom: 0;
|
||||
left: 0;
|
||||
width: 100%;
|
||||
height: 30px;
|
||||
background: linear-gradient(to bottom, rgba(255,255,255,0), var(--dark));
|
||||
}
|
||||
.text-content.expanded {
|
||||
max-height: 1000vh;
|
||||
}
|
||||
.text-content.expanded::after {
|
||||
display: none;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<div class="sidebar">
|
||||
{% for subreddit in post_subreddits %}
|
||||
<a href="{{ subreddit }}">{{ subreddit }}</a>
|
||||
{% endfor %}
|
||||
</div>
|
||||
<div class="content">
|
||||
<h1>Admin Panel</h1>
|
||||
<div class="post">
|
||||
<h3>Subreddits</h3>
|
||||
<table>
|
||||
<tr>
|
||||
<th>Subreddit</th>
|
||||
<th>Minimum Score</th>
|
||||
<th>Fetch by</th>
|
||||
<th>Fetch max</th>
|
||||
<th>Update</th>
|
||||
</tr>
|
||||
{% for subreddit in sub_subreddits %}
|
||||
<tr>
|
||||
<td>/r/{{ subreddit[0] }}</td>
|
||||
<td>{{ subreddit[1] }}</td>
|
||||
<td>{{ subreddit[2] }}</td>
|
||||
<td>{{ subreddit[3] }}</td>
|
||||
<td><button onclick='deleteSubreddit("{{ subreddit[0] }}")'>Delete</button></td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
<tr></tr>
|
||||
<tr></tr>
|
||||
<tr>
|
||||
<form method="post">
|
||||
<td>/r/<input name="name" type="text"></td>
|
||||
<td><input name="score" type="text" value="100"></td>
|
||||
<td>
|
||||
<select name="by">
|
||||
<option value="day">Day</option>
|
||||
<option value="week">Week</option>
|
||||
</select>
|
||||
</td>
|
||||
<td><input name="max" type="text" value="100"></td>
|
||||
<td><button type="submit">Add</button></td>
|
||||
</form>
|
||||
</tr>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<script>
|
||||
function deleteSubreddit(name) {
|
||||
fetch('/admin?name='+name, {
|
||||
method: 'DELETE'
|
||||
}).then(() => {
|
||||
window.location.href = window.location.href;
|
||||
});
|
||||
}
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
Loading…
Reference in New Issue