Import forum profile pictures and host them directly

app/tasks/forumtasks.py

@@ -21,25 +21,29 @@ from app.tasks import celery
 from app.utils import is_username_valid
 from app.utils.phpbbparser import getProfile, getTopicsFromForum
 import urllib.request
+from urllib.parse import urljoin
+from .usertasks import set_profile_picture_from_url
+
 
 @celery.task()
-def checkForumAccount(username, forceNoSave=False):
-	print("Checking " + username)
+def checkForumAccount(forums_username):
+	print("### Checking " + forums_username, file=sys.stderr)
 	try:
-		profile = getProfile("https://forum.minetest.net", username)
-	except OSError:
+		profile = getProfile("https://forum.minetest.net", forums_username)
+	except OSError as e:
+		print(e, file=sys.stderr)
 		return
 
 	if profile is None:
 		return
 
-	user = User.query.filter_by(forums_username=username).first()
+	user = User.query.filter_by(forums_username=forums_username).first()
 
 	# Create user
 	needsSaving = False
 	if user is None:
-		user = User(username)
-		user.forums_username = username
+		user = User(forums_username)
+		user.forums_username = forums_username
 		db.session.add(user)
 
 	# Get github username
@@ -50,33 +54,32 @@ def checkForumAccount(username, forceNoSave=False):
 		needsSaving = True
 
 	pic = profile.avatar
-	if pic and "http" in pic:
+	if pic and pic.startswith("http"):
 		pic = None
 
-	needsSaving = needsSaving or pic != user.profile_pic
-	if pic:
-		user.profile_pic = "https://forum.minetest.net/" + pic
-	else:
-		user.profile_pic = None
-
 	# Save
-	if needsSaving and not forceNoSave:
+	if needsSaving:
 		db.session.commit()
 
+	if pic:
+		pic = urljoin("https://forum.minetest.net/", pic)
+		print(f"####### Picture: {pic}", file=sys.stderr)
+		print(f"####### User pp {user.profile_pic}", file=sys.stderr)
+
+		pic_needs_replacing = user.profile_pic is None or user.profile_pic == "" or \
+				user.profile_pic.startswith("https://forum.minetest.net")
+		if pic_needs_replacing and pic.startswith("https://forum.minetest.net"):
+			print(f"####### Queueing", file=sys.stderr)
+			set_profile_picture_from_url.delay(user.username, pic)
+
 	return needsSaving
 
 
 @celery.task()
-def checkAllForumAccounts(forceNoSave=False):
-	needsSaving = False
+def checkAllForumAccounts():
 	query = User.query.filter(User.forums_username.isnot(None))
 	for user in query.all():
-		needsSaving = checkForumAccount(user.username) or needsSaving
-
-	if needsSaving and not forceNoSave:
-		db.session.commit()
-
-	return needsSaving
+		checkForumAccount(user.forums_username)
 
 
 regex_tag    = re.compile(r"\[([a-z0-9_]+)\]")

app/tasks/usertasks.py

@@ -1,5 +1,5 @@
 # ContentDB
-# Copyright (C) 2021 rubenwardy
+# Copyright (C) 2021-23 rubenwardy
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -15,13 +15,17 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 
-import datetime
+import datetime, requests
+import os
+import sys
 
 from sqlalchemy import or_, and_
 
+from app import app
 from app.models import User, db, UserRank, ThreadReply, Package
+from app.utils import randomString
 from app.utils.models import create_session
-from app.tasks import celery
+from app.tasks import celery, TaskError
 
 
 @celery.task()
@@ -46,3 +50,45 @@ def upgrade_new_members():
 				User.packages.any(Package.approved_at < threshold)))).update({"rank": UserRank.MEMBER}, synchronize_session=False)
 
 		session.commit()
+
+
+@celery.task()
+def set_profile_picture_from_url(username: str, url: str):
+	print("### Setting pp for " + username + " to " + url, file=sys.stderr)
+	user = User.query.filter_by(username=username).first()
+	if user is None:
+		raise TaskError(f"Unable to find user {username}")
+
+	headers = {"Accept": "image/jpeg, image/png, image/gif"}
+	resp = requests.get(url, stream=True, headers=headers, timeout=15)
+	if resp.status_code != 200:
+		raise TaskError(f"Failed to download {url}: {resp.status_code}: {resp.reason}")
+
+	content_type = resp.headers["content-type"]
+	if content_type is None:
+		raise TaskError("Content-Type needed")
+	elif content_type == "image/jpeg":
+		ext = "jpg"
+	elif content_type == "image/png":
+		ext = "png"
+	elif content_type == "image/gif":
+		ext = "gif"
+	else:
+		raise TaskError(f"Unacceptable content-type: {content_type}")
+
+	filename = randomString(10) + "." + ext
+	filepath = os.path.join(app.config["UPLOAD_DIR"], filename)
+	with open(filepath, "wb") as f:
+		size = 0
+		for chunk in resp.iter_content(chunk_size=1024):
+			if chunk:  # filter out keep-alive new chunks
+				size += len(chunk)
+				if size > 3 * 1000 * 1000:  # 3 MB
+					raise TaskError(f"File too large to download {url}")
+
+				f.write(chunk)
+
+	user.profile_pic = "/uploads/" + filename
+	db.session.commit()
+
+	return filepath