Add import users from Krock's mod list feature

app/tasks/forumtasks.py

@@ -1,9 +1,11 @@
-import flask
+import flask, json
 from flask.ext.sqlalchemy import SQLAlchemy
 from app import app
 from app.models import *
 from app.tasks import celery
 from .phpbbparser import getProfile
+import urllib.request
+from urllib.parse import urlparse, quote_plus
 
 @celery.task()
 def checkForumAccount(username, token=None):
@@ -24,10 +26,35 @@ def checkForumAccount(username, token=None):
 	# Get github username
 	github_username = profile.get("github")
 	if github_username is not None and github_username.strip() != "":
-		print("Updated github username")
+		print("Updated github username for " + user.display_name + " to " + github_username)
 		user.github_username = github_username
 		needsSaving = True
 
 	# Save
 	if needsSaving:
 		db.session.commit()
+
+@celery.task()
+def importUsersFromModList():
+	contents = urllib.request.urlopen("http://krock-works.16mb.com/MTstuff/modList.php").read().decode("utf-8")
+	list = json.loads(contents)
+	found = {}
+	imported = []
+
+	for user in User.query.all():
+		found[user.username] = True
+		if user.forums_username is not None:
+			found[user.forums_username] = True
+
+	for x in list:
+		author = x.get("author")
+		if author is not None and not author in found:
+			user = User(author)
+			user.forums_username = author
+			imported.append(author)
+			found[author] = True
+			db.session.add(user)
+
+	db.session.commit()
+	for author in found:
+		checkForumAccount.delay(author, None)

app/tasks/phpbbparser.py

@@ -3,7 +3,10 @@ from bs4 import *
 from urllib.parse import urljoin
 import urllib.request
 import os.path
-import time
+import time, re
+
+def urlEncodeNonAscii(b):
+	return re.sub('[\x80-\xFF]', lambda c: '%%%02x' % ord(c.group(0)), b)
 
 class Profile:
 	def __init__(self, username):
@@ -58,7 +61,7 @@ def __extract_signature(soup):
 		return res[0]
 
 def getProfile(url, username):
-	url = url + "/memberlist.php?mode=viewprofile&un=" + username
+	url = url + "/memberlist.php?mode=viewprofile&un=" + urlEncodeNonAscii(username)
 
 	contents = urllib.request.urlopen(url).read().decode("utf-8")
 	soup = BeautifulSoup(contents, "lxml")