Add forum topic validation

app/__init__.py

@@ -37,5 +37,9 @@ csrf = CsrfProtect(app)
 mail = Mail(app)
 pages = FlatPages(app)
 
+if not app.debug:
+	from .maillogger import register_mail_error_handler
+	register_mail_error_handler(app, mail)
+
 from . import models, tasks
 from .views import *

app/maillogger.py

@@ -0,0 +1,109 @@
+import logging
+from enum import Enum
+from app.tasks.emails import sendEmailRaw
+
+def _has_newline(line):
+	"""Used by has_bad_header to check for \\r or \\n"""
+	if line and ("\r" in line or "\n" in line):
+		return True
+	return False
+
+def _is_bad_subject(subject):
+	"""Copied from: flask_mail.py class Message def has_bad_headers"""
+	if _has_newline(subject):
+		for linenum, line in enumerate(subject.split("\r\n")):
+			if not line:
+				return True
+			if linenum > 0 and line[0] not in "\t ":
+				return True
+			if _has_newline(line):
+				return True
+			if len(line.strip()) == 0:
+				return True
+	return False
+
+
+class FlaskMailSubjectFormatter(logging.Formatter):
+	def format(self, record):
+		record.message = record.getMessage()
+		if self.usesTime():
+			record.asctime = self.formatTime(record, self.datefmt)
+		s = self.formatMessage(record)
+		return s
+
+class FlaskMailTextFormatter(logging.Formatter):
+	pass
+
+# TODO: hier nog niet tevreden over (vooral logger.error(..., exc_info, stack_info))
+class FlaskMailHTMLFormatter(logging.Formatter):
+	pre_template = "<h1>%s</h1><pre>%s</pre>"
+	def formatException(self, exc_info):
+		formatted_exception = logging.Handler.formatException(self, exc_info)
+		return FlaskMailHTMLFormatter.pre_template % ("Exception information", formatted_exception)
+	def formatStack(self, stack_info):
+		return FlaskMailHTMLFormatter.pre_template % ("<h1>Stack information</h1><pre>%s</pre>", stack_info)
+
+
+# see: https://github.com/python/cpython/blob/3.6/Lib/logging/__init__.py (class Handler)
+
+class FlaskMailHandler(logging.Handler):
+	def __init__(self, mailer, subject_template, level=logging.NOTSET):
+		logging.Handler.__init__(self, level)
+		self.mailer = mailer
+		self.send_to = mailer.app.config["MAIL_UTILS_ERROR_SEND_TO"]
+		self.subject_template = subject_template
+		self.html_formatter = None
+
+	def setFormatter(self, text_fmt, html_fmt=None):
+		"""
+		Set the formatters for this handler. Provide at least one formatter.
+		When no text_fmt is provided, no text-part is created for the email body.
+		"""
+		assert (text_fmt, html_fmt) != (None, None), "At least one formatter should be provided"
+		if type(text_fmt)==str:
+			text_fmt = FlaskMailTextFormatter(text_fmt)
+		self.formatter = text_fmt
+		if type(html_fmt)==str:
+			html_fmt = FlaskMailHTMLFormatter(html_fmt)
+		self.html_formatter = html_fmt
+
+	def getSubject(self, record):
+		fmt = FlaskMailSubjectFormatter(self.subject_template)
+		subject = fmt.format(record)
+		#Since templates can cause header problems, and we rather have a incomplete email then an error, we fix this
+		if _is_bad_subject(subject):
+			subject="FlaskMailHandler log-entry from %s [original subject is replaced, because it would result in a bad header]" % self.mailer.app.name
+		return subject
+
+	def emit(self, record):
+		text = self.format(record)				if self.formatter	  else None
+		html = self.html_formatter.format(record) if self.html_formatter else None
+		sendEmailRaw.delay(self.send_to, self.getSubject(record), text, html)
+
+
+def register_mail_error_handler(app, mailer):
+	subject_template = "ContentDB crashed (%(module)s > %(funcName)s)"
+	text_template = """
+Message type: %(levelname)s
+Location:	 %(pathname)s:%(lineno)d
+Module:	   %(module)s
+Function:	 %(funcName)s
+Time:		 %(asctime)s
+Message:
+%(message)s"""
+	html_template = """
+<style>th { text-align: right}</style><table>
+<tr><th>Message type:</th><td>%(levelname)s</td></tr>
+<tr>	<th>Location:</th><td>%(pathname)s:%(lineno)d</td></tr>
+<tr>	  <th>Module:</th><td>%(module)s</td></tr>
+<tr>	<th>Function:</th><td>%(funcName)s</td></tr>
+<tr>		<th>Time:</th><td>%(asctime)s</td></tr>
+</table>
+<h2>Message</h2>
+<pre>%(message)s</pre>"""
+
+	import logging
+	mail_handler = FlaskMailHandler(mailer, subject_template)
+	mail_handler.setLevel(logging.ERROR)
+	mail_handler.setFormatter(text_template, html_template)
+	app.logger.addHandler(mail_handler)

app/models.py

@@ -339,6 +339,8 @@ class Package(db.Model):
 	approved     = db.Column(db.Boolean, nullable=False, default=False)
 	soft_deleted = db.Column(db.Boolean, nullable=False, default=False)
 
+	score        = db.Column(db.Float, nullable=False, default=0)
+
 	review_thread_id = db.Column(db.Integer, db.ForeignKey("thread.id"), nullable=True, default=None)
 	review_thread    = db.relationship("Thread", foreign_keys=[review_thread_id])
 
@@ -385,7 +387,8 @@ class Package(db.Model):
 			"shortDesc": self.shortDesc,
 			"type": self.type.toName(),
 			"release": self.getDownloadRelease().id if self.getDownloadRelease() is not None else None,
-			"thumbnail": (base_url + tnurl) if tnurl is not None else None
+			"thumbnail": (base_url + tnurl) if tnurl is not None else None,
+			"score": round(self.score * 10) / 10
 		}
 
 	def getAsDictionary(self, base_url):
@@ -412,7 +415,9 @@ class Package(db.Model):
 			"screenshots": [base_url + ss.url for ss in self.screenshots],
 
 			"url": base_url + self.getDownloadURL(),
-			"release": self.getDownloadRelease().id if self.getDownloadRelease() is not None else None
+			"release": self.getDownloadRelease().id if self.getDownloadRelease() is not None else None,
+
+			"score": round(self.score * 10) / 10
 		}
 
 	def getThumbnailURL(self):
@@ -498,6 +503,21 @@ class Package(db.Model):
 		else:
 			raise Exception("Permission {} is not related to packages".format(perm.name))
 
+	def recalcScore(self):
+		import datetime
+
+		self.score = 0
+
+		if self.forums is None:
+			return
+
+		topic = ForumTopic.query.get(self.forums)
+		if topic:
+			days   = (datetime.datetime.now() - topic.created_at).days
+			months = days / 30
+			years  = days / 365
+			self.score = topic.views / years + 80*min(6, months)
+
 class MetaPackage(db.Model):
 	id           = db.Column(db.Integer, primary_key=True)
 	name         = db.Column(db.String(100), unique=True, nullable=False)
@@ -743,29 +763,47 @@ REPO_BLACKLIST = [".zip", "mediafire.com", "dropbox.com", "weebly.com", \
 		"digitalaudioconcepts.com", "hg.intevation.org", "www.wtfpl.net", \
 		"imageshack.com", "imgur.com"]
 
-class KrockForumTopic(db.Model):
+class ForumTopic(db.Model):
 	topic_id  = db.Column(db.Integer, primary_key=True, autoincrement=False)
 	author_id = db.Column(db.Integer, db.ForeignKey("user.id"), nullable=False)
 	author    = db.relationship("User")
 
-	ttype     = db.Column(db.Integer, nullable=False)
+	wip       = db.Column(db.Boolean, server_default="0")
+
+	type      = db.Column(db.Enum(PackageType), nullable=False)
 	title     = db.Column(db.String(200), nullable=False)
 	name      = db.Column(db.String(30), nullable=True)
 	link      = db.Column(db.String(200), nullable=True)
 
-	def getType(self):
-		if self.ttype == 1 or self.ttype == 2:
-			return PackageType.MOD
-		elif self.ttype == 6:
-			return PackageType.GAME
+	posts     = db.Column(db.Integer, nullable=False)
+	views     = db.Column(db.Integer, nullable=False)
+
+	created_at = db.Column(db.DateTime, nullable=False, default=datetime.utcnow)
 
 	def getRepoURL(self):
+		if self.link is None:
+			return None
+
 		for item in REPO_BLACKLIST:
 			if item in self.link:
 				return None
 
 		return self.link.replace("repo.or.cz/w/", "repo.or.cz/")
 
+	def getAsDictionary(self):
+		return {
+			"author": self.author.username,
+			"name":   self.name,
+			"type":   self.type.toName(),
+			"title":  self.title,
+			"id":     self.topic_id,
+			"link":   self.link,
+			"posts":  self.posts,
+			"views":  self.views,
+			"is_wip": self.wip,
+			"created_at": self.created_at.isoformat(),
+		}
+
 
 # Setup Flask-User
 db_adapter = SQLAlchemyAdapter(db, User)        # Register the User model

app/scss/components.scss

@@ -451,3 +451,7 @@ table.fancyTable tfoot td {
 .table-topalign td {
 	vertical-align: top;
 }
+
+.wiptopic a {
+	color: #7ac;
+}

app/tasks/emails.py

@@ -22,7 +22,17 @@ from app.tasks import celery
 
 @celery.task()
 def sendVerifyEmail(newEmail, token):
-    msg = Message("Verify email address", recipients=[newEmail])
-    msg.body = "This is a verification email!"
-    msg.html = render_template("emails/verify.html", token=token)
-    mail.send(msg)
+	msg = Message("Verify email address", recipients=[newEmail])
+	msg.body = "This is a verification email!"
+	msg.html = render_template("emails/verify.html", token=token)
+	mail.send(msg)
+
+@celery.task()
+def sendEmailRaw(to, subject, text, html):
+	from flask_mail import Message
+	msg = Message(subject, recipients=to)
+	if text:
+		msg.body = text
+	if html:
+		msg.html = html
+	mail.send(msg)

app/tasks/forumtasks.py

@@ -15,12 +15,12 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 
-import flask, json
+import flask, json, re
 from flask.ext.sqlalchemy import SQLAlchemy
 from app import app
 from app.models import *
 from app.tasks import celery
-from .phpbbparser import getProfile
+from .phpbbparser import getProfile, getTopicsFromForum
 import urllib.request
 from urllib.parse import urlparse, quote_plus
 
@@ -51,71 +51,91 @@ def checkForumAccount(username, token=None):
 	if needsSaving:
 		db.session.commit()
 
-@celery.task()
-def importUsersFromModList():
+
+regex_tag    = re.compile(r"\[([a-z0-9_]+)\]")
+BANNED_NAMES = ["mod", "game", "old", "outdated", "wip", "api", "beta", "alpha", "git"]
+def getNameFromTaglist(taglist):
+	for tag in reversed(regex_tag.findall(taglist)):
+		if len(tag) < 30 and not tag in BANNED_NAMES and \
+				not re.match(r"^[a-z]?[0-9]+$", tag):
+			return tag
+
+	return None
+
+regex_title = re.compile(r"^((?:\[[^\]]+\] *)*)([^\[]+) *((?:\[[^\]]+\] *)*)[^\[]*$")
+def parseTitle(title):
+	m = regex_title.match(title)
+	if m is None:
+		print("Invalid title format: " + title)
+		return title, getNameFromTaglist(title)
+	else:
+		return m.group(2).strip(), getNameFromTaglist(m.group(3))
+
+def getLinksFromModSearch():
+	links = {}
+
 	contents = urllib.request.urlopen("http://krock-works.16mb.com/MTstuff/modList.php").read().decode("utf-8")
-	list = json.loads(contents)
-	found = {}
-	imported = []
+	for x in json.loads(contents):
+		link = x.get("link")
+		if link is not None:
+			links[int(x["topicId"])] = link
 
-	for user in User.query.all():
-		found[user.username] = True
-		if user.forums_username is not None:
-			found[user.forums_username] = True
-
-	for x in list:
-		author = x.get("author")
-		if author is not None and not author in found:
-			user = User(author)
-			user.forums_username = author
-			imported.append(author)
-			found[author] = True
-			db.session.add(user)
-
-	db.session.commit()
-	for author in found:
-		checkForumAccount.delay(author, None)
-
-
-BANNED_NAMES = ["mod", "game", "old", "outdated", "wip", "api"]
-ALLOWED_TYPES = [1, 2, 6]
+	return links
 
 @celery.task()
-def importKrocksModList():
-	contents = urllib.request.urlopen("http://krock-works.16mb.com/MTstuff/modList.php").read().decode("utf-8")
-	list = json.loads(contents)
+def importTopicList():
+	links_by_id = getLinksFromModSearch()
+
+	info_by_id = {}
+	getTopicsFromForum(11, out=info_by_id, extra={ 'type': PackageType.MOD,  'wip': False })
+	getTopicsFromForum(9,  out=info_by_id, extra={ 'type': PackageType.MOD,  'wip': True  })
+	getTopicsFromForum(15, out=info_by_id, extra={ 'type': PackageType.GAME, 'wip': False })
+	getTopicsFromForum(50, out=info_by_id, extra={ 'type': PackageType.GAME, 'wip': True  })
+
+	# Caches
 	username_to_user = {}
+	topics_by_id     = {}
+	for topic in ForumTopic.query.all():
+		topics_by_id[topic.topic_id] = topic
 
-	KrockForumTopic.query.delete()
+	# Create or update
+	for info in info_by_id.values():
+		id = int(info["id"])
 
-	for x in list:
-		type = int(x["type"])
-		if not type in ALLOWED_TYPES:
-			continue
-
-		username = x["author"]
+		# Get author
+		username = info["author"]
 		user = username_to_user.get(username)
 		if user is None:
 			user = User.query.filter_by(forums_username=username).first()
-			assert(user is not None)
+			if user is None:
+				print(username + " not found!")
+				user = User(username)
+				user.forums_username = username
+				db.session.add(user)
 			username_to_user[username] = user
 
-		import re
-		tags = re.findall("\[([a-z0-9_]+)\]", x["title"])
-		name = None
-		for tag in reversed(tags):
-			if len(tag) < 30 and not tag in BANNED_NAMES and \
-					not re.match("^([a-z][0-9]+)$", tag):
-				name = tag
-				break
+		# Get / add row
+		topic = topics_by_id.get(id)
+		if topic is None:
+			topic = ForumTopic()
+			db.session.add(topic)
 
-		topic = KrockForumTopic()
-		topic.topic_id  = x["topicId"]
-		topic.author_id = user.id
-		topic.ttype     = type
-		topic.title     = x["title"]
-		topic.name      = name
-		topic.link      = x.get("link")
-		db.session.add(topic)
+		# Parse title
+		title, name = parseTitle(info["title"])
+
+		# Get link
+		link = links_by_id.get(id)
+
+		# Fill row
+		topic.topic_id   = id
+		topic.author     = user
+		topic.type       = info["type"]
+		topic.title      = title
+		topic.name       = name
+		topic.link       = link
+		topic.wip        = info["wip"]
+		topic.posts      = info["posts"]
+		topic.views      = info["views"]
+		topic.created_at = info["date"]
 
 	db.session.commit()

app/tasks/phpbbparser.py

@@ -5,6 +5,7 @@
 import urllib, socket
 from bs4 import *
 from urllib.parse import urljoin
+from datetime import datetime
 import urllib.request
 import os.path
 import time, re
@@ -77,3 +78,72 @@ def getProfile(url, username):
 		__extract_properties(profile, soup)
 
 		return profile
+
+
+regex_id = re.compile(r"^.*t=([0-9]+).*$")
+
+def parseForumListPage(id, page, out, extra=None):
+	num_per_page = 30
+	start = page*num_per_page+1
+	print(" - Fetching page {} (topics {}-{})".format(page, start, start+num_per_page))
+
+	url = "https://forum.minetest.net/viewforum.php?f=" + str(id) + "&start=" + str(start)
+	r = urllib.request.urlopen(url).read().decode("utf-8")
+	soup = BeautifulSoup(r, "html.parser")
+
+	for row in soup.find_all("li", class_="row"):
+		classes = row.get("class")
+		if "sticky" in classes or "announce" in classes or "global-announce" in classes:
+			continue
+
+		topic = row.find("dl")
+
+		# Link info
+		link   = topic.find(class_="topictitle")
+		id	   = regex_id.match(link.get("href")).group(1)
+		title  = link.find(text=True)
+
+		# Date
+		left   = topic.find("dt")
+		date   = left.get_text().split("»")[1].strip()
+		date   = datetime.strptime(date, "%a %b %d, %Y %H:%M")
+		author = left.find_all("a")[-1].get_text().strip()
+
+		# Get counts
+		posts  = topic.find(class_="posts").find(text=True)
+		views  = topic.find(class_="views").find(text=True)
+
+		if id in out:
+			print("   - got {} again, title: {}".format(id, title))
+			assert(title == out[id]['title'])
+			return False
+
+		row = {
+			"id"    : id,
+			"title" : title,
+			"author": author,
+			"posts" : posts,
+			"views" : views,
+			"date"  : date
+		}
+
+		if extra is not None:
+			for key, value in extra.items():
+				row[key] = value
+
+		out[id] = row
+
+	return False
+
+def getTopicsFromForum(id, out={}, extra=None):
+	print("Fetching all topics from forum {}".format(id))
+	page = 0
+	while parseForumListPage(id, page, out, extra):
+		page = page + 1
+
+	return out
+
+def dumpTitlesToFile(topics, path):
+	with open(path, "w") as out_file:
+		for topic in topics.values():
+			out_file.write(topic["title"] + "\n")

app/templates/admin/list.html

@@ -17,11 +17,11 @@
 		<form method="post" action="" class="box-body">
 			<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
 			<select name="action">
-				<option value="importusers">Create users from mod list</option>
-				<option value="importmodlist">Import Krock's mod list</option>
+				<option value="importmodlist">Import forum topics</option>
 				<option value="importscreenshots" selected>Import screenshots from VCS</option>
 				<option value="importdepends">Import dependencies from downloads</option>
 				<option value="modprovides">Set provides to mod name</option>
+				<option value="recalcscores">Recalc pakage scores</option>
 			</select>
 			<input type="submit" value="Perform" />
 		</form>

app/templates/index.html

@@ -12,7 +12,7 @@ Welcome
 		<p>
 			Minetest's official content repository.
 			Browse {{ count }} packages,
-			all available under a free and open source
+			majority of which available under a free and open source
 			license.
 		</p>
 
@@ -25,6 +25,14 @@ Welcome
 
 <main>
 	{% from "macros/packagegridtile.html" import render_pkggrid %}
-	{{ render_pkggrid(packages) }}
+
+	<h2>Popular</h2>
+	{{ render_pkggrid(popular) }}
+
+	<a href="{{ url_for('packages_page') }}" class="button">Show More</a>
+
+	<h2 style="margin-top:2em;">Newly Added</h2>
+	{{ render_pkggrid(new) }}
+
 </main>
 {% endblock %}

app/templates/macros/topictable.html

@@ -2,6 +2,7 @@
 <table>
 	<tr>
 		<th>Id</th>
+		<th></th>
 		<th>Title</th>
 		{% if show_author %}<th>Author</th>{% endif %}
 		<th>Name</th>
@@ -9,14 +10,20 @@
 		<th>Actions</th>
 	</tr>
 	{% for topic in topics %}
-		<tr>
+		<tr{% if topic.wip %} class="wiptopic"{% endif %}>
 			<td>{{ topic.topic_id }}</td>
-			<td>[{{ topic.getType().value }}] <a href="https://forum.minetest.net/viewtopic.php?t={{ topic.topic_id}}">{{ topic.title }}</a></td>
+			<td>
+				[{{ topic.type.value }}]
+			</td>
+			<td>
+				<a href="https://forum.minetest.net/viewtopic.php?t={{ topic.topic_id}}">{{ topic.title }}</a>
+				{% if topic.wip %}[WIP]{% endif %}
+			</td>
 			{% if show_author %}
 				<td><a href="{{ url_for('user_profile_page', username=topic.author.username) }}">{{ topic.author.display_name}}</a></td>
 			{% endif %}
 			<td>{{ topic.name or ""}}</td>
-			<td><a href="{{ topic.link }}">{{ topic.link | domain }}</a></td>
+			<td>{% if topic.link %}<a href="{{ topic.link }}">{{ topic.link | domain }}</a>{% endif %}</td>
 			<td>
 				<a href="{{ url_for('create_edit_package_page', author=topic.author.username, repo=topic.getRepoURL(), forums=topic.topic_id, title=topic.title, bname=topic.name) }}">Create</a>
 			</td>

app/templates/packages/view.html

@@ -24,6 +24,9 @@
 			{% elif (package.type == package.type.GAME or package.type == package.type.TXP) and package.screenshots.count() == 0 %}
 				You need to add at least one screenshot.
 
+			{% elif topic_error_lvl == "error" %}
+				Please fix the below topic issue(s).
+
 			{% else %}
 				{% if package.screenshots.count() == 0 %}
 					<b>You should add at least one screenshot, but this isn't required.</b><br />
@@ -44,6 +47,14 @@
 			<div style="clear: both;"></div>
 		</div>
 
+		{% if topic_error %}
+			<div class="box box_grey alert alert-{{ topic_error_lvl }}">
+				<span class="icon_message"></span>
+				{{ topic_error | safe }}
+				<div style="clear: both;"></div>
+			</div>
+		{% endif %}
+
 		{% if package.author == current_user or package.checkPerm(current_user, "APPROVE_NEW") %}
 			{% if review_thread %}
 				<h2>&#x1f512; {{ review_thread.title }}</h2>
@@ -292,10 +303,11 @@
 		<ul>
 			{% for t in similar_topics %}
 				<li>
-					[{{ t.getType().value }}]
+					[{{ t.type.value }}]
 					<a href="https://forum.minetest.net/viewtopic.php?t={{ t.topic_id }}">
 						{{ t.title }} by {{ t.author.display_name }}
 					</a>
+					{% if t.wip %}[WIP]{% endif %}
 				</li>
 			{% endfor %}
 		</ul>

app/templates/todo/list.html

@@ -65,6 +65,6 @@
 	<p>
 		There are
 		<a href="{{ url_for('todo_topics_page') }}">{{ topics_to_add }} packages</a>
-		to be added to cdb, based on forum topics picked up by Krock's mod search.
+		to be added to cdb, based on cdb's forum parser.
 	</p>
 {% endblock %}

app/templates/users/user_profile_page.html

@@ -104,12 +104,9 @@
 
 		<div class="box-body">
 			<p>
-				List of your topics without a matching package.
-				Powered by Krock's Mod Search.
+				List of your forum topics which do not have a matching package.
 			</p>
 
-			
-
 			{% from "macros/topictable.html" import render_topictable %}
 			{{ render_topictable(topics_to_add, show_author=False) }}
 		</div>

app/views/__init__.py

@@ -46,10 +46,11 @@ def send_upload(path):
 @app.route("/")
 @menu.register_menu(app, ".", "Home")
 def home_page():
-	query = Package.query.filter_by(approved=True, soft_deleted=False)
-	count = query.count()
-	packages = query.order_by(db.desc(Package.created_at)).limit(15).all()
-	return render_template("index.html", packages=packages, count=count)
+	query   = Package.query.filter_by(approved=True, soft_deleted=False)
+	count   = query.count()
+	new     = query.order_by(db.desc(Package.created_at)).limit(15).all()
+	popular = query.order_by(db.desc(Package.score)).limit(6).all()
+	return render_template("index.html", new=new, popular=popular, count=count)
 
 from . import users, githublogin, packages, meta, threads, api
 from . import sass, tasks, admin, notifications, tagseditor, thumbnails

app/views/admin.py

@@ -21,7 +21,7 @@ from flask.ext import menu
 from app import app
 from app.models import *
 from app.tasks.importtasks import importRepoScreenshot, importAllDependencies
-from app.tasks.forumtasks  import importUsersFromModList, importKrocksModList
+from app.tasks.forumtasks  import importTopicList
 from flask_wtf import FlaskForm
 from wtforms import *
 from app.utils import loginUser, rank_required
@@ -31,11 +31,8 @@ from app.utils import loginUser, rank_required
 def admin_page():
 	if request.method == "POST":
 		action = request.form["action"]
-		if action == "importusers":
-			task = importUsersFromModList.delay()
-			return redirect(url_for("check_task", id=task.id, r=url_for("user_list_page")))
-		elif action == "importmodlist":
-			task = importKrocksModList.delay()
+		if action == "importmodlist":
+			task = importTopicList.delay()
 			return redirect(url_for("check_task", id=task.id, r=url_for("todo_topics_page")))
 		elif action == "importscreenshots":
 			packages = Package.query \
@@ -67,6 +64,12 @@ def admin_page():
 
 			db.session.commit()
 			return redirect(url_for("admin_page"))
+		elif action == "recalcscores":
+			for p in Package.query.all():
+				p.recalcScore()
+
+			db.session.commit()
+			return redirect(url_for("admin_page"))
 
 		else:
 			flash("Unknown action: " + action, "error")

app/views/api.py

@@ -33,3 +33,11 @@ def api_packages_page():
 @is_package_page
 def api_package_page(package):
 	return jsonify(package.getAsDictionary(app.config["BASE_URL"]))
+
+
+@app.route("/api/topics/")
+def api_topics_page():
+	query = ForumTopic.query \
+			.order_by(db.asc(ForumTopic.wip), db.asc(ForumTopic.name), db.asc(ForumTopic.title))
+	pkgs = [t.getAsDictionary() for t in query.all()]
+	return jsonify(pkgs)

app/views/packages/__init__.py

@@ -47,6 +47,8 @@ def build_packages_query():
 	if search is not None and search.strip() != "":
 		query = query.filter(Package.title.ilike('%' + search + '%'))
 
+	query = query.order_by(db.desc(Package.score))
+
 	return query, title
 
 @menu.register_menu(app, ".mods", "Mods", order=11, endpoint_arguments_constructor=lambda: { 'type': 'mod' })
@@ -100,11 +102,11 @@ def package_page(package):
 			package.checkPerm(current_user, Permission.APPROVE_NEW)
 
 	similar_topics = None if not show_similar_topics else \
-			KrockForumTopic.query \
+			ForumTopic.query \
 				.filter_by(name=package.name) \
-				.filter(KrockForumTopic.topic_id != package.forums) \
-				.filter(~ db.exists().where(Package.forums==KrockForumTopic.topic_id)) \
-				.order_by(db.asc(KrockForumTopic.name), db.asc(KrockForumTopic.title)) \
+				.filter(ForumTopic.topic_id != package.forums) \
+				.filter(~ db.exists().where(Package.forums==ForumTopic.topic_id)) \
+				.order_by(db.asc(ForumTopic.name), db.asc(ForumTopic.title)) \
 				.all()
 
 	releases = getReleases(package)
@@ -114,10 +116,31 @@ def package_page(package):
 	if review_thread is not None and not review_thread.checkPerm(current_user, Permission.SEE_THREAD):
 		review_thread = None
 
+	topic_error = None
+	topic_error_lvl = "warning"
+	if not package.approved and package.forums is not None:
+		errors = []
+		if Package.query.filter_by(forums=package.forums, soft_deleted=False).count() > 1:
+			errors.append("<b>Error: Another package already uses this forum topic!</b>")
+			topic_error_lvl = "error"
+
+		topic = ForumTopic.query.get(package.forums)
+		if topic is not None:
+			if topic.author != package.author:
+				errors.append("<b>Error: Forum topic author doesn't match package author.</b>")
+				topic_error_lvl = "error"
+
+			if topic.wip:
+				errors.append("Warning: Forum topic is in WIP section, make sure package meets playability standards.")
+		elif package.type != PackageType.TXP:
+			errors.append("Warning: Forum topic not found. This may happen if the topic has only just been created.")
+
+		topic_error = "<br />".join(errors)
+
 	return render_template("packages/view.html", \
 			package=package, releases=releases, requests=requests, \
 			alternatives=alternatives, similar_topics=similar_topics, \
-			review_thread=review_thread)
+			review_thread=review_thread, topic_error=topic_error, topic_error_lvl=topic_error_lvl)
 
 
 @app.route("/packages/<author>/<name>/download/")

app/views/packages/todo.py

@@ -41,8 +41,8 @@ def todo_page():
 		screenshots = PackageScreenshot.query.filter_by(approved=False).all()
 
 
-	topics_to_add = KrockForumTopic.query \
-			.filter(~ db.exists().where(Package.forums==KrockForumTopic.topic_id)) \
+	topics_to_add = ForumTopic.query \
+			.filter(~ db.exists().where(Package.forums==ForumTopic.topic_id)) \
 			.count()
 
 	return render_template("todo/list.html", title="Reports and Work Queue",
@@ -54,11 +54,11 @@ def todo_page():
 @app.route("/todo/topics/")
 @login_required
 def todo_topics_page():
-	total = KrockForumTopic.query.count()
+	total = ForumTopic.query.count()
 
-	topics = KrockForumTopic.query \
-			.filter(~ db.exists().where(Package.forums==KrockForumTopic.topic_id)) \
-			.order_by(db.asc(KrockForumTopic.name), db.asc(KrockForumTopic.title)) \
+	topics = ForumTopic.query \
+			.filter(~ db.exists().where(Package.forums==ForumTopic.topic_id)) \
+			.order_by(db.asc(ForumTopic.wip), db.asc(ForumTopic.name), db.asc(ForumTopic.title)) \
 			.all()
 
 	return render_template("todo/topics.html", topics=topics, total=total)

app/views/users.py

@@ -98,10 +98,10 @@ def user_profile_page(username):
 
 	topics_to_add = None
 	if current_user == user or user.checkPerm(current_user, Permission.CHANGE_AUTHOR):
-		topics_to_add = KrockForumTopic.query \
+		topics_to_add = ForumTopic.query \
 					.filter_by(author_id=user.id) \
-					.filter(~ db.exists().where(Package.forums==KrockForumTopic.topic_id)) \
-					.order_by(db.asc(KrockForumTopic.name), db.asc(KrockForumTopic.title)) \
+					.filter(~ db.exists().where(Package.forums==ForumTopic.topic_id)) \
+					.order_by(db.asc(ForumTopic.name), db.asc(ForumTopic.title)) \
 					.all()
 
 	# Process GET or invalid POST

config.example.cfg

@@ -22,3 +22,4 @@ MAIL_DEFAULT_SENDER=""
 MAIL_SERVER=""
 MAIL_PORT=587
 MAIL_USE_TLS=True
+MAIL_UTILS_ERROR_SEND_TO=[""]

migrations/versions/11b6ef362f98_.py

@@ -0,0 +1,28 @@
+"""empty message
+
+Revision ID: 11b6ef362f98
+Revises: 9fc23495713b
+Create Date: 2018-07-04 01:01:45.440662
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '11b6ef362f98'
+down_revision = '9fc23495713b'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('package', sa.Column('score', sa.Float(), nullable=False, server_default="0.0"))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('package', 'score')
+    # ### end Alembic commands ###

migrations/versions/9e2ac631efb0_.py

@@ -0,0 +1,28 @@
+"""empty message
+
+Revision ID: 9e2ac631efb0
+Revises: 11b6ef362f98
+Create Date: 2018-07-06 23:16:50.507010
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '9e2ac631efb0'
+down_revision = '11b6ef362f98'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('forum_topic', sa.Column('wip', sa.Boolean(), nullable=False, server_default="0"))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('forum_topic', 'wip')
+    # ### end Alembic commands ###

migrations/versions/9fc23495713b_.py

@@ -0,0 +1,55 @@
+"""empty message
+
+Revision ID: 9fc23495713b
+Revises: de004661c5e1
+Create Date: 2018-07-04 00:03:20.123285
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '9fc23495713b'
+down_revision = 'de004661c5e1'
+branch_labels = None
+depends_on = None
+from sqlalchemy.dialects.postgresql import ENUM
+
+type_enum = ENUM('MOD', 'GAME', 'TXP', name='packagetype', create_type=False)
+
+def upgrade():
+    type_enum.create(op.get_bind(), checkfirst=True)
+
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('krock_forum_topic')
+    op.create_table('forum_topic',
+    sa.Column('topic_id', sa.Integer(), autoincrement=False, nullable=False),
+    sa.Column('author_id', sa.Integer(), nullable=False),
+    sa.Column('type', type_enum, nullable=True),
+    sa.Column('title', sa.String(length=200), nullable=False),
+    sa.Column('name', sa.String(length=30), nullable=True),
+    sa.Column('link', sa.String(length=200), nullable=True),
+    sa.Column('posts', sa.Integer(), nullable=False),
+    sa.Column('views', sa.Integer(), nullable=False),
+    sa.Column('created_at', sa.DateTime(), nullable=False),
+    sa.ForeignKeyConstraint(['author_id'], ['user.id'], ),
+    sa.PrimaryKeyConstraint('topic_id')
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('forum_topic')
+    op.create_table('krock_forum_topic',
+    sa.Column('topic_id', sa.Integer(), autoincrement=False, nullable=False),
+    sa.Column('author_id', sa.Integer(), nullable=False),
+    sa.Column('ttype', sa.Integer(), nullable=False),
+    sa.Column('title', sa.String(length=200), nullable=False),
+    sa.Column('name', sa.String(length=30), nullable=True),
+    sa.Column('link', sa.String(length=50), nullable=True),
+    sa.ForeignKeyConstraint(['author_id'], ['user.id'], ),
+    sa.PrimaryKeyConstraint('topic_id')
+    )
+    # ### end Alembic commands ###