Update scoring algorithm to take licenses and screenshots into account

Fixes #97

app/__init__.py

@@ -37,5 +37,9 @@ csrf = CsrfProtect(app)
 mail = Mail(app)
 pages = FlatPages(app)
 
+if not app.debug:
+	from .maillogger import register_mail_error_handler
+	register_mail_error_handler(app, mail)
+
 from . import models, tasks
 from .views import *

app/flatpages/help/ranks_permissions.md

@@ -34,6 +34,8 @@ title: Ranks and Permissions
 			<th>N</th>
 			<th>Y</th>
 			<th>N</th>
+			<th>Y</th>
+			<th>N</th>
 		</tr>
 	</thead>
 	<tbody>
@@ -58,7 +60,7 @@ title: Ranks and Permissions
 			<th></th>
 			<th></th> <!-- member -->
 			<th></th>
-			<th>✓</th> <!-- trusted member -->
+			<th></th> <!-- trusted member -->
 			<th></th>
 			<th>✓</th> <!-- editor -->
 			<th>✓</th>
@@ -101,7 +103,7 @@ title: Ranks and Permissions
 			<td>Approve Screenshot</td>
 			<th></th> <!-- new -->
 			<th></th>
-			<th>✓</th> <!-- member -->
+			<th></th> <!-- member -->
 			<th></th>
 			<th>✓</th> <!-- trusted member -->
 			<th></th>
@@ -187,6 +189,21 @@ title: Ranks and Permissions
 			<th>✓</th> <!-- admin -->
 			<th>✓</th>
 		</tr>
+		<tr>
+			<td>See Private Thread</td>
+			<th>✓</th> <!-- new -->
+			<th></th>
+			<th>✓</th> <!-- member -->
+			<th></th>
+			<th>✓</th> <!-- trusted member -->
+			<th></th>
+			<th>✓</th> <!-- editor -->
+			<th>✓</th>
+			<th>✓</th> <!-- moderator -->
+			<th>✓</th>
+			<th>✓</th> <!-- admin -->
+			<th>✓</th>
+		</tr>
 		<tr>
 			<td>Set Email</td>
 			<th>✓</th> <!-- new -->

app/flatpages/help/wtfpl.md

@@ -0,0 +1,42 @@
+title: WTFPL is a terrible license
+no_h1: true
+
+<div id="warning" class="box box_grey alert alert-warning">
+	<span class="icon_message"></span>
+
+	Please reconsider the choice of WTFPL as a license.
+
+	<script src="/static/jquery.min.js"></script>
+	<script>
+		// @author rubenwardy
+		// @license magnet:?xt=urn:btih:1f739d935676111cfff4b4693e3816e664797050&dn=gpl-3.0.txt GPL-v3-or-Later
+
+		var params = new URLSearchParams(location.search);
+		var r      = params.get("r");
+		if (r)
+			document.write("<a class='alert_right button' href='" + r + "'>Okay</a>");
+		else
+			$("#warning").hide();
+	</script>
+</div>
+
+# WTFPL is a terrible license
+
+The use of WTFPL as a license is discouraged for multiple reasons.
+
+* **No Warranty disclaimer:** This could open you up to being sued.<sup>[1]</sup>
+* **Swearing:** This prevents settings like schools from using your content.
+* **Not OSI Approved:** Same as public domain?
+
+The Open Source Initiative chose not to approve the license as an open-source
+license, saying:<sup>[3]</sup>
+
+> It's no different from dedication to the public domain.
+> Author has submitted license approval request – author is free to make public domain dedication.
+> Although he agrees with the recommendation, Mr. Michlmayr notes that public domain doesn't exist in Europe. Recommend: Reject.
+
+## Sources
+
+1. [WTFPL is harmful to software developers](https://cubicspot.blogspot.com/2017/04/wtfpl-is-harmful-to-software-developers.html)
+2. [FSF](https://www.gnu.org/licenses/license-list.en.html)
+3. [OSI](https://opensource.org/minutes20090304)

app/flatpages/policy_and_guidance.md

@@ -4,6 +4,20 @@ title: Package Inclusion Policy and Guidance
 	<b>Note:</b> This is a draft
 </div>
 
+## 0. Overview
+
+ContentDB is for the community, and as such listings should be useful to the
+community. To help with this, there are a few rules to improve the quality of
+the listings and to combat abuse.
+
+* No inappropriate content.
+* Content must be playable/useful, but not necessarily finished.
+* Don't use the name of another mod unless your mod is a fork or reimplementation.
+* Licenses must allow derivatives, redistribution, and must not discriminate.
+* Don't put promotions are advertisements in package listings, except for
+  donation and personal website links which are permitted in the long description.
+
+
 ## 1. General
 
 It is not permitted to submit abusive, obscene, vulgar, slanderous, hateful,
@@ -22,14 +36,17 @@ The submission of malware is strictly prohibited. This includes software which
 does not do as it advertises, for example if it posts telemetry without stating
 clearly that it does in the package meta.
 
-ContentDB should only currently contain playable content, ie: stuff that would
+ContentDB should only currently contain playable content - content which is
-be in Mod Releases and Game Releases. Please don't upload any Work In Progress (WIP)
+sufficiently complete to be useful to end users. It's fine to add stuff which
-things. This will probably change in future if/when an "early access" feature is
+is still a work in progress (WIP) as long as it adds sufficient value -
-added.
+Mineclone 2 is a good example of a WIP package which may break between releases
+but still has value. Note that this doesn't mean that you should add a thing
+you started working on yesterday, it's worth adding all the basic stuff to
+make your package useful.
 
-Adding non-player facing mods, such as libraries and server tools, is perfectly fine.
+Adding non-player facing mods, such as libraries and server tools, is perfectly fine
-ContentDB isn't just for player-facing things, and adding libraries allows them to be
+and encouraged. ContentDB isn't just for player-facing things, and adding
-installed when a mod depends on it.
+libraries allows them to be installed when a mod depends on it.
 
 
 ## 3. Technical Names
@@ -48,6 +65,8 @@ package and give the name to the correct one.
 If you submit a package where you don't have the right to the name you will be asked
 to change the name of the package, or your package won't be accepted.
 
+We reserve the right to issue exceptions for this where we feel necessary.
+
 ### 3.2 Mod Forks and Reimplementations
 
 An exception to the above is that mods are allowed to have the same name as a
@@ -66,24 +85,31 @@ Please ensure that you correctly credit any resources (code, assets, or otherwis
 that you have used in your package.
 
 **The use of licenses which do not allow derivatives or redistribution is not
-permitted. This includes CC-ND (No-Derivatives) and lots of closed source licenses.**
+permitted. This includes CC-ND (No-Derivatives) and lots of closed source licenses.
+The use of licenses which discriminate between groups of people or forbid the use
+of the content on servers or singleplayer is also not permitted.**
 
 However, closed sourced licenses are allowed if they allow the above.
 
-If the license you use is not on the list then please choose the correct "Other"
+If the license you use is not on the list then please select "Other", and we'll
-option.
+get around to adding it.
 
 Please note that the definitions of "free" and "non-free" is the same as that
 of the [Free Software Foundation](https://www.gnu.org/philosophy/free-sw.en.html).
 
 ### 4.2 Recommended Licenses
 
+It is highly recommended that you use a free and open source software license.
+FOSS licenses result in a sharing community and will increase the number of potential users your package has.
+Using a closed source license will result in your package being massively penalised in the search results and package lists.
+
 It is recommended that you use a proper license for code with a warranty
 disclaimer, such as the (L)GPL or MIT. You should also use a proper media license
 for media, such as a Creative Commons license.
 
-The use of WTFPL is discouraged as it doesn't contain a valid warranty disclaimer,
+The use of WTFPL is discouraged as it doesn't contain a [valid warranty disclaimer](https://cubicspot.blogspot.com/2017/04/wtfpl-is-harmful-to-software-developers.html),
-and also includes swearing which dissuades teachers from using your content.
+and also includes swearing which prevents settings like schools from using your content.
+[Read more](/help/wtfpl/).
 
 Public domain is not a valid license in many countries, please use CC0 or MIT instead.
 

app/maillogger.py

@@ -0,0 +1,109 @@
+import logging
+from enum import Enum
+from app.tasks.emails import sendEmailRaw
+
+def _has_newline(line):
+	"""Used by has_bad_header to check for \\r or \\n"""
+	if line and ("\r" in line or "\n" in line):
+		return True
+	return False
+
+def _is_bad_subject(subject):
+	"""Copied from: flask_mail.py class Message def has_bad_headers"""
+	if _has_newline(subject):
+		for linenum, line in enumerate(subject.split("\r\n")):
+			if not line:
+				return True
+			if linenum > 0 and line[0] not in "\t ":
+				return True
+			if _has_newline(line):
+				return True
+			if len(line.strip()) == 0:
+				return True
+	return False
+
+
+class FlaskMailSubjectFormatter(logging.Formatter):
+	def format(self, record):
+		record.message = record.getMessage()
+		if self.usesTime():
+			record.asctime = self.formatTime(record, self.datefmt)
+		s = self.formatMessage(record)
+		return s
+
+class FlaskMailTextFormatter(logging.Formatter):
+	pass
+
+# TODO: hier nog niet tevreden over (vooral logger.error(..., exc_info, stack_info))
+class FlaskMailHTMLFormatter(logging.Formatter):
+	pre_template = "<h1>%s</h1><pre>%s</pre>"
+	def formatException(self, exc_info):
+		formatted_exception = logging.Handler.formatException(self, exc_info)
+		return FlaskMailHTMLFormatter.pre_template % ("Exception information", formatted_exception)
+	def formatStack(self, stack_info):
+		return FlaskMailHTMLFormatter.pre_template % ("<h1>Stack information</h1><pre>%s</pre>", stack_info)
+
+
+# see: https://github.com/python/cpython/blob/3.6/Lib/logging/__init__.py (class Handler)
+
+class FlaskMailHandler(logging.Handler):
+	def __init__(self, mailer, subject_template, level=logging.NOTSET):
+		logging.Handler.__init__(self, level)
+		self.mailer = mailer
+		self.send_to = mailer.app.config["MAIL_UTILS_ERROR_SEND_TO"]
+		self.subject_template = subject_template
+		self.html_formatter = None
+
+	def setFormatter(self, text_fmt, html_fmt=None):
+		"""
+		Set the formatters for this handler. Provide at least one formatter.
+		When no text_fmt is provided, no text-part is created for the email body.
+		"""
+		assert (text_fmt, html_fmt) != (None, None), "At least one formatter should be provided"
+		if type(text_fmt)==str:
+			text_fmt = FlaskMailTextFormatter(text_fmt)
+		self.formatter = text_fmt
+		if type(html_fmt)==str:
+			html_fmt = FlaskMailHTMLFormatter(html_fmt)
+		self.html_formatter = html_fmt
+
+	def getSubject(self, record):
+		fmt = FlaskMailSubjectFormatter(self.subject_template)
+		subject = fmt.format(record)
+		#Since templates can cause header problems, and we rather have a incomplete email then an error, we fix this
+		if _is_bad_subject(subject):
+			subject="FlaskMailHandler log-entry from %s [original subject is replaced, because it would result in a bad header]" % self.mailer.app.name
+		return subject
+
+	def emit(self, record):
+		text = self.format(record)				if self.formatter	  else None
+		html = self.html_formatter.format(record) if self.html_formatter else None
+		sendEmailRaw.delay(self.send_to, self.getSubject(record), text, html)
+
+
+def register_mail_error_handler(app, mailer):
+	subject_template = "ContentDB crashed (%(module)s > %(funcName)s)"
+	text_template = """
+Message type: %(levelname)s
+Location:	 %(pathname)s:%(lineno)d
+Module:	   %(module)s
+Function:	 %(funcName)s
+Time:		 %(asctime)s
+Message:
+%(message)s"""
+	html_template = """
+<style>th { text-align: right}</style><table>
+<tr><th>Message type:</th><td>%(levelname)s</td></tr>
+<tr>	<th>Location:</th><td>%(pathname)s:%(lineno)d</td></tr>
+<tr>	  <th>Module:</th><td>%(module)s</td></tr>
+<tr>	<th>Function:</th><td>%(funcName)s</td></tr>
+<tr>		<th>Time:</th><td>%(asctime)s</td></tr>
+</table>
+<h2>Message</h2>
+<pre>%(message)s</pre>"""
+
+	import logging
+	mail_handler = FlaskMailHandler(mailer, subject_template)
+	mail_handler.setLevel(logging.ERROR)
+	mail_handler.setFormatter(text_template, html_template)
+	app.logger.addHandler(mail_handler)

app/models.py

@@ -77,6 +77,7 @@ class Permission(enum.Enum):
 	CHANGE_EMAIL       = "CHANGE_EMAIL"
 	EDIT_EDITREQUEST   = "EDIT_EDITREQUEST"
 	SEE_THREAD         = "SEE_THREAD"
+	CREATE_THREAD      = "CREATE_THREAD"
 
 	# Only return true if the permission is valid for *all* contexts
 	# See Package.checkPerm for package-specific contexts
@@ -85,9 +86,10 @@ class Permission(enum.Enum):
 			return False
 
 		if self == Permission.APPROVE_NEW or \
-				self == Permission.APPROVE_CHANGES or \
+				self == Permission.APPROVE_CHANGES    or \
-				self == Permission.APPROVE_RELEASE or \
+				self == Permission.APPROVE_RELEASE    or \
-				self == Permission.APPROVE_SCREENSHOT:
+				self == Permission.APPROVE_SCREENSHOT or \
+				self == Permission.SEE_THREAD:
 			return user.rank.atLeast(UserRank.EDITOR)
 		else:
 			raise Exception("Non-global permission checked globally. Use Package.checkPerm or User.checkPerm instead.")
@@ -209,6 +211,13 @@ class PackageType(enum.Enum):
 	def __str__(self):
 		return self.name
 
+	@classmethod
+	def get(cls, name):
+		try:
+			return PackageType[name.upper()]
+		except KeyError:
+			return None
+
 	@classmethod
 	def choices(cls):
 		return [(choice, choice.value) for choice in cls]
@@ -339,6 +348,8 @@ class Package(db.Model):
 	approved     = db.Column(db.Boolean, nullable=False, default=False)
 	soft_deleted = db.Column(db.Boolean, nullable=False, default=False)
 
+	score        = db.Column(db.Float, nullable=False, default=0)
+
 	review_thread_id = db.Column(db.Integer, db.ForeignKey("thread.id"), nullable=True, default=None)
 	review_thread    = db.relationship("Thread", foreign_keys=[review_thread_id])
 
@@ -385,7 +396,8 @@ class Package(db.Model):
 			"shortDesc": self.shortDesc,
 			"type": self.type.toName(),
 			"release": self.getDownloadRelease().id if self.getDownloadRelease() is not None else None,
-			"thumbnail": (base_url + tnurl) if tnurl is not None else None
+			"thumbnail": (base_url + tnurl) if tnurl is not None else None,
+			"score": round(self.score * 10) / 10
 		}
 
 	def getAsDictionary(self, base_url):
@@ -412,7 +424,9 @@ class Package(db.Model):
 			"screenshots": [base_url + ss.url for ss in self.screenshots],
 
 			"url": base_url + self.getDownloadURL(),
-			"release": self.getDownloadRelease().id if self.getDownloadRelease() is not None else None
+			"release": self.getDownloadRelease().id if self.getDownloadRelease() is not None else None,
+
+			"score": round(self.score * 10) / 10
 		}
 
 	def getThumbnailURL(self):
@@ -474,7 +488,7 @@ class Package(db.Model):
 		isOwner = user == self.author
 
 		# Members can edit their own packages, and editors can edit any packages
-		if perm == Permission.MAKE_RELEASE or perm == Permission.ADD_SCREENSHOTS:
+		if perm == Permission.MAKE_RELEASE or perm == Permission.ADD_SCREENSHOTS or perm == Permission.CREATE_THREAD:
 			return isOwner or user.rank.atLeast(UserRank.EDITOR)
 
 		if perm == Permission.EDIT_PACKAGE or perm == Permission.APPROVE_CHANGES:
@@ -483,12 +497,11 @@ class Package(db.Model):
 			else:
 				return user.rank.atLeast(UserRank.EDITOR)
 
-		# Editors can change authors
+		# Editors can change authors and approve new packages
-		elif perm == Permission.CHANGE_AUTHOR:
+		elif perm == Permission.APPROVE_NEW or perm == Permission.CHANGE_AUTHOR:
 			return user.rank.atLeast(UserRank.EDITOR)
 
-		elif perm == Permission.APPROVE_NEW or perm == Permission.APPROVE_RELEASE \
+		elif perm == Permission.APPROVE_RELEASE or perm == Permission.APPROVE_SCREENSHOT:
-				or perm == Permission.APPROVE_SCREENSHOT:
 			return user.rank.atLeast(UserRank.TRUSTED_MEMBER if isOwner else UserRank.EDITOR)
 
 		# Moderators can delete packages
@@ -498,6 +511,25 @@ class Package(db.Model):
 		else:
 			raise Exception("Permission {} is not related to packages".format(perm.name))
 
+	def recalcScore(self):
+		import datetime
+
+		self.score = 10
+
+		if self.forums is not None:
+			topic = ForumTopic.query.get(self.forums)
+			if topic:
+				days   = (datetime.datetime.now() - topic.created_at).days
+				months = days / 30
+				years  = days / 365
+				self.score = topic.views / max(years, 0.0416) + 80*min(max(months, 0.5), 6)
+
+		if self.getMainScreenshotURL() is None:
+			self.score *= 0.8
+
+		if not self.license.is_foss or not self.media_license.is_foss:
+			self.score *= 0.1
+
 class MetaPackage(db.Model):
 	id           = db.Column(db.Integer, primary_key=True)
 	name         = db.Column(db.String(100), unique=True, nullable=False)
@@ -571,6 +603,7 @@ class PackageRelease(db.Model):
 	url          = db.Column(db.String(200), nullable=False)
 	approved     = db.Column(db.Boolean, nullable=False, default=False)
 	task_id      = db.Column(db.String(37), nullable=True)
+	commit_hash  = db.Column(db.String(41), nullable=True, default=None)
 
 
 	def getEditURL(self):
@@ -579,9 +612,24 @@ class PackageRelease(db.Model):
 				name=self.package.name,
 				id=self.id)
 
+	def getDownloadURL(self):
+		return url_for("download_release_page",
+				author=self.package.author.username,
+				name=self.package.name,
+				id=self.id)
+
+
 	def __init__(self):
 		self.releaseDate = datetime.now()
 
+
+class PackageReview(db.Model):
+	id         = db.Column(db.Integer, primary_key=True)
+	package_id = db.Column(db.Integer, db.ForeignKey("package.id"))
+	thread_id  = db.Column(db.Integer, db.ForeignKey("thread.id"), nullable=False)
+	recommend  = db.Column(db.Boolean, nullable=False, default=True)
+
+
 class PackageScreenshot(db.Model):
 	id         = db.Column(db.Integer, primary_key=True)
 	package_id = db.Column(db.Integer, db.ForeignKey("package.id"))
@@ -709,6 +757,15 @@ class Thread(db.Model):
 	watchers   = db.relationship("User", secondary=watchers, lazy="subquery", \
 						backref=db.backref("watching", lazy=True))
 
+
+	def getSubscribeURL(self):
+		return url_for("thread_subscribe_page",
+				id=self.id)
+
+	def getUnsubscribeURL(self):
+		return url_for("thread_unsubscribe_page",
+				id=self.id)
+
 	def checkPerm(self, user, perm):
 		if not user.is_authenticated:
 			return not self.private
@@ -734,38 +791,52 @@ class ThreadReply(db.Model):
 	created_at = db.Column(db.DateTime, nullable=False, default=datetime.utcnow)
 
 
-
-
-
-
 REPO_BLACKLIST = [".zip", "mediafire.com", "dropbox.com", "weebly.com", \
 		"minetest.net", "dropboxusercontent.com", "4shared.com", \
 		"digitalaudioconcepts.com", "hg.intevation.org", "www.wtfpl.net", \
 		"imageshack.com", "imgur.com"]
 
-class KrockForumTopic(db.Model):
+class ForumTopic(db.Model):
 	topic_id  = db.Column(db.Integer, primary_key=True, autoincrement=False)
 	author_id = db.Column(db.Integer, db.ForeignKey("user.id"), nullable=False)
 	author    = db.relationship("User")
 
-	ttype     = db.Column(db.Integer, nullable=False)
+	wip       = db.Column(db.Boolean, server_default="0")
+
+	type      = db.Column(db.Enum(PackageType), nullable=False)
 	title     = db.Column(db.String(200), nullable=False)
 	name      = db.Column(db.String(30), nullable=True)
 	link      = db.Column(db.String(200), nullable=True)
 
-	def getType(self):
+	posts     = db.Column(db.Integer, nullable=False)
-		if self.ttype == 1 or self.ttype == 2:
+	views     = db.Column(db.Integer, nullable=False)
-			return PackageType.MOD
+
-		elif self.ttype == 6:
+	created_at = db.Column(db.DateTime, nullable=False, default=datetime.utcnow)
-			return PackageType.GAME
 
 	def getRepoURL(self):
+		if self.link is None:
+			return None
+
 		for item in REPO_BLACKLIST:
 			if item in self.link:
 				return None
 
 		return self.link.replace("repo.or.cz/w/", "repo.or.cz/")
 
+	def getAsDictionary(self):
+		return {
+			"author": self.author.username,
+			"name":   self.name,
+			"type":   self.type.toName(),
+			"title":  self.title,
+			"id":     self.topic_id,
+			"link":   self.link,
+			"posts":  self.posts,
+			"views":  self.views,
+			"is_wip": self.wip,
+			"created_at": self.created_at.isoformat(),
+		}
+
 
 # Setup Flask-User
 db_adapter = SQLAlchemyAdapter(db, User)        # Register the User model

app/scss/components.scss

@@ -99,6 +99,10 @@ a:hover {
 	font-size: 100%;
 }
 
+select > * {
+	color: #222;
+}
+
 input[type=text], input[type=password], textarea, select, .bulletselector {
 	text-align: left;
 }
@@ -451,3 +455,7 @@ table.fancyTable tfoot td {
 .table-topalign td {
 	vertical-align: top;
 }
+
+.wiptopic a {
+	color: #7ac;
+}

app/tasks/emails.py

@@ -22,7 +22,17 @@ from app.tasks import celery
 
 @celery.task()
 def sendVerifyEmail(newEmail, token):
-    msg = Message("Verify email address", recipients=[newEmail])
+	msg = Message("Verify email address", recipients=[newEmail])
-    msg.body = "This is a verification email!"
+	msg.body = "This is a verification email!"
-    msg.html = render_template("emails/verify.html", token=token)
+	msg.html = render_template("emails/verify.html", token=token)
-    mail.send(msg)
+	mail.send(msg)
+
+@celery.task()
+def sendEmailRaw(to, subject, text, html):
+	from flask_mail import Message
+	msg = Message(subject, recipients=to)
+	if text:
+		msg.body = text
+	if html:
+		msg.html = html
+	mail.send(msg)

app/tasks/forumtasks.py

@@ -15,12 +15,12 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 
-import flask, json
+import flask, json, re
 from flask.ext.sqlalchemy import SQLAlchemy
 from app import app
 from app.models import *
 from app.tasks import celery
-from .phpbbparser import getProfile
+from .phpbbparser import getProfile, getTopicsFromForum
 import urllib.request
 from urllib.parse import urlparse, quote_plus
 
@@ -51,71 +51,91 @@ def checkForumAccount(username, token=None):
 	if needsSaving:
 		db.session.commit()
 
-@celery.task()
+
-def importUsersFromModList():
+regex_tag    = re.compile(r"\[([a-z0-9_]+)\]")
+BANNED_NAMES = ["mod", "game", "old", "outdated", "wip", "api", "beta", "alpha", "git"]
+def getNameFromTaglist(taglist):
+	for tag in reversed(regex_tag.findall(taglist)):
+		if len(tag) < 30 and not tag in BANNED_NAMES and \
+				not re.match(r"^[a-z]?[0-9]+$", tag):
+			return tag
+
+	return None
+
+regex_title = re.compile(r"^((?:\[[^\]]+\] *)*)([^\[]+) *((?:\[[^\]]+\] *)*)[^\[]*$")
+def parseTitle(title):
+	m = regex_title.match(title)
+	if m is None:
+		print("Invalid title format: " + title)
+		return title, getNameFromTaglist(title)
+	else:
+		return m.group(2).strip(), getNameFromTaglist(m.group(3))
+
+def getLinksFromModSearch():
+	links = {}
+
 	contents = urllib.request.urlopen("http://krock-works.16mb.com/MTstuff/modList.php").read().decode("utf-8")
-	list = json.loads(contents)
+	for x in json.loads(contents):
-	found = {}
+		link = x.get("link")
-	imported = []
+		if link is not None:
+			links[int(x["topicId"])] = link
 
-	for user in User.query.all():
+	return links
-		found[user.username] = True
-		if user.forums_username is not None:
-			found[user.forums_username] = True
-
-	for x in list:
-		author = x.get("author")
-		if author is not None and not author in found:
-			user = User(author)
-			user.forums_username = author
-			imported.append(author)
-			found[author] = True
-			db.session.add(user)
-
-	db.session.commit()
-	for author in found:
-		checkForumAccount.delay(author, None)
-
-
-BANNED_NAMES = ["mod", "game", "old", "outdated", "wip", "api"]
-ALLOWED_TYPES = [1, 2, 6]
 
 @celery.task()
-def importKrocksModList():
+def importTopicList():
-	contents = urllib.request.urlopen("http://krock-works.16mb.com/MTstuff/modList.php").read().decode("utf-8")
+	links_by_id = getLinksFromModSearch()
-	list = json.loads(contents)
+
+	info_by_id = {}
+	getTopicsFromForum(11, out=info_by_id, extra={ 'type': PackageType.MOD,  'wip': False })
+	getTopicsFromForum(9,  out=info_by_id, extra={ 'type': PackageType.MOD,  'wip': True  })
+	getTopicsFromForum(15, out=info_by_id, extra={ 'type': PackageType.GAME, 'wip': False })
+	getTopicsFromForum(50, out=info_by_id, extra={ 'type': PackageType.GAME, 'wip': True  })
+
+	# Caches
 	username_to_user = {}
+	topics_by_id     = {}
+	for topic in ForumTopic.query.all():
+		topics_by_id[topic.topic_id] = topic
 
-	KrockForumTopic.query.delete()
+	# Create or update
+	for info in info_by_id.values():
+		id = int(info["id"])
 
-	for x in list:
+		# Get author
-		type = int(x["type"])
+		username = info["author"]
-		if not type in ALLOWED_TYPES:
-			continue
-
-		username = x["author"]
 		user = username_to_user.get(username)
 		if user is None:
 			user = User.query.filter_by(forums_username=username).first()
-			assert(user is not None)
+			if user is None:
+				print(username + " not found!")
+				user = User(username)
+				user.forums_username = username
+				db.session.add(user)
 			username_to_user[username] = user
 
-		import re
+		# Get / add row
-		tags = re.findall("\[([a-z0-9_]+)\]", x["title"])
+		topic = topics_by_id.get(id)
-		name = None
+		if topic is None:
-		for tag in reversed(tags):
+			topic = ForumTopic()
-			if len(tag) < 30 and not tag in BANNED_NAMES and \
+			db.session.add(topic)
-					not re.match("^([a-z][0-9]+)$", tag):
-				name = tag
-				break
 
-		topic = KrockForumTopic()
+		# Parse title
-		topic.topic_id  = x["topicId"]
+		title, name = parseTitle(info["title"])
-		topic.author_id = user.id
+
-		topic.ttype     = type
+		# Get link
-		topic.title     = x["title"]
+		link = links_by_id.get(id)
-		topic.name      = name
+
-		topic.link      = x.get("link")
+		# Fill row
-		db.session.add(topic)
+		topic.topic_id   = id
+		topic.author     = user
+		topic.type       = info["type"]
+		topic.title      = title
+		topic.name       = name
+		topic.link       = link
+		topic.wip        = info["wip"]
+		topic.posts      = info["posts"]
+		topic.views      = info["views"]
+		topic.created_at = info["date"]
 
 	db.session.commit()

app/tasks/importtasks.py

@@ -344,9 +344,10 @@ def makeVCSReleaseFromGithub(id, branch, release, url):
 	if len(commits) == 0 or not "sha" in commits[0]:
 		raise TaskError("No commits found")
 
-	release.url = urlmaker.getCommitDownload(commits[0]["sha"])
+	release.url          = urlmaker.getCommitDownload(commits[0]["sha"])
+	release.task_id     = None
+	release.commit_hash = commits[0]["sha"]
 	print(release.url)
-	release.task_id = None
 	db.session.commit()
 
 	return release.url
@@ -372,11 +373,12 @@ def makeVCSRelease(id, branch):
 			filename = randomString(10) + ".zip"
 			destPath = os.path.join("app/public/uploads", filename)
 			with open(destPath, "wb") as fp:
-				repo.archive(fp)
+				repo.archive(fp, format="zip")
 
-			release.url = "/uploads/" + filename
+			release.url         = "/uploads/" + filename
+			release.task_id     = None
+			release.commit_hash = repo.head.object.hexsha
 			print(release.url)
-			release.task_id = None
 			db.session.commit()
 
 			return release.url

app/tasks/phpbbparser.py

@@ -5,6 +5,7 @@
 import urllib, socket
 from bs4 import *
 from urllib.parse import urljoin
+from datetime import datetime
 import urllib.request
 import os.path
 import time, re
@@ -77,3 +78,72 @@ def getProfile(url, username):
 		__extract_properties(profile, soup)
 
 		return profile
+
+
+regex_id = re.compile(r"^.*t=([0-9]+).*$")
+
+def parseForumListPage(id, page, out, extra=None):
+	num_per_page = 30
+	start = page*num_per_page+1
+	print(" - Fetching page {} (topics {}-{})".format(page, start, start+num_per_page))
+
+	url = "https://forum.minetest.net/viewforum.php?f=" + str(id) + "&start=" + str(start)
+	r = urllib.request.urlopen(url).read().decode("utf-8")
+	soup = BeautifulSoup(r, "html.parser")
+
+	for row in soup.find_all("li", class_="row"):
+		classes = row.get("class")
+		if "sticky" in classes or "announce" in classes or "global-announce" in classes:
+			continue
+
+		topic = row.find("dl")
+
+		# Link info
+		link   = topic.find(class_="topictitle")
+		id	   = regex_id.match(link.get("href")).group(1)
+		title  = link.find(text=True)
+
+		# Date
+		left   = topic.find("dt")
+		date   = left.get_text().split("»")[1].strip()
+		date   = datetime.strptime(date, "%a %b %d, %Y %H:%M")
+		author = left.find_all("a")[-1].get_text().strip()
+
+		# Get counts
+		posts  = topic.find(class_="posts").find(text=True)
+		views  = topic.find(class_="views").find(text=True)
+
+		if id in out:
+			print("   - got {} again, title: {}".format(id, title))
+			assert(title == out[id]['title'])
+			return False
+
+		row = {
+			"id"    : id,
+			"title" : title,
+			"author": author,
+			"posts" : posts,
+			"views" : views,
+			"date"  : date
+		}
+
+		if extra is not None:
+			for key, value in extra.items():
+				row[key] = value
+
+		out[id] = row
+
+	return True
+
+def getTopicsFromForum(id, out={}, extra=None):
+	print("Fetching all topics from forum {}".format(id))
+	page = 0
+	while parseForumListPage(id, page, out, extra):
+		page = page + 1
+
+	return out
+
+def dumpTitlesToFile(topics, path):
+	with open(path, "w") as out_file:
+		for topic in topics.values():
+			out_file.write(topic["title"] + "\n")

app/templates/admin/licenses/edit.html

@@ -0,0 +1,25 @@
+{% extends "base.html" %}
+
+{% block title %}
+	{% if license %}
+		Edit {{ license.name }}
+	{% else %}
+		New license
+	{% endif %}
+{% endblock %}
+
+{% block content %}
+	<p>
+		<a href="{{ url_for('license_list_page') }}">Back to list</a> |
+		<a href="{{ url_for('createedit_license_page') }}">New License</a>
+	</p>
+
+	{% from "macros/forms.html" import render_field, render_submit_field %}
+	<form method="POST" action="" enctype="multipart/form-data">
+		{{ form.hidden_tag() }}
+
+		{{ render_field(form.name) }}
+		{{ render_field(form.is_foss) }}
+		{{ render_submit_field(form.submit) }}
+	</form>
+{% endblock %}

app/templates/admin/licenses/list.html

@@ -0,0 +1,16 @@
+{% extends "base.html" %}
+
+{% block title %}
+Licenses
+{% endblock %}
+
+{% block content %}
+	<p>
+		<a href="{{ url_for('createedit_license_page') }}">New Tag</a>
+	</p>
+	<ul>
+		{% for l in licenses %}
+			<li><a href="{{ url_for('createedit_license_page', name=l.name) }}">{{ l.name }}</a> [{{ l.is_foss and "Free" or "Non-free"}}]</li>
+		{% endfor %}
+	</ul>
+{% endblock %}

app/templates/admin/list.html

@@ -1,13 +1,15 @@
 {% extends "base.html" %}
 
 {% block title %}
-	Admin Tools
+Admin
 {% endblock %}
 
 {% block content %}
 	<ul>
+		<li><a href="db/">Database</a></li>
 		<li><a href="{{ url_for('user_list_page') }}">User list</a></li>
 		<li><a href="{{ url_for('tag_list_page') }}">Tag Editor</a></li>
+		<li><a href="{{ url_for('license_list_page') }}">License Editor</a></li>
 		<li><a href="{{ url_for('switch_user_page') }}">Sign in as another user</a></li>
 	</ul>
 
@@ -17,11 +19,12 @@
 		<form method="post" action="" class="box-body">
 			<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
 			<select name="action">
-				<option value="importusers">Create users from mod list</option>
+				<option value="importmodlist" selected>Import forum topics</option>
-				<option value="importmodlist">Import Krock's mod list</option>
+				<option value="recalcscores">Recalculate package scores</option>
-				<option value="importscreenshots" selected>Import screenshots from VCS</option>
+				<!-- <option value="importscreenshots">Import screenshots from VCS</option> -->
-				<option value="importdepends">Import dependencies from downloads</option>
+				<!-- <option value="importdepends">Import dependencies from downloads</option> -->
-				<option value="modprovides">Set provides to mod name</option>
+				<!-- <option value="modprovides">Set provides to mod name</option> -->
+				<!-- <option value="vcsrelease">Create VCS releases</option> -->
 			</select>
 			<input type="submit" value="Perform" />
 		</form>

app/templates/base.html

@@ -65,6 +65,7 @@
 							{% endif %}
 							{% if current_user.rank == current_user.rank.MODERATOR %}
 								<li><a href="{{ url_for('tag_list_page') }}">Tag Editor</a></li>
+								<li><a href="{{ url_for('license_list_page') }}">License Editor</a></li>
 							{% endif %}
 							<li><a href="{{ url_for('user.logout') }}">Sign out</a></li>
 						</ul>

app/templates/flatpage.html

@@ -5,7 +5,7 @@
 {% endblock %}
 
 {% block content %}
-	<h1>{{ page['title'] }}</h1>
+	{% if not page["no_h1"] %}<h1>{{ page['title'] }}</h1>{% endif %}
 
 	{{ page.html | safe }}
 {% endblock %}

app/templates/index.html

@@ -12,7 +12,7 @@ Welcome
 		<p>
 			Minetest's official content repository.
 			Browse {{ count }} packages,
-			all available under a free and open source
+			majority of which available under a free and open source
 			license.
 		</p>
 
@@ -25,6 +25,14 @@ Welcome
 
 <main>
 	{% from "macros/packagegridtile.html" import render_pkggrid %}
-	{{ render_pkggrid(packages) }}
+
+	<h2>Popular</h2>
+	{{ render_pkggrid(popular) }}
+
+	<a href="{{ url_for('packages_page') }}" class="button">Show More</a>
+
+	<h2 style="margin-top:2em;">Newly Added</h2>
+	{{ render_pkggrid(new) }}
+
 </main>
 {% endblock %}

app/templates/macros/threads.html

@@ -10,7 +10,7 @@
 					<div class="clearboth"></div>
 				</div>
 				<div class="msg">
-					{{ r.comment }}
+					{{ r.comment | markdown }}
 				</div>
 			</li>
 		{% endfor %}
@@ -19,7 +19,7 @@
 	{% if current_user.is_authenticated %}
 		<form method="post" action="{{ url_for('thread_page', id=thread.id)}}" class="comment_form">
 			<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
-			<textarea required maxlength=500 name="comment"></textarea><br />
+			<textarea required maxlength=500 name="comment" placeholder="Markdown supported"></textarea><br />
 			<input type="submit" value="Comment" />
 		</form>
 	{% endif %}
@@ -28,7 +28,9 @@
 {% macro render_threadlist(threads) -%}
 	<ul>
 		{% for t in threads %}
-			<li><a href="{{ url_for('thread_page', id=t.id) }}">{{ t.title }}</a> by {{ t.author.display_name }}</li>
+			<li>{% if t.private %}&#x1f512; {% endif %}<a href="{{ url_for('thread_page', id=t.id) }}">{{ t.title }}</a> by {{ t.author.display_name }}</li>
+		{% else %}
+			<li><i>No threads found</i></li>
 		{% endfor %}
 	</ul>
 {% endmacro %}

app/templates/macros/topics.html

@@ -0,0 +1,52 @@
+{% macro render_topics_table(topics, show_author=True) -%}
+<table>
+	<tr>
+		<th>Id</th>
+		<th></th>
+		<th>Title</th>
+		{% if show_author %}<th>Author</th>{% endif %}
+		<th>Name</th>
+		<th>Link</th>
+		<th>Actions</th>
+	</tr>
+	{% for topic in topics %}
+		<tr{% if topic.wip %} class="wiptopic"{% endif %}>
+			<td>{{ topic.topic_id }}</td>
+			<td>
+				[{{ topic.type.value }}]
+			</td>
+			<td>
+				<a href="https://forum.minetest.net/viewtopic.php?t={{ topic.topic_id}}">{{ topic.title }}</a>
+				{% if topic.wip %}[WIP]{% endif %}
+			</td>
+			{% if show_author %}
+				<td><a href="{{ url_for('user_profile_page', username=topic.author.username) }}">{{ topic.author.display_name}}</a></td>
+			{% endif %}
+			<td>{{ topic.name or ""}}</td>
+			<td>{% if topic.link %}<a href="{{ topic.link }}">{{ topic.link | domain }}</a>{% endif %}</td>
+			<td>
+				<a href="{{ url_for('create_edit_package_page', author=topic.author.username, repo=topic.getRepoURL(), forums=topic.topic_id, title=topic.title, bname=topic.name) }}">Create</a>
+			</td>
+		</tr>
+	{% endfor %}
+</table>
+{% endmacro %}
+
+
+{% macro render_topics(topics, current_user, show_author=True) -%}
+<ul>
+	{% for topic in topics %}
+		<li{% if topic.wip %} class="wiptopic"{% endif %}>
+			<a href="https://forum.minetest.net/viewtopic.php?t={{ topic.topic_id}}">{{ topic.title }}</a>
+			{% if topic.wip %}[WIP]{% endif %}
+			{% if topic.name %}[{{ topic.name }}]{% endif %}
+			{% if show_author %}
+				by <a href="{{ url_for('user_profile_page', username=topic.author.username) }}">{{ topic.author.display_name }}</a>
+			{% endif %}
+			{% if topic.author == current_user or topic.author.checkPerm(current_user, "CHANGE_AUTHOR") %}
+				| <a href="{{ url_for('create_edit_package_page', author=topic.author.username, repo=topic.getRepoURL(), forums=topic.topic_id, title=topic.title, bname=topic.name) }}">Create</a>
+			{% endif %}
+		</li>
+	{% endfor %}
+</ul>
+{% endmacro %}

app/templates/macros/topictable.html

@@ -1,26 +0,0 @@
-{% macro render_topictable(topics, show_author=True) -%}
-<table>
-	<tr>
-		<th>Id</th>
-		<th>Title</th>
-		{% if show_author %}<th>Author</th>{% endif %}
-		<th>Name</th>
-		<th>Link</th>
-		<th>Actions</th>
-	</tr>
-	{% for topic in topics %}
-		<tr>
-			<td>{{ topic.topic_id }}</td>
-			<td>[{{ topic.getType().value }}] <a href="https://forum.minetest.net/viewtopic.php?t={{ topic.topic_id}}">{{ topic.title }}</a></td>
-			{% if show_author %}
-				<td><a href="{{ url_for('user_profile_page', username=topic.author.username) }}">{{ topic.author.display_name}}</a></td>
-			{% endif %}
-			<td>{{ topic.name or ""}}</td>
-			<td><a href="{{ topic.link }}">{{ topic.link | domain }}</a></td>
-			<td>
-				<a href="{{ url_for('create_edit_package_page', author=topic.author.username, repo=topic.getRepoURL(), forums=topic.topic_id, title=topic.title, bname=topic.name) }}">Create</a>
-			</td>
-		</tr>
-	{% endfor %}
-</table>
-{% endmacro %}

app/templates/packages/list.html

@@ -9,6 +9,7 @@
 		{% if type %}<input type="hidden" name="type" value="{{ type }}" />{% endif %}
 		<input type="text" name="q" value="{{ query or ''}}" />
 		<input type="submit" value="Search" />
+		<input type="submit" name="lucky" value="I'm feeling lucky" />
 
 		<p>
 			Found {{ packages_count }} packages.
@@ -37,4 +38,12 @@
 		<li>{{ page }} / {{ page_max }}</li>
 		{% if next_url %}<li><a href="{{ next_url }}">Next</a></li> {% endif %}
 	</ul>
+
+	{% if topics %}
+		<h2 style="margin-top:2em;">More content from the forums</h2>
+
+		{% from "macros/topics.html" import render_topics %}
+		{{ render_topics(topics, current_user) }}
+	{% endif %}
+
 {% endblock %}

app/templates/packages/release_edit.html

@@ -21,8 +21,12 @@
 			Url: <a href="{{ release.url }}">{{ release.url }}</a><br />
 		{% endif %}
 
+		{% if release.commit_hash %}
+			Commit Hash: {{ release.commit_hash }}<br />
+		{% endif %}
+
 		{% if release.task_id %}
-			Importing... <a href="{ url_for('check_task', id=release.task_id, r=release.getEditURL()) }}">view task</a><br />
+			Importing... <a href="{{ url_for('check_task', id=release.task_id, r=release.getEditURL()) }}">view task</a><br />
 			{% if package.checkPerm(current_user, "CHANGE_RELEASE_URL") %}
 				{{ render_field(form.task_id) }}
 			{% endif %}
@@ -38,23 +42,4 @@
 
 		{{ render_submit_field(form.submit) }}
 	</form>
-
-	{% if package.checkPerm(current_user, "APPROVE_RELEASE") %}
-		<div class="box box_grey">
-			<h2>Approval Checklist</h2>
-			<ul>
-				<li>Link leads to a valid download, ie: is a zip file which
-					has either init.lua or modpack.txt if a mod, mods/ if a game, or textures if a texture pack.
-					It's okay if they're inside an immediate folder, like so:
-
-<pre>
-example.zip/
-└── example
-    └── init.lua
-</pre>
-				</li>
-				<li>There's no obfuscated code.</li>
-			</ul>
-		</div>
-	{% endif %}
 {% endblock %}

app/templates/packages/screenshot_edit.html

@@ -5,7 +5,9 @@
 {% endblock %}
 
 {% block content %}
-	<img src="{{ screenshot.getThumbnailURL() }}" alt="{{ screenshot.title }}" />
+	<a href="{{ screenshot.url }}">
+		<img src="{{ screenshot.getThumbnailURL() }}" alt="{{ screenshot.title }}" />
+	</a>
 
 	{% from "macros/forms.html" import render_field, render_submit_field %}
 	<form method="POST" action="" enctype="multipart/form-data">

app/templates/packages/view.html

@@ -24,6 +24,12 @@
 			{% elif (package.type == package.type.GAME or package.type == package.type.TXP) and package.screenshots.count() == 0 %}
 				You need to add at least one screenshot.
 
+			{% elif topic_error_lvl == "error" %}
+				Please fix the below topic issue(s).
+
+			{% elif "Other" in package.license.name or "Other" in package.media_license.name %}
+				Please wait for the license to be added to CDB.
+
 			{% else %}
 				{% if package.screenshots.count() == 0 %}
 					<b>You should add at least one screenshot, but this isn't required.</b><br />
@@ -44,13 +50,23 @@
 			<div style="clear: both;"></div>
 		</div>
 
+		{% if topic_error %}
+			<div class="box box_grey alert alert-{{ topic_error_lvl }}">
+				<span class="icon_message"></span>
+				{{ topic_error | safe }}
+				<div style="clear: both;"></div>
+			</div>
+		{% endif %}
+
 		{% if package.author == current_user or package.checkPerm(current_user, "APPROVE_NEW") %}
 			{% if review_thread %}
-				<h2>&#x1f512; {{ review_thread.title }}</h2>
+				<h2>{% if review_thread.private %}&#x1f512;{% endif %} {{ review_thread.title }}</h2>
-				<p><i>
+				{% if review_thread.private %}
-					This thread is only visible to the package owner and users of
+					<p><i>
-					Editor rank or above.
+						This thread is only visible to the package owner and users of
-				</i></p>
+						Editor rank or above.
+					</i></p>
+				{% endif %}
 
 				{% from "macros/threads.html" import render_thread %}
 				{{ render_thread(review_thread, current_user) }}
@@ -158,6 +174,9 @@
 				{% if package.checkPerm(current_user, "MAKE_RELEASE") %}
 					<li><a href="{{ package.getCreateReleaseURL() }}">Create Release</a></li>
 				{% endif %}
+				{% if package.approved and package.checkPerm(current_user, "CREATE_THREAD") %}
+					<li><a href="{{ url_for('new_thread_page', pid=package.id) }}">Open Thread</a></li>
+				{% endif %}
 				{% if package.checkPerm(current_user, "DELETE_PACKAGE") %}
 					<li><a href="{{ package.getDeleteURL() }}">Delete</a></li>
 				{% endif %}
@@ -177,7 +196,8 @@
 				<li>
 					{% if not rel.approved %}<i>{% endif %}
 
-					<a href="{{ rel.url }}">{{ rel.title }}</a>,
+					<a href="{{ rel.getDownloadURL() }}">{{ rel.title }}</a>{% if rel.commit_hash %}
+					[{{ rel.commit_hash | truncate(5, end='') }}]{% endif %},
 					created {{ rel.releaseDate | datetime }}.
 					{% if rel.task_id %}
 						<a href="{{ url_for('check_task', id=rel.task_id, r=package.getDetailsURL()) }}">Importing...</a>
@@ -292,12 +312,24 @@
 		<ul>
 			{% for t in similar_topics %}
 				<li>
-					[{{ t.getType().value }}]
+					[{{ t.type.value }}]
 					<a href="https://forum.minetest.net/viewtopic.php?t={{ t.topic_id }}">
 						{{ t.title }} by {{ t.author.display_name }}
 					</a>
+					{% if t.wip %}[WIP]{% endif %}
 				</li>
 			{% endfor %}
 		</ul>
 	{% endif %}
+
+	{% if threads %}
+		<h3>Threads</h3>
+
+		{% if package.approved and package.checkPerm(current_user, "CREATE_THREAD") %}
+			<p><a href="{{ url_for('new_thread_page', pid=package.id) }}">Open Thread</a></p>
+		{% endif %}
+
+		{% from "macros/threads.html" import render_threadlist %}
+		{{ render_threadlist(threads) }}
+	{% endif %}
 {% endblock %}

app/templates/threads/view.html

@@ -6,11 +6,24 @@ Threads
 
 {% block content %}
 	<h1>{% if thread.private %}&#x1f512; {% endif %}{{ thread.title }}</h1>
+	{% if thread.package or current_user.is_authenticated %}
+		{% if thread.package %}
+			<p>Package: <a href="{{ thread.package.getDetailsURL() }}">{{ thread.package.title }}</a></p>
+		{% endif %}
 
-	{% if thread.package %}
+		{% if current_user.is_authenticated %}
-		<p>
+			{% if current_user in thread.watchers %}
-			Package: <a href="{{ thread.package.getDetailsURL() }}">{{ thread.package.title }}</a>
+				<form method="post" action="{{ thread.getUnsubscribeURL() }}">
-		</p>
+					<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
+					<input type="submit" value="Unsubscribe" />
+				</form>
+			{% else %}
+				<form method="post" action="{{ thread.getSubscribeURL() }}">
+					<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
+					<input type="submit" value="Subscribe" />
+				</form>
+			{% endif %}
+		{% endif %}
 	{% endif %}
 
 	{% if thread.private %}

app/templates/todo/list.html

@@ -65,6 +65,6 @@
 	<p>
 		There are
 		<a href="{{ url_for('todo_topics_page') }}">{{ topics_to_add }} packages</a>
-		to be added to cdb, based on forum topics picked up by Krock's mod search.
+		to be added to cdb, based on cdb's forum parser.
 	</p>
 {% endblock %}

app/templates/todo/topics.html

@@ -12,6 +12,6 @@ Topics to be Added
 		{{ topics | count }} remaining.
 	</p>
 
-	{% from "macros/topictable.html" import render_topictable %}
+	{% from "macros/topics.html" import render_topics_table %}
-	{{ render_topictable(topics) }}
+	{{ render_topics_table(topics) }}
 {% endblock %}

app/templates/users/user_profile_page.html

@@ -104,14 +104,11 @@
 
 		<div class="box-body">
 			<p>
-				List of your topics without a matching package.
+				List of your forum topics which do not have a matching package.
-				Powered by Krock's Mod Search.
 			</p>
 
-			
+			{% from "macros/topics.html" import render_topics_table %}
-
+			{{ render_topics_table(topics_to_add, show_author=False) }}
-			{% from "macros/topictable.html" import render_topictable %}
-			{{ render_topictable(topics_to_add, show_author=False) }}
 		</div>
 	</div>
 {% endif %}

app/utils.py

@@ -50,6 +50,25 @@ def doFileUpload(file, allowedExtensions, fileTypeName):
 	file.save(os.path.join("app/public/uploads", filename))
 	return "/uploads/" + filename
 
+def make_flask_user_password(plaintext_str):
+	# http://passlib.readthedocs.io/en/stable/modular_crypt_format.html
+	# http://passlib.readthedocs.io/en/stable/lib/passlib.hash.bcrypt.html#format-algorithm
+	# Flask_User stores passwords in the Modular Crypt Format.
+	# https://github.com/lingthio/Flask-User/blob/master/flask_user/user_manager__settings.py#L166
+	#   Note that Flask_User allows customizing password algorithms.
+	#   USER_PASSLIB_CRYPTCONTEXT_SCHEMES defaults to bcrypt but if
+	#   default changes or is customized, the code below needs adapting.
+	# Individual password values will look like:
+	#   $2b$12$.az4S999Ztvy/wa3UdQvMOpcki1Qn6VYPXmEFMIdWQyYs7ULnH.JW
+	#   $XX$RR$SSSSSSSSSSSSSSSSSSSSSSHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH
+	# $XX : Selects algorithm (2b is bcrypt).
+	# $RR : Selects bcrypt key expansion rounds (12 is 2**12 rounds).
+	# $SSS... : 22 chars of (random, per-password) salt
+	#  HHH... : 31 remaining chars of password hash (note no dollar sign)
+	import bcrypt
+	plaintext = plaintext_str.encode("UTF-8")
+	password = bcrypt.hashpw(plaintext, bcrypt.gensalt())
+	return password.decode("UTF-8")
 
 def _do_login_user(user, remember_me=False):
 	def _call_or_get(v):

app/views/__init__.py

@@ -46,13 +46,15 @@ def send_upload(path):
 @app.route("/")
 @menu.register_menu(app, ".", "Home")
 def home_page():
-	query = Package.query.filter_by(approved=True, soft_deleted=False)
+	query   = Package.query.filter_by(approved=True, soft_deleted=False)
-	count = query.count()
+	count   = query.count()
-	packages = query.order_by(db.desc(Package.created_at)).limit(15).all()
+	new     = query.order_by(db.desc(Package.created_at)).limit(15).all()
-	return render_template("index.html", packages=packages, count=count)
+	popular = query.order_by(db.desc(Package.score)).limit(6).all()
+	return render_template("index.html", new=new, popular=popular, count=count)
 
 from . import users, githublogin, packages, meta, threads, api
-from . import sass, tasks, admin, notifications, tagseditor, thumbnails
+from . import tasks, admin, notifications, tagseditor, licenseseditor
+from . import sass, thumbnails
 
 @menu.register_menu(app, ".help", "Help", order=19, endpoint_arguments_constructor=lambda: { 'path': 'help' })
 @app.route('/<path:path>/')

app/views/admin.py

@@ -20,22 +20,52 @@ from flask_user import *
 from flask.ext import menu
 from app import app
 from app.models import *
-from app.tasks.importtasks import importRepoScreenshot, importAllDependencies
+from celery import uuid
-from app.tasks.forumtasks  import importUsersFromModList, importKrocksModList
+from app.tasks.importtasks import importRepoScreenshot, importAllDependencies, makeVCSRelease
+from app.tasks.forumtasks  import importTopicList
 from flask_wtf import FlaskForm
 from wtforms import *
-from app.utils import loginUser, rank_required
+from app.utils import loginUser, rank_required, triggerNotif
+import datetime
+from flask_admin import Admin
+from flask_admin.contrib.sqla import ModelView
+
+class MyModelView(ModelView):
+	def is_accessible(self):
+		return current_user.is_authenticated and current_user.rank.atLeast(UserRank.ADMIN)
+
+	def inaccessible_callback(self, name, **kwargs):
+		# redirect to login page if user doesn't have access
+		if current_user.is_authenticated:
+			abort(403)
+		else:
+			return redirect(url_for('user.login', next=request.url))
+
+admin = Admin(app, name='ContentDB', template_mode='bootstrap3', url="/admin/db")
+admin.add_view(MyModelView(User, db.session))
+admin.add_view(MyModelView(Package, db.session))
+admin.add_view(MyModelView(Dependency, db.session))
+admin.add_view(MyModelView(EditRequest, db.session))
+admin.add_view(MyModelView(EditRequestChange, db.session))
+admin.add_view(MyModelView(ForumTopic, db.session))
+admin.add_view(MyModelView(License, db.session))
+admin.add_view(MyModelView(MetaPackage, db.session))
+admin.add_view(MyModelView(Notification, db.session))
+admin.add_view(MyModelView(PackageRelease, db.session))
+admin.add_view(MyModelView(PackageScreenshot, db.session))
+admin.add_view(MyModelView(Tag, db.session))
+admin.add_view(MyModelView(Thread, db.session))
+admin.add_view(MyModelView(ThreadReply, db.session))
+admin.add_view(MyModelView(UserEmailVerification, db.session))
+
 
 @app.route("/admin/", methods=["GET", "POST"])
 @rank_required(UserRank.ADMIN)
 def admin_page():
 	if request.method == "POST":
 		action = request.form["action"]
-		if action == "importusers":
+		if action == "importmodlist":
-			task = importUsersFromModList.delay()
+			task = importTopicList.delay()
-			return redirect(url_for("check_task", id=task.id, r=url_for("user_list_page")))
-		elif action == "importmodlist":
-			task = importKrocksModList.delay()
 			return redirect(url_for("check_task", id=task.id, r=url_for("todo_topics_page")))
 		elif action == "importscreenshots":
 			packages = Package.query \
@@ -67,6 +97,31 @@ def admin_page():
 
 			db.session.commit()
 			return redirect(url_for("admin_page"))
+		elif action == "recalcscores":
+			for p in Package.query.all():
+				p.recalcScore()
+
+			db.session.commit()
+			return redirect(url_for("admin_page"))
+		elif action == "vcsrelease":
+			for package in Package.query.filter(Package.repo.isnot(None)).all():
+				if package.releases.count() != 0:
+					continue
+
+				rel = PackageRelease()
+				rel.package  = package
+				rel.title	= datetime.date.today().isoformat()
+				rel.url	  = ""
+				rel.task_id  = uuid()
+				rel.approved = True
+				db.session.add(rel)
+				db.session.commit()
+
+				makeVCSRelease.apply_async((rel.id, "master"), task_id=rel.task_id)
+
+				msg = "{}: Release {} created".format(package.title, rel.title)
+				triggerNotif(package.author, current_user, msg, rel.getEditURL())
+				db.session.commit()
 
 		else:
 			flash("Unknown action: " + action, "error")
@@ -74,6 +129,7 @@ def admin_page():
 	deleted_packages = Package.query.filter_by(soft_deleted=True).all()
 	return render_template("admin/list.html", deleted_packages=deleted_packages)
 
+
 class SwitchUserForm(FlaskForm):
 	username = StringField("Username")
 	submit = SubmitField("Switch")

app/views/api.py

@@ -20,11 +20,13 @@ from flask_user import *
 from app import app
 from app.models import *
 from app.utils import is_package_page
-from .packages import build_packages_query
+from .packages import QueryBuilder
 
 @app.route("/api/packages/")
 def api_packages_page():
-	query, _ = build_packages_query()
+	qb    = QueryBuilder()
+	query = qb.buildPackageQuery()
+
 	pkgs = [package.getAsDictionaryShort(app.config["BASE_URL"]) \
 			for package in query.all() if package.getDownloadRelease() is not None]
 	return jsonify(pkgs)
@@ -33,3 +35,11 @@ def api_packages_page():
 @is_package_page
 def api_package_page(package):
 	return jsonify(package.getAsDictionary(app.config["BASE_URL"]))
+
+
+@app.route("/api/topics/")
+def api_topics_page():
+	query = ForumTopic.query \
+			.order_by(db.asc(ForumTopic.wip), db.asc(ForumTopic.name), db.asc(ForumTopic.title))
+	pkgs = [t.getAsDictionary() for t in query.all()]
+	return jsonify(pkgs)

app/views/githublogin.py

@@ -51,9 +51,9 @@ def github_authorized(oauth_token):
 	if current_user and current_user.is_authenticated:
 		if userByGithub is None:
 			current_user.github_username = username
-			db.session.add(auth)
 			db.session.commit()
-			return redirect(url_for("gitAccount", id=auth.id))
+			flash("Linked github to account", "success")
+			return redirect(url_for("home_page"))
 		else:
 			flash("Github account is already associated with another user", "danger")
 			return redirect(url_for("home_page"))

app/views/licenseseditor.py

@@ -0,0 +1,62 @@
+# Content DB
+# Copyright (C) 2018  rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+
+from flask import *
+from flask_user import *
+from app import app
+from app.models import *
+from flask_wtf import FlaskForm
+from wtforms import *
+from wtforms.validators import *
+from app.utils import rank_required
+
+@app.route("/licenses/")
+@rank_required(UserRank.MODERATOR)
+def license_list_page():
+	return render_template("admin/licenses/list.html", licenses=License.query.order_by(db.asc(License.name)).all())
+
+class LicenseForm(FlaskForm):
+	name	 = StringField("Name", [InputRequired(), Length(3,100)])
+	is_foss  = BooleanField("Is FOSS")
+	submit   = SubmitField("Save")
+
+@app.route("/licenses/new/", methods=["GET", "POST"])
+@app.route("/licenses/<name>/edit/", methods=["GET", "POST"])
+@rank_required(UserRank.MODERATOR)
+def createedit_license_page(name=None):
+	license = None
+	if name is not None:
+		license = License.query.filter_by(name=name).first()
+		if license is None:
+			abort(404)
+
+	form = LicenseForm(formdata=request.form, obj=license)
+	if request.method == "GET" and license is None:
+		form.is_foss.data = True
+	elif request.method == "POST" and form.validate():
+		if license is None:
+			license = License(form.name.data)
+			db.session.add(license)
+			flash("Created license " + form.name.data, "success")
+		else:
+			flash("Updated license " + form.name.data, "success")
+
+		form.populate_obj(license)
+		db.session.commit()
+		return redirect(url_for("license_list_page"))
+
+	return render_template("admin/licenses/edit.html", license=license, form=form)

app/views/packages/__init__.py

@@ -29,25 +29,59 @@ from flask_wtf import FlaskForm
 from wtforms import *
 from wtforms.validators import *
 from wtforms.ext.sqlalchemy.fields import QuerySelectField, QuerySelectMultipleField
+from sqlalchemy import or_, any_
 
-def build_packages_query():
-	type_name = request.args.get("type")
-	type = None
-	if type_name is not None:
-		type = PackageType[type_name.upper()]
 
-	title = "Packages"
+class QueryBuilder:
-	query = Package.query.filter_by(soft_deleted=False)
+	title  = None
+	types  = None
+	search = None
 
-	if type is not None:
+	def __init__(self):
-		title = type.value + "s"
+		title = "Packages"
-		query = query.filter_by(type=type, approved=True)
 
-	search = request.args.get("q")
+		# Get request types
-	if search is not None and search.strip() != "":
+		types = request.args.getlist("type")
-		query = query.filter(Package.title.ilike('%' + search + '%'))
+		types = [PackageType.get(tname) for tname in types]
+		types = [type for type in types if type is not None]
+		if len(types) > 0:
+			title = ", ".join([type.value + "s" for type in types])
 
-	return query, title
+		self.title  = title
+		self.types  = types
+		self.search = request.args.get("q")
+		self.lucky  = "lucky" in request.args
+		self.limit  = 1 if self.lucky else None
+
+	def buildPackageQuery(self):
+		query = Package.query.filter_by(soft_deleted=False, approved=True)
+
+		if len(self.types) > 0:
+			query = query.filter(Package.type.in_(self.types))
+
+		if self.search is not None and self.search.strip() != "":
+			query = query.filter(Package.title.ilike('%' + self.search + '%'))
+
+		query = query.order_by(db.desc(Package.score))
+
+		if self.limit:
+			query = query.limit(self.limit)
+
+		return query
+
+	def buildTopicQuery(self):
+		topics = ForumTopic.query \
+				.filter(~ db.exists().where(Package.forums==ForumTopic.topic_id)) \
+				.order_by(db.asc(ForumTopic.wip), db.asc(ForumTopic.name), db.asc(ForumTopic.title)) \
+				.filter(ForumTopic.title.ilike('%' + self.search + '%'))
+
+		if len(self.types) > 0:
+			topics = topics.filter(ForumTopic.type.in_(self.types))
+
+		if self.limit:
+			topics = topics.limit(self.limit)
+
+		return topics
 
 @menu.register_menu(app, ".mods", "Mods", order=11, endpoint_arguments_constructor=lambda: { 'type': 'mod' })
 @menu.register_menu(app, ".games", "Games", order=12, endpoint_arguments_constructor=lambda: { 'type': 'game' })
@@ -57,7 +91,19 @@ def packages_page():
 	if shouldReturnJson():
 		return redirect(url_for("api_packages_page"))
 
-	query, title = build_packages_query()
+	qb    = QueryBuilder()
+	query = qb.buildPackageQuery()
+	title = qb.title
+
+	if qb.lucky:
+		package = query.first()
+		if package:
+			return redirect(package.getDetailsURL())
+
+		topic = qb.buildTopicQuery().first()
+		if topic:
+			return redirect("https://forum.minetest.net/viewtopic.php?t=" + str(topic.topic_id))
+
 	page  = int(request.args.get("page") or 1)
 	num   = min(42, int(request.args.get("n") or 100))
 	query = query.paginate(page, num, True)
@@ -70,8 +116,13 @@ def packages_page():
 	prev_url = url_for("packages_page", type=type_name, q=search, page=query.prev_num) \
 			if query.has_prev else None
 
+	topics = None
+	if qb.search and not query.has_next:
+		topics = qb.buildTopicQuery().all()
+
 	tags = Tag.query.all()
-	return render_template("packages/list.html", title=title, packages=query.items, \
+	return render_template("packages/list.html", \
+			title=title, packages=query.items, topics=topics, \
 			query=search, tags=tags, type=type_name, \
 			next_url=next_url, prev_url=prev_url, page=page, page_max=query.pages, packages_count=query.total)
 
@@ -100,24 +151,54 @@ def package_page(package):
 			package.checkPerm(current_user, Permission.APPROVE_NEW)
 
 	similar_topics = None if not show_similar_topics else \
-			KrockForumTopic.query \
+			ForumTopic.query \
 				.filter_by(name=package.name) \
-				.filter(KrockForumTopic.topic_id != package.forums) \
+				.filter(ForumTopic.topic_id != package.forums) \
-				.filter(~ db.exists().where(Package.forums==KrockForumTopic.topic_id)) \
+				.filter(~ db.exists().where(Package.forums==ForumTopic.topic_id)) \
-				.order_by(db.asc(KrockForumTopic.name), db.asc(KrockForumTopic.title)) \
+				.order_by(db.asc(ForumTopic.name), db.asc(ForumTopic.title)) \
 				.all()
 
 	releases = getReleases(package)
 	requests = [r for r in package.requests if r.status == 0]
 
-	review_thread = Thread.query.filter_by(package_id=package.id, private=True).first()
+	review_thread = package.review_thread
 	if review_thread is not None and not review_thread.checkPerm(current_user, Permission.SEE_THREAD):
 		review_thread = None
 
+	topic_error = None
+	topic_error_lvl = "warning"
+	if not package.approved and package.forums is not None:
+		errors = []
+		if Package.query.filter_by(forums=package.forums, soft_deleted=False).count() > 1:
+			errors.append("<b>Error: Another package already uses this forum topic!</b>")
+			topic_error_lvl = "error"
+
+		topic = ForumTopic.query.get(package.forums)
+		if topic is not None:
+			if topic.author != package.author:
+				errors.append("<b>Error: Forum topic author doesn't match package author.</b>")
+				topic_error_lvl = "error"
+
+			if topic.wip:
+				errors.append("Warning: Forum topic is in WIP section, make sure package meets playability standards.")
+		elif package.type != PackageType.TXP:
+			errors.append("Warning: Forum topic not found. This may happen if the topic has only just been created.")
+
+		topic_error = "<br />".join(errors)
+
+
+	threads = Thread.query.filter_by(package_id=package.id)
+	if not current_user.is_authenticated:
+		threads = threads.filter_by(private=False)
+	elif not current_user.rank.atLeast(UserRank.EDITOR) and not current_user == package.author:
+		threads = threads.filter(or_(Thread.private == False, Thread.author == current_user))
+
+
 	return render_template("packages/view.html", \
 			package=package, releases=releases, requests=requests, \
 			alternatives=alternatives, similar_topics=similar_topics, \
-			review_thread=review_thread)
+			review_thread=review_thread, topic_error=topic_error, topic_error_lvl=topic_error_lvl, \
+			threads=threads.all())
 
 
 @app.route("/packages/<author>/<name>/download/")
@@ -142,8 +223,8 @@ class PackageForm(FlaskForm):
 	shortDesc     = StringField("Short Description (Plaintext)", [InputRequired(), Length(1,200)])
 	desc          = TextAreaField("Long Description (Markdown)", [Optional(), Length(0,10000)])
 	type          = SelectField("Type", [InputRequired()], choices=PackageType.choices(), coerce=PackageType.coerce, default=PackageType.MOD)
-	license       = QuerySelectField("License", [InputRequired()], query_factory=lambda: License.query, get_pk=lambda a: a.id, get_label=lambda a: a.name)
+	license       = QuerySelectField("License", [InputRequired()], query_factory=lambda: License.query.order_by(db.asc(License.name)), get_pk=lambda a: a.id, get_label=lambda a: a.name)
-	media_license = QuerySelectField("Media License", [InputRequired()], query_factory=lambda: License.query, get_pk=lambda a: a.id, get_label=lambda a: a.name)
+	media_license = QuerySelectField("Media License", [InputRequired()], query_factory=lambda: License.query.order_by(db.asc(License.name)), get_pk=lambda a: a.id, get_label=lambda a: a.name)
 	provides_str  = StringField("Provides (mods included in package)", [Optional(), Length(0,1000)])
 	tags          = QuerySelectMultipleField('Tags', query_factory=lambda: Tag.query.order_by(db.asc(Tag.name)), get_pk=lambda a: a.id, get_label=lambda a: a.title)
 	harddep_str   = StringField("Hard Dependencies", [Optional(), Length(0,1000)])
@@ -244,11 +325,15 @@ def create_edit_package_page(author=None, name=None):
 
 		db.session.commit() # save
 
+		next_url = package.getDetailsURL()
 		if wasNew and package.repo is not None:
 			task = importRepoScreenshot.delay(package.id)
-			return redirect(url_for("check_task", id=task.id, r=package.getDetailsURL()))
+			next_url = url_for("check_task", id=task.id, r=next_url)
 
-		return redirect(package.getDetailsURL())
+		if wasNew and ("WTFPL" in package.license.name or "WTFPL" in package.media_license.name):
+			next_url = url_for("flatpage", path="help/wtfpl", r=next_url)
+
+		return redirect(next_url)
 
 	package_query = Package.query.filter_by(approved=True, soft_deleted=False)
 	if package is not None:

app/views/packages/releases.py

@@ -91,6 +91,23 @@ def create_release_page(package):
 
 	return render_template("packages/release_new.html", package=package, form=form)
 
+@app.route("/packages/<author>/<name>/releases/<id>/download/")
+@is_package_page
+def download_release_page(package, id):
+	release = PackageRelease.query.get(id)
+	if release is None or release.package != package:
+		abort(404)
+
+	if release is None:
+		if "application/zip" in request.accept_mimetypes and \
+				not "text/html" in request.accept_mimetypes:
+			return "", 204
+		else:
+			flash("No download available.", "error")
+			return redirect(package.getDetailsURL())
+	else:
+		return redirect(release.url, code=300)
+
 @app.route("/packages/<author>/<name>/releases/<id>/", methods=["GET", "POST"])
 @login_required
 @is_package_page

app/views/packages/screenshots.py

@@ -54,7 +54,7 @@ def create_screenshot_page(package, id=None):
 		if uploadedPath is not None:
 			ss = PackageScreenshot()
 			ss.package = package
-			ss.title   = form["title"].data
+			ss.title   = form["title"].data or "Untitled"
 			ss.url     = uploadedPath
 			db.session.add(ss)
 
@@ -91,7 +91,7 @@ def edit_screenshot_page(package, id):
 			wasApproved = screenshot.approved
 
 			if canEdit:
-				screenshot.title = form["title"].data
+				screenshot.title = form["title"].data or "Untitled"
 
 			if canApprove:
 				screenshot.approved = form["approved"].data

app/views/packages/todo.py

@@ -41,8 +41,8 @@ def todo_page():
 		screenshots = PackageScreenshot.query.filter_by(approved=False).all()
 
 
-	topics_to_add = KrockForumTopic.query \
+	topics_to_add = ForumTopic.query \
-			.filter(~ db.exists().where(Package.forums==KrockForumTopic.topic_id)) \
+			.filter(~ db.exists().where(Package.forums==ForumTopic.topic_id)) \
 			.count()
 
 	return render_template("todo/list.html", title="Reports and Work Queue",
@@ -54,11 +54,11 @@ def todo_page():
 @app.route("/todo/topics/")
 @login_required
 def todo_topics_page():
-	total = KrockForumTopic.query.count()
+	total = ForumTopic.query.count()
 
-	topics = KrockForumTopic.query \
+	topics = ForumTopic.query \
-			.filter(~ db.exists().where(Package.forums==KrockForumTopic.topic_id)) \
+			.filter(~ db.exists().where(Package.forums==ForumTopic.topic_id)) \
-			.order_by(db.asc(KrockForumTopic.name), db.asc(KrockForumTopic.title)) \
+			.order_by(db.asc(ForumTopic.wip), db.asc(ForumTopic.name), db.asc(ForumTopic.title)) \
 			.all()
 
 	return render_template("todo/topics.html", topics=topics, total=total)

app/views/tagseditor.py

@@ -27,7 +27,7 @@ from app.utils import rank_required
 @app.route("/tags/")
 @rank_required(UserRank.MODERATOR)
 def tag_list_page():
-	return render_template("tags/list.html", tags=Tag.query.order_by(db.asc(Tag.title)).all())
+	return render_template("admin/tags/list.html", tags=Tag.query.order_by(db.asc(Tag.title)).all())
 
 class TagForm(FlaskForm):
 	title	 = StringField("Title", [InputRequired(), Length(3,100)])
@@ -54,4 +54,4 @@ def createedit_tag_page(name=None):
 		db.session.commit()
 		return redirect(url_for("createedit_tag_page", name=tag.name))
 
-	return render_template("tags/edit.html", tag=tag, form=form)
+	return render_template("admin/tags/edit.html", tag=tag, form=form)

app/views/threads.py

@@ -27,8 +27,45 @@ from wtforms.validators import *
 
 @app.route("/threads/")
 def threads_page():
-	threads = Thread.query.filter_by(private=False).all()
+	query = Thread.query
-	return render_template("threads/list.html", threads=threads)
+	if not Permission.SEE_THREAD.check(current_user):
+		query = query.filter_by(private=False)
+	return render_template("threads/list.html", threads=query.all())
+
+
+@app.route("/threads/<int:id>/subscribe/", methods=["POST"])
+@login_required
+def thread_subscribe_page(id):
+	thread = Thread.query.get(id)
+	if thread is None or not thread.checkPerm(current_user, Permission.SEE_THREAD):
+		abort(404)
+
+	if current_user in thread.watchers:
+		flash("Already subscribed!", "success")
+	else:
+		flash("Subscribed to thread", "success")
+		thread.watchers.append(current_user)
+		db.session.commit()
+
+	return redirect(url_for("thread_page", id=id))
+
+
+@app.route("/threads/<int:id>/unsubscribe/", methods=["POST"])
+@login_required
+def thread_unsubscribe_page(id):
+	thread = Thread.query.get(id)
+	if thread is None or not thread.checkPerm(current_user, Permission.SEE_THREAD):
+		abort(404)
+
+	if current_user in thread.watchers:
+		flash("Unsubscribed!", "success")
+		thread.watchers.remove(current_user)
+		db.session.commit()
+	else:
+		flash("Not subscribed to thread", "success")
+
+	return redirect(url_for("thread_page", id=id))
+
 
 @app.route("/threads/<int:id>/", methods=["GET", "POST"])
 def thread_page(id):
@@ -90,7 +127,7 @@ def new_thread_page():
 			flash("Unable to find that package!", "error")
 
 	# Don't allow making threads on approved packages for now
-	if package is None or package.approved:
+	if package is None:
 		abort(403)
 
 	def_is_private   = request.args.get("private") or False
@@ -100,8 +137,7 @@ def new_thread_page():
 	is_review_thread = package is not None and not package.approved
 
 	# Check that user can make the thread
-	if is_review_thread and not (package.author == current_user or \
+	if not package.checkPerm(current_user, Permission.CREATE_THREAD):
-			package.checkPerm(current_user, Permission.APPROVE_NEW)):
 		flash("Unable to create thread!", "error")
 		return redirect(url_for("home_page"))
 

app/views/thumbnails.py

@@ -18,7 +18,7 @@
 from flask import *
 from app import app
 
-import glob, os
+import os
 from PIL import Image
 
 ALLOWED_RESOLUTIONS=[(350,233)]
@@ -29,6 +29,33 @@ def mkdir(path):
 
 mkdir("app/public/thumbnails/")
 
+def resize_and_crop(img_path, modified_path, size):
+	img = Image.open(img_path)
+
+	# Get current and desired ratio for the images
+	img_ratio = img.size[0] / float(img.size[1])
+	ratio = size[0] / float(size[1])
+
+	# Is more portrait than target, scale and crop
+	if ratio > img_ratio:
+		img = img.resize((int(size[0]), int(size[0] * img.size[1] / img.size[0])),
+				Image.BICUBIC)
+		box = (0, (img.size[1] - size[1]) / 2, img.size[0], (img.size[1] + size[1]) / 2)
+		img = img.crop(box)
+
+	# Is more landscape than target, scale and crop
+	elif ratio < img_ratio:
+		img = img.resize((int(size[1] * img.size[0] / img.size[1]), int(size[1])),
+				Image.BICUBIC)
+		box = ((img.size[0] - size[0]) / 2, 0, (img.size[0] + size[0]) / 2, img.size[1])
+		img = img.crop(box)
+
+	# Is exactly the same ratio as target
+	else:
+		img = img.resize(size, Image.BICUBIC)
+
+	img.save(modified_path)
+
 @app.route("/thumbnails/<img>")
 @app.route("/thumbnails/<int:w>x<int:h>/<img>")
 def make_thumbnail(img, w=350, h=233):
@@ -40,7 +67,5 @@ def make_thumbnail(img, w=350, h=233):
 	cache_filepath  = "public/thumbnails/{}x{}/{}".format(w, h, img)
 	source_filepath = "public/uploads/" + img
 
-	im = Image.open("app/" + source_filepath)
+	resize_and_crop("app/" + source_filepath, "app/" + cache_filepath, (w, h))
-	im.thumbnail((w, h), Image.ANTIALIAS)
-	im.save("app/" + cache_filepath, optimize=True)
 	return send_file(cache_filepath)

app/views/users.py

@@ -98,10 +98,10 @@ def user_profile_page(username):
 
 	topics_to_add = None
 	if current_user == user or user.checkPerm(current_user, Permission.CHANGE_AUTHOR):
-		topics_to_add = KrockForumTopic.query \
+		topics_to_add = ForumTopic.query \
 					.filter_by(author_id=user.id) \
-					.filter(~ db.exists().where(Package.forums==KrockForumTopic.topic_id)) \
+					.filter(~ db.exists().where(Package.forums==ForumTopic.topic_id)) \
-					.order_by(db.asc(KrockForumTopic.name), db.asc(KrockForumTopic.title)) \
+					.order_by(db.asc(ForumTopic.name), db.asc(ForumTopic.title)) \
 					.all()
 
 	# Process GET or invalid POST

config.example.cfg

@@ -22,3 +22,4 @@ MAIL_DEFAULT_SENDER=""
 MAIL_SERVER=""
 MAIL_PORT=587
 MAIL_USE_TLS=True
+MAIL_UTILS_ERROR_SEND_TO=[""]

migrations/versions/11b6ef362f98_.py

@@ -0,0 +1,28 @@
+"""empty message
+
+Revision ID: 11b6ef362f98
+Revises: 9fc23495713b
+Create Date: 2018-07-04 01:01:45.440662
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '11b6ef362f98'
+down_revision = '9fc23495713b'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('package', sa.Column('score', sa.Float(), nullable=False, server_default="0.0"))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('package', 'score')
+    # ### end Alembic commands ###

migrations/versions/44e138485931_.py

@@ -0,0 +1,28 @@
+"""empty message
+
+Revision ID: 44e138485931
+Revises: 9e2ac631efb0
+Create Date: 2018-07-28 14:45:28.879331
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '44e138485931'
+down_revision = '9e2ac631efb0'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('package_release', sa.Column('commit_hash', sa.String(length=41), nullable=True, server_default=None))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('package_release', 'commit_hash')
+    # ### end Alembic commands ###

migrations/versions/9e2ac631efb0_.py

@@ -0,0 +1,28 @@
+"""empty message
+
+Revision ID: 9e2ac631efb0
+Revises: 11b6ef362f98
+Create Date: 2018-07-06 23:16:50.507010
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '9e2ac631efb0'
+down_revision = '11b6ef362f98'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('forum_topic', sa.Column('wip', sa.Boolean(), nullable=False, server_default="0"))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('forum_topic', 'wip')
+    # ### end Alembic commands ###

migrations/versions/9fc23495713b_.py

@@ -0,0 +1,55 @@
+"""empty message
+
+Revision ID: 9fc23495713b
+Revises: de004661c5e1
+Create Date: 2018-07-04 00:03:20.123285
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '9fc23495713b'
+down_revision = 'de004661c5e1'
+branch_labels = None
+depends_on = None
+from sqlalchemy.dialects.postgresql import ENUM
+
+type_enum = ENUM('MOD', 'GAME', 'TXP', name='packagetype', create_type=False)
+
+def upgrade():
+    type_enum.create(op.get_bind(), checkfirst=True)
+
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('krock_forum_topic')
+    op.create_table('forum_topic',
+    sa.Column('topic_id', sa.Integer(), autoincrement=False, nullable=False),
+    sa.Column('author_id', sa.Integer(), nullable=False),
+    sa.Column('type', type_enum, nullable=True),
+    sa.Column('title', sa.String(length=200), nullable=False),
+    sa.Column('name', sa.String(length=30), nullable=True),
+    sa.Column('link', sa.String(length=200), nullable=True),
+    sa.Column('posts', sa.Integer(), nullable=False),
+    sa.Column('views', sa.Integer(), nullable=False),
+    sa.Column('created_at', sa.DateTime(), nullable=False),
+    sa.ForeignKeyConstraint(['author_id'], ['user.id'], ),
+    sa.PrimaryKeyConstraint('topic_id')
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('forum_topic')
+    op.create_table('krock_forum_topic',
+    sa.Column('topic_id', sa.Integer(), autoincrement=False, nullable=False),
+    sa.Column('author_id', sa.Integer(), nullable=False),
+    sa.Column('ttype', sa.Integer(), nullable=False),
+    sa.Column('title', sa.String(length=200), nullable=False),
+    sa.Column('name', sa.String(length=30), nullable=True),
+    sa.Column('link', sa.String(length=50), nullable=True),
+    sa.ForeignKeyConstraint(['author_id'], ['user.id'], ),
+    sa.PrimaryKeyConstraint('topic_id')
+    )
+    # ### end Alembic commands ###

requirements.txt

@@ -1,4 +1,4 @@
-Flask>=0.12.2
+Flask>=0.12.2,<1.0
 Flask-SQLAlchemy>=2.3
 Flask-Login>=0.4.1
 Flask-User==0.6.19
@@ -14,3 +14,4 @@ Flask-FlatPages==0.6
 Flask-Migrate==2.1.1
 pillow==5.1.0
 GitPython==2.1.10
+flask-admin==1.5.1

setup.py

@@ -23,6 +23,7 @@ if not "FLASK_CONFIG" in os.environ:
 test_data = len(sys.argv) >= 2 and sys.argv[1].strip() == "-t"
 
 from app.models import *
+from app.utils import make_flask_user_password
 
 def defineDummyData(licenses, tags, ruben):
 	ez = User("Shara")
@@ -342,6 +343,8 @@ db.create_all()
 print("Filling database...")
 
 ruben = User("rubenwardy")
+ruben.active = True
+ruben.password = make_flask_user_password("tuckfrump")
 ruben.github_username = "rubenwardy"
 ruben.forums_username = "rubenwardy"
 ruben.rank = UserRank.ADMIN
@@ -359,12 +362,12 @@ for tag in ["Inventory", "Mapgen", "Building", \
 licenses = {}
 for license in ["GPLv2.1", "GPLv3", "LGPLv2.1", "LGPLv3", "AGPLv2.1", "AGPLv3",
 				"Apache", "BSD 3-Clause", "BSD 2-Clause", "CC0", "CC-BY-SA",
-				"CC-BY", "MIT", "ZLib"]:
+				"CC-BY", "MIT", "ZLib", "Other (Free)"]:
 	row = License(license)
 	licenses[row.name] = row
 	db.session.add(row)
 
-for license in ["CC-BY-NC-SA"]:
+for license in ["CC-BY-NC-SA", "Other (Non-free)"]:
 	row = License(license, False)
 	licenses[row.name] = row
 	db.session.add(row)