Fix forum topic scanner only scanning one page

Fixes #101

app/__init__.py

@@ -37,5 +37,9 @@ csrf = CsrfProtect(app)
 mail = Mail(app)
 pages = FlatPages(app)
 
+if not app.debug:
+	from .maillogger import register_mail_error_handler
+	register_mail_error_handler(app, mail)
+
 from . import models, tasks
 from .views import *

app/flatpages/policy_and_guidance.md

@@ -39,7 +39,7 @@ installed when a mod depends on it.
 The first package to use a name based on the creation of its forum topic or
 contentdb submission has the right to the technical name. The use of a package
 on a server or in private doesn't reserve its name. No other packages of the same
-type may use the same name, except for the exception given by 2.2.
+type may use the same name, except for the exception given by 3.2.
 
 If it turns out that we made a mistake by approving a package and that the
 name should have been given to another package, then we *may* unapprove the

app/maillogger.py

@@ -0,0 +1,109 @@
+import logging
+from enum import Enum
+from app.tasks.emails import sendEmailRaw
+
+def _has_newline(line):
+	"""Used by has_bad_header to check for \\r or \\n"""
+	if line and ("\r" in line or "\n" in line):
+		return True
+	return False
+
+def _is_bad_subject(subject):
+	"""Copied from: flask_mail.py class Message def has_bad_headers"""
+	if _has_newline(subject):
+		for linenum, line in enumerate(subject.split("\r\n")):
+			if not line:
+				return True
+			if linenum > 0 and line[0] not in "\t ":
+				return True
+			if _has_newline(line):
+				return True
+			if len(line.strip()) == 0:
+				return True
+	return False
+
+
+class FlaskMailSubjectFormatter(logging.Formatter):
+	def format(self, record):
+		record.message = record.getMessage()
+		if self.usesTime():
+			record.asctime = self.formatTime(record, self.datefmt)
+		s = self.formatMessage(record)
+		return s
+
+class FlaskMailTextFormatter(logging.Formatter):
+	pass
+
+# TODO: hier nog niet tevreden over (vooral logger.error(..., exc_info, stack_info))
+class FlaskMailHTMLFormatter(logging.Formatter):
+	pre_template = "<h1>%s</h1><pre>%s</pre>"
+	def formatException(self, exc_info):
+		formatted_exception = logging.Handler.formatException(self, exc_info)
+		return FlaskMailHTMLFormatter.pre_template % ("Exception information", formatted_exception)
+	def formatStack(self, stack_info):
+		return FlaskMailHTMLFormatter.pre_template % ("<h1>Stack information</h1><pre>%s</pre>", stack_info)
+
+
+# see: https://github.com/python/cpython/blob/3.6/Lib/logging/__init__.py (class Handler)
+
+class FlaskMailHandler(logging.Handler):
+	def __init__(self, mailer, subject_template, level=logging.NOTSET):
+		logging.Handler.__init__(self, level)
+		self.mailer = mailer
+		self.send_to = mailer.app.config["MAIL_UTILS_ERROR_SEND_TO"]
+		self.subject_template = subject_template
+		self.html_formatter = None
+
+	def setFormatter(self, text_fmt, html_fmt=None):
+		"""
+		Set the formatters for this handler. Provide at least one formatter.
+		When no text_fmt is provided, no text-part is created for the email body.
+		"""
+		assert (text_fmt, html_fmt) != (None, None), "At least one formatter should be provided"
+		if type(text_fmt)==str:
+			text_fmt = FlaskMailTextFormatter(text_fmt)
+		self.formatter = text_fmt
+		if type(html_fmt)==str:
+			html_fmt = FlaskMailHTMLFormatter(html_fmt)
+		self.html_formatter = html_fmt
+
+	def getSubject(self, record):
+		fmt = FlaskMailSubjectFormatter(self.subject_template)
+		subject = fmt.format(record)
+		#Since templates can cause header problems, and we rather have a incomplete email then an error, we fix this
+		if _is_bad_subject(subject):
+			subject="FlaskMailHandler log-entry from %s [original subject is replaced, because it would result in a bad header]" % self.mailer.app.name
+		return subject
+
+	def emit(self, record):
+		text = self.format(record)				if self.formatter	  else None
+		html = self.html_formatter.format(record) if self.html_formatter else None
+		sendEmailRaw.delay(self.send_to, self.getSubject(record), text, html)
+
+
+def register_mail_error_handler(app, mailer):
+	subject_template = "ContentDB crashed (%(module)s > %(funcName)s)"
+	text_template = """
+Message type: %(levelname)s
+Location:	 %(pathname)s:%(lineno)d
+Module:	   %(module)s
+Function:	 %(funcName)s
+Time:		 %(asctime)s
+Message:
+%(message)s"""
+	html_template = """
+<style>th { text-align: right}</style><table>
+<tr><th>Message type:</th><td>%(levelname)s</td></tr>
+<tr>	<th>Location:</th><td>%(pathname)s:%(lineno)d</td></tr>
+<tr>	  <th>Module:</th><td>%(module)s</td></tr>
+<tr>	<th>Function:</th><td>%(funcName)s</td></tr>
+<tr>		<th>Time:</th><td>%(asctime)s</td></tr>
+</table>
+<h2>Message</h2>
+<pre>%(message)s</pre>"""
+
+	import logging
+	mail_handler = FlaskMailHandler(mailer, subject_template)
+	mail_handler.setLevel(logging.ERROR)
+	mail_handler.setFormatter(text_template, html_template)
+	app.logger.addHandler(mail_handler)

app/models.py

@@ -85,9 +85,10 @@ class Permission(enum.Enum):
 			return False
 
 		if self == Permission.APPROVE_NEW or \
-				self == Permission.APPROVE_CHANGES or \
-				self == Permission.APPROVE_RELEASE or \
-				self == Permission.APPROVE_SCREENSHOT:
+				self == Permission.APPROVE_CHANGES    or \
+				self == Permission.APPROVE_RELEASE    or \
+				self == Permission.APPROVE_SCREENSHOT or \
+				self == Permission.SEE_THREAD:
 			return user.rank.atLeast(UserRank.EDITOR)
 		else:
 			raise Exception("Non-global permission checked globally. Use Package.checkPerm or User.checkPerm instead.")
@@ -339,6 +340,8 @@ class Package(db.Model):
 	approved     = db.Column(db.Boolean, nullable=False, default=False)
 	soft_deleted = db.Column(db.Boolean, nullable=False, default=False)
 
+	score        = db.Column(db.Float, nullable=False, default=0)
+
 	review_thread_id = db.Column(db.Integer, db.ForeignKey("thread.id"), nullable=True, default=None)
 	review_thread    = db.relationship("Thread", foreign_keys=[review_thread_id])
 
@@ -376,7 +379,7 @@ class Package(db.Model):
 		for e in PackagePropertyKey:
 			setattr(self, e.name, getattr(package, e.name))
 
-	def getAsDictionary(self, base_url):
+	def getAsDictionaryShort(self, base_url):
 		tnurl = self.getThumbnailURL()
 		return {
 			"name": self.name,
@@ -384,12 +387,38 @@ class Package(db.Model):
 			"author": self.author.display_name,
 			"shortDesc": self.shortDesc,
 			"type": self.type.toName(),
+			"release": self.getDownloadRelease().id if self.getDownloadRelease() is not None else None,
+			"thumbnail": (base_url + tnurl) if tnurl is not None else None,
+			"score": round(self.score * 10) / 10
+		}
+
+	def getAsDictionary(self, base_url):
+		tnurl = self.getThumbnailURL()
+		return {
+			"author": self.author.display_name,
+			"name": self.name,
+			"title": self.title,
+			"shortDesc": self.shortDesc,
+			"desc": self.desc,
+			"type": self.type.toName(),
+			"createdAt": self.created_at,
+
 			"license": self.license.name,
+			"mediaLicense": self.media_license.name,
+
 			"repo": self.repo,
+			"website": self.website,
+			"issueTracker": self.issueTracker,
+			"forums": self.forums,
+
+			"provides": [x.name for x in self.provides],
+			"thumbnail": (base_url + tnurl) if tnurl is not None else None,
+			"screenshots": [base_url + ss.url for ss in self.screenshots],
+
 			"url": base_url + self.getDownloadURL(),
 			"release": self.getDownloadRelease().id if self.getDownloadRelease() is not None else None,
-			"screenshots": [base_url + ss.url for ss in self.screenshots],
-			"thumbnail": (base_url + tnurl) if tnurl is not None else None
+
+			"score": round(self.score * 10) / 10
 		}
 
 	def getThumbnailURL(self):
@@ -475,6 +504,21 @@ class Package(db.Model):
 		else:
 			raise Exception("Permission {} is not related to packages".format(perm.name))
 
+	def recalcScore(self):
+		import datetime
+
+		self.score = 0
+
+		if self.forums is None:
+			return
+
+		topic = ForumTopic.query.get(self.forums)
+		if topic:
+			days   = (datetime.datetime.now() - topic.created_at).days
+			months = days / 30
+			years  = days / 365
+			self.score = topic.views / years + 80*min(6, months)
+
 class MetaPackage(db.Model):
 	id           = db.Column(db.Integer, primary_key=True)
 	name         = db.Column(db.String(100), unique=True, nullable=False)
@@ -695,7 +739,7 @@ class Thread(db.Model):
 		elif type(perm) != Permission:
 			raise Exception("Unknown permission given to Thread.checkPerm()")
 
-		isOwner = user == self.author
+		isOwner = user == self.author or (self.package is not None and self.package.author == user)
 
 		if perm == Permission.SEE_THREAD:
 			return not self.private or isOwner or user.rank.atLeast(UserRank.EDITOR)
@@ -720,29 +764,47 @@ REPO_BLACKLIST = [".zip", "mediafire.com", "dropbox.com", "weebly.com", \
 		"digitalaudioconcepts.com", "hg.intevation.org", "www.wtfpl.net", \
 		"imageshack.com", "imgur.com"]
 
-class KrockForumTopic(db.Model):
+class ForumTopic(db.Model):
 	topic_id  = db.Column(db.Integer, primary_key=True, autoincrement=False)
 	author_id = db.Column(db.Integer, db.ForeignKey("user.id"), nullable=False)
 	author    = db.relationship("User")
 
-	ttype     = db.Column(db.Integer, nullable=False)
+	wip       = db.Column(db.Boolean, server_default="0")
+
+	type      = db.Column(db.Enum(PackageType), nullable=False)
 	title     = db.Column(db.String(200), nullable=False)
 	name      = db.Column(db.String(30), nullable=True)
 	link      = db.Column(db.String(200), nullable=True)
 
-	def getType(self):
-		if self.ttype == 1 or self.ttype == 2:
-			return PackageType.MOD
-		elif self.ttype == 6:
-			return PackageType.GAME
+	posts     = db.Column(db.Integer, nullable=False)
+	views     = db.Column(db.Integer, nullable=False)
+
+	created_at = db.Column(db.DateTime, nullable=False, default=datetime.utcnow)
 
 	def getRepoURL(self):
+		if self.link is None:
+			return None
+
 		for item in REPO_BLACKLIST:
 			if item in self.link:
 				return None
 
 		return self.link.replace("repo.or.cz/w/", "repo.or.cz/")
 
+	def getAsDictionary(self):
+		return {
+			"author": self.author.username,
+			"name":   self.name,
+			"type":   self.type.toName(),
+			"title":  self.title,
+			"id":     self.topic_id,
+			"link":   self.link,
+			"posts":  self.posts,
+			"views":  self.views,
+			"is_wip": self.wip,
+			"created_at": self.created_at.isoformat(),
+		}
+
 
 # Setup Flask-User
 db_adapter = SQLAlchemyAdapter(db, User)        # Register the User model

app/scss/components.scss

@@ -451,3 +451,7 @@ table.fancyTable tfoot td {
 .table-topalign td {
 	vertical-align: top;
 }
+
+.wiptopic a {
+	color: #7ac;
+}

app/tasks/emails.py

@@ -22,7 +22,17 @@ from app.tasks import celery
 
 @celery.task()
 def sendVerifyEmail(newEmail, token):
-    msg = Message("Verify email address", recipients=[newEmail])
-    msg.body = "This is a verification email!"
-    msg.html = render_template("emails/verify.html", token=token)
-    mail.send(msg)
+	msg = Message("Verify email address", recipients=[newEmail])
+	msg.body = "This is a verification email!"
+	msg.html = render_template("emails/verify.html", token=token)
+	mail.send(msg)
+
+@celery.task()
+def sendEmailRaw(to, subject, text, html):
+	from flask_mail import Message
+	msg = Message(subject, recipients=to)
+	if text:
+		msg.body = text
+	if html:
+		msg.html = html
+	mail.send(msg)

app/tasks/forumtasks.py

@@ -15,12 +15,12 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 
-import flask, json
+import flask, json, re
 from flask.ext.sqlalchemy import SQLAlchemy
 from app import app
 from app.models import *
 from app.tasks import celery
-from .phpbbparser import getProfile
+from .phpbbparser import getProfile, getTopicsFromForum
 import urllib.request
 from urllib.parse import urlparse, quote_plus
 
@@ -51,71 +51,91 @@ def checkForumAccount(username, token=None):
 	if needsSaving:
 		db.session.commit()
 
-@celery.task()
-def importUsersFromModList():
+
+regex_tag    = re.compile(r"\[([a-z0-9_]+)\]")
+BANNED_NAMES = ["mod", "game", "old", "outdated", "wip", "api", "beta", "alpha", "git"]
+def getNameFromTaglist(taglist):
+	for tag in reversed(regex_tag.findall(taglist)):
+		if len(tag) < 30 and not tag in BANNED_NAMES and \
+				not re.match(r"^[a-z]?[0-9]+$", tag):
+			return tag
+
+	return None
+
+regex_title = re.compile(r"^((?:\[[^\]]+\] *)*)([^\[]+) *((?:\[[^\]]+\] *)*)[^\[]*$")
+def parseTitle(title):
+	m = regex_title.match(title)
+	if m is None:
+		print("Invalid title format: " + title)
+		return title, getNameFromTaglist(title)
+	else:
+		return m.group(2).strip(), getNameFromTaglist(m.group(3))
+
+def getLinksFromModSearch():
+	links = {}
+
 	contents = urllib.request.urlopen("http://krock-works.16mb.com/MTstuff/modList.php").read().decode("utf-8")
-	list = json.loads(contents)
-	found = {}
-	imported = []
+	for x in json.loads(contents):
+		link = x.get("link")
+		if link is not None:
+			links[int(x["topicId"])] = link
 
-	for user in User.query.all():
-		found[user.username] = True
-		if user.forums_username is not None:
-			found[user.forums_username] = True
-
-	for x in list:
-		author = x.get("author")
-		if author is not None and not author in found:
-			user = User(author)
-			user.forums_username = author
-			imported.append(author)
-			found[author] = True
-			db.session.add(user)
-
-	db.session.commit()
-	for author in found:
-		checkForumAccount.delay(author, None)
-
-
-BANNED_NAMES = ["mod", "game", "old", "outdated", "wip", "api"]
-ALLOWED_TYPES = [1, 2, 6]
+	return links
 
 @celery.task()
-def importKrocksModList():
-	contents = urllib.request.urlopen("http://krock-works.16mb.com/MTstuff/modList.php").read().decode("utf-8")
-	list = json.loads(contents)
+def importTopicList():
+	links_by_id = getLinksFromModSearch()
+
+	info_by_id = {}
+	getTopicsFromForum(11, out=info_by_id, extra={ 'type': PackageType.MOD,  'wip': False })
+	getTopicsFromForum(9,  out=info_by_id, extra={ 'type': PackageType.MOD,  'wip': True  })
+	getTopicsFromForum(15, out=info_by_id, extra={ 'type': PackageType.GAME, 'wip': False })
+	getTopicsFromForum(50, out=info_by_id, extra={ 'type': PackageType.GAME, 'wip': True  })
+
+	# Caches
 	username_to_user = {}
+	topics_by_id     = {}
+	for topic in ForumTopic.query.all():
+		topics_by_id[topic.topic_id] = topic
 
-	KrockForumTopic.query.delete()
+	# Create or update
+	for info in info_by_id.values():
+		id = int(info["id"])
 
-	for x in list:
-		type = int(x["type"])
-		if not type in ALLOWED_TYPES:
-			continue
-
-		username = x["author"]
+		# Get author
+		username = info["author"]
 		user = username_to_user.get(username)
 		if user is None:
 			user = User.query.filter_by(forums_username=username).first()
-			assert(user is not None)
+			if user is None:
+				print(username + " not found!")
+				user = User(username)
+				user.forums_username = username
+				db.session.add(user)
 			username_to_user[username] = user
 
-		import re
-		tags = re.findall("\[([a-z0-9_]+)\]", x["title"])
-		name = None
-		for tag in reversed(tags):
-			if len(tag) < 30 and not tag in BANNED_NAMES and \
-					not re.match("^([a-z][0-9]+)$", tag):
-				name = tag
-				break
+		# Get / add row
+		topic = topics_by_id.get(id)
+		if topic is None:
+			topic = ForumTopic()
+			db.session.add(topic)
 
-		topic = KrockForumTopic()
-		topic.topic_id  = x["topicId"]
-		topic.author_id = user.id
-		topic.ttype     = type
-		topic.title     = x["title"]
-		topic.name      = name
-		topic.link      = x.get("link")
-		db.session.add(topic)
+		# Parse title
+		title, name = parseTitle(info["title"])
+
+		# Get link
+		link = links_by_id.get(id)
+
+		# Fill row
+		topic.topic_id   = id
+		topic.author     = user
+		topic.type       = info["type"]
+		topic.title      = title
+		topic.name       = name
+		topic.link       = link
+		topic.wip        = info["wip"]
+		topic.posts      = info["posts"]
+		topic.views      = info["views"]
+		topic.created_at = info["date"]
 
 	db.session.commit()

app/tasks/importtasks.py

@@ -372,7 +372,7 @@ def makeVCSRelease(id, branch):
 			filename = randomString(10) + ".zip"
 			destPath = os.path.join("app/public/uploads", filename)
 			with open(destPath, "wb") as fp:
-				repo.archive(fp)
+				repo.archive(fp, format="zip")
 
 			release.url = "/uploads/" + filename
 			print(release.url)

app/tasks/phpbbparser.py

@@ -5,6 +5,7 @@
 import urllib, socket
 from bs4 import *
 from urllib.parse import urljoin
+from datetime import datetime
 import urllib.request
 import os.path
 import time, re
@@ -77,3 +78,72 @@ def getProfile(url, username):
 		__extract_properties(profile, soup)
 
 		return profile
+
+
+regex_id = re.compile(r"^.*t=([0-9]+).*$")
+
+def parseForumListPage(id, page, out, extra=None):
+	num_per_page = 30
+	start = page*num_per_page+1
+	print(" - Fetching page {} (topics {}-{})".format(page, start, start+num_per_page))
+
+	url = "https://forum.minetest.net/viewforum.php?f=" + str(id) + "&start=" + str(start)
+	r = urllib.request.urlopen(url).read().decode("utf-8")
+	soup = BeautifulSoup(r, "html.parser")
+
+	for row in soup.find_all("li", class_="row"):
+		classes = row.get("class")
+		if "sticky" in classes or "announce" in classes or "global-announce" in classes:
+			continue
+
+		topic = row.find("dl")
+
+		# Link info
+		link   = topic.find(class_="topictitle")
+		id	   = regex_id.match(link.get("href")).group(1)
+		title  = link.find(text=True)
+
+		# Date
+		left   = topic.find("dt")
+		date   = left.get_text().split("»")[1].strip()
+		date   = datetime.strptime(date, "%a %b %d, %Y %H:%M")
+		author = left.find_all("a")[-1].get_text().strip()
+
+		# Get counts
+		posts  = topic.find(class_="posts").find(text=True)
+		views  = topic.find(class_="views").find(text=True)
+
+		if id in out:
+			print("   - got {} again, title: {}".format(id, title))
+			assert(title == out[id]['title'])
+			return False
+
+		row = {
+			"id"    : id,
+			"title" : title,
+			"author": author,
+			"posts" : posts,
+			"views" : views,
+			"date"  : date
+		}
+
+		if extra is not None:
+			for key, value in extra.items():
+				row[key] = value
+
+		out[id] = row
+
+	return True
+
+def getTopicsFromForum(id, out={}, extra=None):
+	print("Fetching all topics from forum {}".format(id))
+	page = 0
+	while parseForumListPage(id, page, out, extra):
+		page = page + 1
+
+	return out
+
+def dumpTitlesToFile(topics, path):
+	with open(path, "w") as out_file:
+		for topic in topics.values():
+			out_file.write(topic["title"] + "\n")

app/templates/admin/list.html

@@ -17,11 +17,12 @@
 		<form method="post" action="" class="box-body">
 			<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
 			<select name="action">
-				<option value="importusers">Create users from mod list</option>
-				<option value="importmodlist">Import Krock's mod list</option>
+				<option value="importmodlist">Import forum topics</option>
 				<option value="importscreenshots" selected>Import screenshots from VCS</option>
 				<option value="importdepends">Import dependencies from downloads</option>
 				<option value="modprovides">Set provides to mod name</option>
+				<option value="recalcscores">Recalc package scores</option>
+				<option value="vcsrelease">Create VCS releases</option>
 			</select>
 			<input type="submit" value="Perform" />
 		</form>

app/templates/index.html

@@ -12,7 +12,7 @@ Welcome
 		<p>
 			Minetest's official content repository.
 			Browse {{ count }} packages,
-			all available under a free and open source
+			majority of which available under a free and open source
 			license.
 		</p>
 
@@ -25,6 +25,14 @@ Welcome
 
 <main>
 	{% from "macros/packagegridtile.html" import render_pkggrid %}
-	{{ render_pkggrid(packages) }}
+
+	<h2>Popular</h2>
+	{{ render_pkggrid(popular) }}
+
+	<a href="{{ url_for('packages_page') }}" class="button">Show More</a>
+
+	<h2 style="margin-top:2em;">Newly Added</h2>
+	{{ render_pkggrid(new) }}
+
 </main>
 {% endblock %}

app/templates/macros/threads.html

@@ -29,6 +29,8 @@
 	<ul>
 		{% for t in threads %}
 			<li><a href="{{ url_for('thread_page', id=t.id) }}">{{ t.title }}</a> by {{ t.author.display_name }}</li>
+		{% else %}
+			<li><i>No threads found</i></li>
 		{% endfor %}
 	</ul>
 {% endmacro %}

app/templates/macros/topictable.html

@@ -2,6 +2,7 @@
 <table>
 	<tr>
 		<th>Id</th>
+		<th></th>
 		<th>Title</th>
 		{% if show_author %}<th>Author</th>{% endif %}
 		<th>Name</th>
@@ -9,14 +10,20 @@
 		<th>Actions</th>
 	</tr>
 	{% for topic in topics %}
-		<tr>
+		<tr{% if topic.wip %} class="wiptopic"{% endif %}>
 			<td>{{ topic.topic_id }}</td>
-			<td>[{{ topic.getType().value }}] <a href="https://forum.minetest.net/viewtopic.php?t={{ topic.topic_id}}">{{ topic.title }}</a></td>
+			<td>
+				[{{ topic.type.value }}]
+			</td>
+			<td>
+				<a href="https://forum.minetest.net/viewtopic.php?t={{ topic.topic_id}}">{{ topic.title }}</a>
+				{% if topic.wip %}[WIP]{% endif %}
+			</td>
 			{% if show_author %}
 				<td><a href="{{ url_for('user_profile_page', username=topic.author.username) }}">{{ topic.author.display_name}}</a></td>
 			{% endif %}
 			<td>{{ topic.name or ""}}</td>
-			<td><a href="{{ topic.link }}">{{ topic.link | domain }}</a></td>
+			<td>{% if topic.link %}<a href="{{ topic.link }}">{{ topic.link | domain }}</a>{% endif %}</td>
 			<td>
 				<a href="{{ url_for('create_edit_package_page', author=topic.author.username, repo=topic.getRepoURL(), forums=topic.topic_id, title=topic.title, bname=topic.name) }}">Create</a>
 			</td>

app/templates/packages/view.html

@@ -24,6 +24,9 @@
 			{% elif (package.type == package.type.GAME or package.type == package.type.TXP) and package.screenshots.count() == 0 %}
 				You need to add at least one screenshot.
 
+			{% elif topic_error_lvl == "error" %}
+				Please fix the below topic issue(s).
+
 			{% else %}
 				{% if package.screenshots.count() == 0 %}
 					<b>You should add at least one screenshot, but this isn't required.</b><br />
@@ -44,6 +47,14 @@
 			<div style="clear: both;"></div>
 		</div>
 
+		{% if topic_error %}
+			<div class="box box_grey alert alert-{{ topic_error_lvl }}">
+				<span class="icon_message"></span>
+				{{ topic_error | safe }}
+				<div style="clear: both;"></div>
+			</div>
+		{% endif %}
+
 		{% if package.author == current_user or package.checkPerm(current_user, "APPROVE_NEW") %}
 			{% if review_thread %}
 				<h2>&#x1f512; {{ review_thread.title }}</h2>
@@ -292,10 +303,11 @@
 		<ul>
 			{% for t in similar_topics %}
 				<li>
-					[{{ t.getType().value }}]
+					[{{ t.type.value }}]
 					<a href="https://forum.minetest.net/viewtopic.php?t={{ t.topic_id }}">
 						{{ t.title }} by {{ t.author.display_name }}
 					</a>
+					{% if t.wip %}[WIP]{% endif %}
 				</li>
 			{% endfor %}
 		</ul>

app/templates/todo/list.html

@@ -65,6 +65,6 @@
 	<p>
 		There are
 		<a href="{{ url_for('todo_topics_page') }}">{{ topics_to_add }} packages</a>
-		to be added to cdb, based on forum topics picked up by Krock's mod search.
+		to be added to cdb, based on cdb's forum parser.
 	</p>
 {% endblock %}

app/templates/users/user_profile_page.html

@@ -104,12 +104,9 @@
 
 		<div class="box-body">
 			<p>
-				List of your topics without a matching package.
-				Powered by Krock's Mod Search.
+				List of your forum topics which do not have a matching package.
 			</p>
 
-			
-
 			{% from "macros/topictable.html" import render_topictable %}
 			{{ render_topictable(topics_to_add, show_author=False) }}
 		</div>

app/views/__init__.py

@@ -46,12 +46,14 @@ def send_upload(path):
 @app.route("/")
 @menu.register_menu(app, ".", "Home")
 def home_page():
-	query = Package.query.filter_by(approved=True, soft_deleted=False)
-	count = query.count()
-	packages = query.order_by(db.desc(Package.created_at)).limit(15).all()
-	return render_template("index.html", packages=packages, count=count)
+	query   = Package.query.filter_by(approved=True, soft_deleted=False)
+	count   = query.count()
+	new     = query.order_by(db.desc(Package.created_at)).limit(15).all()
+	popular = query.order_by(db.desc(Package.score)).limit(6).all()
+	return render_template("index.html", new=new, popular=popular, count=count)
 
-from . import users, githublogin, packages, sass, tasks, admin, notifications, tagseditor, meta, thumbnails, threads
+from . import users, githublogin, packages, meta, threads, api
+from . import sass, tasks, admin, notifications, tagseditor, thumbnails
 
 @menu.register_menu(app, ".help", "Help", order=19, endpoint_arguments_constructor=lambda: { 'path': 'help' })
 @app.route('/<path:path>/')

app/views/admin.py

@@ -20,22 +20,21 @@ from flask_user import *
 from flask.ext import menu
 from app import app
 from app.models import *
-from app.tasks.importtasks import importRepoScreenshot, importAllDependencies
-from app.tasks.forumtasks  import importUsersFromModList, importKrocksModList
+from celery import uuid
+from app.tasks.importtasks import importRepoScreenshot, importAllDependencies, makeVCSRelease
+from app.tasks.forumtasks  import importTopicList
 from flask_wtf import FlaskForm
 from wtforms import *
-from app.utils import loginUser, rank_required
+from app.utils import loginUser, rank_required, triggerNotif
+import datetime
 
 @app.route("/admin/", methods=["GET", "POST"])
 @rank_required(UserRank.ADMIN)
 def admin_page():
 	if request.method == "POST":
 		action = request.form["action"]
-		if action == "importusers":
-			task = importUsersFromModList.delay()
-			return redirect(url_for("check_task", id=task.id, r=url_for("user_list_page")))
-		elif action == "importmodlist":
-			task = importKrocksModList.delay()
+		if action == "importmodlist":
+			task = importTopicList.delay()
 			return redirect(url_for("check_task", id=task.id, r=url_for("todo_topics_page")))
 		elif action == "importscreenshots":
 			packages = Package.query \
@@ -67,6 +66,31 @@ def admin_page():
 
 			db.session.commit()
 			return redirect(url_for("admin_page"))
+		elif action == "recalcscores":
+			for p in Package.query.all():
+				p.recalcScore()
+
+			db.session.commit()
+			return redirect(url_for("admin_page"))
+		elif action == "vcsrelease":
+			for package in Package.query.filter(Package.repo.isnot(None)).all():
+				if package.releases.count() != 0:
+					continue
+
+				rel = PackageRelease()
+				rel.package  = package
+				rel.title    = datetime.date.today().isoformat()
+				rel.url      = ""
+				rel.task_id  = uuid()
+				rel.approved = True
+				db.session.add(rel)
+				db.session.commit()
+
+				makeVCSRelease.apply_async((rel.id, "master"), task_id=rel.task_id)
+
+				msg = "{}: Release {} created".format(package.title, rel.title)
+				triggerNotif(package.author, current_user, msg, rel.getEditURL())
+				db.session.commit()
 
 		else:
 			flash("Unknown action: " + action, "error")

app/views/api.py

@@ -0,0 +1,43 @@
+# Content DB
+# Copyright (C) 2018  rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+
+from flask import *
+from flask_user import *
+from app import app
+from app.models import *
+from app.utils import is_package_page
+from .packages import build_packages_query
+
+@app.route("/api/packages/")
+def api_packages_page():
+	query, _ = build_packages_query()
+	pkgs = [package.getAsDictionaryShort(app.config["BASE_URL"]) \
+			for package in query.all() if package.getDownloadRelease() is not None]
+	return jsonify(pkgs)
+
+@app.route("/api/packages/<author>/<name>/")
+@is_package_page
+def api_package_page(package):
+	return jsonify(package.getAsDictionary(app.config["BASE_URL"]))
+
+
+@app.route("/api/topics/")
+def api_topics_page():
+	query = ForumTopic.query \
+			.order_by(db.asc(ForumTopic.wip), db.asc(ForumTopic.name), db.asc(ForumTopic.title))
+	pkgs = [t.getAsDictionary() for t in query.all()]
+	return jsonify(pkgs)

app/views/packages/__init__.py

@@ -30,48 +30,52 @@ from wtforms import *
 from wtforms.validators import *
 from wtforms.ext.sqlalchemy.fields import QuerySelectField, QuerySelectMultipleField
 
-# TODO: the following could be made into one route, except I"m not sure how
-# to do the menu
-
-@menu.register_menu(app, ".mods", "Mods", order=11, endpoint_arguments_constructor=lambda: { 'type': 'mod' })
-@menu.register_menu(app, ".games", "Games", order=12, endpoint_arguments_constructor=lambda: { 'type': 'game' })
-@menu.register_menu(app, ".txp", "Texture Packs", order=13, endpoint_arguments_constructor=lambda: { 'type': 'txp' })
-@app.route("/packages/")
-def packages_page():
+def build_packages_query():
 	type_name = request.args.get("type")
 	type = None
 	if type_name is not None:
 		type = PackageType[type_name.upper()]
 
 	title = "Packages"
-	query = Package.query.filter_by(soft_deleted=False)
+	query = Package.query.filter_by(soft_deleted=False, approved=True)
 
 	if type is not None:
 		title = type.value + "s"
-		query = query.filter_by(type=type, approved=True)
+		query = query.filter_by(type=type)
 
 	search = request.args.get("q")
 	if search is not None and search.strip() != "":
 		query = query.filter(Package.title.ilike('%' + search + '%'))
 
+	query = query.order_by(db.desc(Package.score))
+
+	return query, title
+
+@menu.register_menu(app, ".mods", "Mods", order=11, endpoint_arguments_constructor=lambda: { 'type': 'mod' })
+@menu.register_menu(app, ".games", "Games", order=12, endpoint_arguments_constructor=lambda: { 'type': 'game' })
+@menu.register_menu(app, ".txp", "Texture Packs", order=13, endpoint_arguments_constructor=lambda: { 'type': 'txp' })
+@app.route("/packages/")
+def packages_page():
 	if shouldReturnJson():
-		pkgs = [package.getAsDictionary(app.config["BASE_URL"]) \
-				for package in query.all() if package.getDownloadRelease() is not None]
-		return jsonify(pkgs)
-	else:
-		page  = int(request.args.get("page") or 1)
-		num   = min(42, int(request.args.get("n") or 100))
-		query = query.paginate(page, num, True)
+		return redirect(url_for("api_packages_page"))
 
-		next_url = url_for("packages_page", type=type_name, q=search, page=query.next_num) \
-				if query.has_next else None
-		prev_url = url_for("packages_page", type=type_name, q=search, page=query.prev_num) \
-				if query.has_prev else None
+	query, title = build_packages_query()
+	page  = int(request.args.get("page") or 1)
+	num   = min(42, int(request.args.get("n") or 100))
+	query = query.paginate(page, num, True)
 
-		tags = Tag.query.all()
-		return render_template("packages/list.html", title=title, packages=query.items, \
-				query=search, tags=tags, type=type_name, \
-				next_url=next_url, prev_url=prev_url, page=page, page_max=query.pages, packages_count=query.total)
+	search = request.args.get("q")
+	type_name = request.args.get("type")
+
+	next_url = url_for("packages_page", type=type_name, q=search, page=query.next_num) \
+			if query.has_next else None
+	prev_url = url_for("packages_page", type=type_name, q=search, page=query.prev_num) \
+			if query.has_prev else None
+
+	tags = Tag.query.all()
+	return render_template("packages/list.html", title=title, packages=query.items, \
+			query=search, tags=tags, type=type_name, \
+			next_url=next_url, prev_url=prev_url, page=page, page_max=query.pages, packages_count=query.total)
 
 
 def getReleases(package):
@@ -84,41 +88,59 @@ def getReleases(package):
 @app.route("/packages/<author>/<name>/")
 @is_package_page
 def package_page(package):
-	if shouldReturnJson():
-		return jsonify(package.getAsDictionary(app.config["BASE_URL"]))
-	else:
-		clearNotifications(package.getDetailsURL())
+	clearNotifications(package.getDetailsURL())
 
-		alternatives = None
-		if package.type == PackageType.MOD:
-			alternatives = Package.query \
-					.filter_by(name=package.name, type=PackageType.MOD, soft_deleted=False) \
-					.filter(Package.id != package.id) \
-					.order_by(db.asc(Package.title)) \
-					.all()
+	alternatives = None
+	if package.type == PackageType.MOD:
+		alternatives = Package.query \
+				.filter_by(name=package.name, type=PackageType.MOD, soft_deleted=False) \
+				.filter(Package.id != package.id) \
+				.order_by(db.asc(Package.title)) \
+				.all()
 
-		show_similar_topics = current_user == package.author or \
-				package.checkPerm(current_user, Permission.APPROVE_NEW)
+	show_similar_topics = current_user == package.author or \
+			package.checkPerm(current_user, Permission.APPROVE_NEW)
 
-		similar_topics = None if not show_similar_topics else \
-				KrockForumTopic.query \
-					.filter_by(name=package.name) \
-					.filter(KrockForumTopic.topic_id != package.forums) \
-					.filter(~ db.exists().where(Package.forums==KrockForumTopic.topic_id)) \
-					.order_by(db.asc(KrockForumTopic.name), db.asc(KrockForumTopic.title)) \
-					.all()
+	similar_topics = None if not show_similar_topics else \
+			ForumTopic.query \
+				.filter_by(name=package.name) \
+				.filter(ForumTopic.topic_id != package.forums) \
+				.filter(~ db.exists().where(Package.forums==ForumTopic.topic_id)) \
+				.order_by(db.asc(ForumTopic.name), db.asc(ForumTopic.title)) \
+				.all()
 
-		releases = getReleases(package)
-		requests = [r for r in package.requests if r.status == 0]
+	releases = getReleases(package)
+	requests = [r for r in package.requests if r.status == 0]
 
-		review_thread = Thread.query.filter_by(package_id=package.id, private=True).first()
-		if review_thread is not None and not review_thread.checkPerm(current_user, Permission.SEE_THREAD):
-			review_thread = None
+	review_thread = Thread.query.filter_by(package_id=package.id, private=True).first()
+	if review_thread is not None and not review_thread.checkPerm(current_user, Permission.SEE_THREAD):
+		review_thread = None
 
-		return render_template("packages/view.html", \
-				package=package, releases=releases, requests=requests, \
-				alternatives=alternatives, similar_topics=similar_topics, \
-				review_thread=review_thread)
+	topic_error = None
+	topic_error_lvl = "warning"
+	if not package.approved and package.forums is not None:
+		errors = []
+		if Package.query.filter_by(forums=package.forums, soft_deleted=False).count() > 1:
+			errors.append("<b>Error: Another package already uses this forum topic!</b>")
+			topic_error_lvl = "error"
+
+		topic = ForumTopic.query.get(package.forums)
+		if topic is not None:
+			if topic.author != package.author:
+				errors.append("<b>Error: Forum topic author doesn't match package author.</b>")
+				topic_error_lvl = "error"
+
+			if topic.wip:
+				errors.append("Warning: Forum topic is in WIP section, make sure package meets playability standards.")
+		elif package.type != PackageType.TXP:
+			errors.append("Warning: Forum topic not found. This may happen if the topic has only just been created.")
+
+		topic_error = "<br />".join(errors)
+
+	return render_template("packages/view.html", \
+			package=package, releases=releases, requests=requests, \
+			alternatives=alternatives, similar_topics=similar_topics, \
+			review_thread=review_thread, topic_error=topic_error, topic_error_lvl=topic_error_lvl)
 
 
 @app.route("/packages/<author>/<name>/download/")

app/views/packages/todo.py

@@ -41,8 +41,8 @@ def todo_page():
 		screenshots = PackageScreenshot.query.filter_by(approved=False).all()
 
 
-	topics_to_add = KrockForumTopic.query \
-			.filter(~ db.exists().where(Package.forums==KrockForumTopic.topic_id)) \
+	topics_to_add = ForumTopic.query \
+			.filter(~ db.exists().where(Package.forums==ForumTopic.topic_id)) \
 			.count()
 
 	return render_template("todo/list.html", title="Reports and Work Queue",
@@ -54,11 +54,11 @@ def todo_page():
 @app.route("/todo/topics/")
 @login_required
 def todo_topics_page():
-	total = KrockForumTopic.query.count()
+	total = ForumTopic.query.count()
 
-	topics = KrockForumTopic.query \
-			.filter(~ db.exists().where(Package.forums==KrockForumTopic.topic_id)) \
-			.order_by(db.asc(KrockForumTopic.name), db.asc(KrockForumTopic.title)) \
+	topics = ForumTopic.query \
+			.filter(~ db.exists().where(Package.forums==ForumTopic.topic_id)) \
+			.order_by(db.asc(ForumTopic.wip), db.asc(ForumTopic.name), db.asc(ForumTopic.title)) \
 			.all()
 
 	return render_template("todo/topics.html", topics=topics, total=total)

app/views/threads.py

@@ -27,8 +27,10 @@ from wtforms.validators import *
 
 @app.route("/threads/")
 def threads_page():
-	threads = Thread.query.filter_by(private=False).all()
-	return render_template("threads/list.html", threads=threads)
+	query = Thread.query
+	if not Permission.SEE_THREAD.check(current_user):
+		query = query.filter_by(private=False)
+	return render_template("threads/list.html", threads=query.all())
 
 @app.route("/threads/<int:id>/", methods=["GET", "POST"])
 def thread_page(id):

app/views/users.py

@@ -98,10 +98,10 @@ def user_profile_page(username):
 
 	topics_to_add = None
 	if current_user == user or user.checkPerm(current_user, Permission.CHANGE_AUTHOR):
-		topics_to_add = KrockForumTopic.query \
+		topics_to_add = ForumTopic.query \
 					.filter_by(author_id=user.id) \
-					.filter(~ db.exists().where(Package.forums==KrockForumTopic.topic_id)) \
-					.order_by(db.asc(KrockForumTopic.name), db.asc(KrockForumTopic.title)) \
+					.filter(~ db.exists().where(Package.forums==ForumTopic.topic_id)) \
+					.order_by(db.asc(ForumTopic.name), db.asc(ForumTopic.title)) \
 					.all()
 
 	# Process GET or invalid POST

config.example.cfg

@@ -22,3 +22,4 @@ MAIL_DEFAULT_SENDER=""
 MAIL_SERVER=""
 MAIL_PORT=587
 MAIL_USE_TLS=True
+MAIL_UTILS_ERROR_SEND_TO=[""]

migrations/versions/11b6ef362f98_.py

@@ -0,0 +1,28 @@
+"""empty message
+
+Revision ID: 11b6ef362f98
+Revises: 9fc23495713b
+Create Date: 2018-07-04 01:01:45.440662
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '11b6ef362f98'
+down_revision = '9fc23495713b'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('package', sa.Column('score', sa.Float(), nullable=False, server_default="0.0"))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('package', 'score')
+    # ### end Alembic commands ###

migrations/versions/9e2ac631efb0_.py

@@ -0,0 +1,28 @@
+"""empty message
+
+Revision ID: 9e2ac631efb0
+Revises: 11b6ef362f98
+Create Date: 2018-07-06 23:16:50.507010
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '9e2ac631efb0'
+down_revision = '11b6ef362f98'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('forum_topic', sa.Column('wip', sa.Boolean(), nullable=False, server_default="0"))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('forum_topic', 'wip')
+    # ### end Alembic commands ###

migrations/versions/9fc23495713b_.py

@@ -0,0 +1,55 @@
+"""empty message
+
+Revision ID: 9fc23495713b
+Revises: de004661c5e1
+Create Date: 2018-07-04 00:03:20.123285
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '9fc23495713b'
+down_revision = 'de004661c5e1'
+branch_labels = None
+depends_on = None
+from sqlalchemy.dialects.postgresql import ENUM
+
+type_enum = ENUM('MOD', 'GAME', 'TXP', name='packagetype', create_type=False)
+
+def upgrade():
+    type_enum.create(op.get_bind(), checkfirst=True)
+
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('krock_forum_topic')
+    op.create_table('forum_topic',
+    sa.Column('topic_id', sa.Integer(), autoincrement=False, nullable=False),
+    sa.Column('author_id', sa.Integer(), nullable=False),
+    sa.Column('type', type_enum, nullable=True),
+    sa.Column('title', sa.String(length=200), nullable=False),
+    sa.Column('name', sa.String(length=30), nullable=True),
+    sa.Column('link', sa.String(length=200), nullable=True),
+    sa.Column('posts', sa.Integer(), nullable=False),
+    sa.Column('views', sa.Integer(), nullable=False),
+    sa.Column('created_at', sa.DateTime(), nullable=False),
+    sa.ForeignKeyConstraint(['author_id'], ['user.id'], ),
+    sa.PrimaryKeyConstraint('topic_id')
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('forum_topic')
+    op.create_table('krock_forum_topic',
+    sa.Column('topic_id', sa.Integer(), autoincrement=False, nullable=False),
+    sa.Column('author_id', sa.Integer(), nullable=False),
+    sa.Column('ttype', sa.Integer(), nullable=False),
+    sa.Column('title', sa.String(length=200), nullable=False),
+    sa.Column('name', sa.String(length=30), nullable=True),
+    sa.Column('link', sa.String(length=50), nullable=True),
+    sa.ForeignKeyConstraint(['author_id'], ['user.id'], ),
+    sa.PrimaryKeyConstraint('topic_id')
+    )
+    # ### end Alembic commands ###