Document top packages algorithm

Fixes #190

README.md

@@ -1,7 +1,7 @@
 # Content Database
+[![Build status](https://gitlab.com/minetest/contentdb/badges/master/pipeline.svg)](https://gitlab.com/minetest/contentdb/pipelines)
 
-Content database for Minetest mods, games, and more.
-
+Content database for Minetest mods, games, and more.\
 Developed by rubenwardy, license GPLv3.0+.
 
 ## How-tos

app/blueprints/api/endpoints.py

@@ -37,6 +37,16 @@ def packages():
 	return jsonify(pkgs)
 
 
+@bp.route("/api/scores/")
+def package_scores():
+	qb    = QueryBuilder(request.args)
+	query = qb.buildPackageQuery()
+
+	pkgs = [{ "author": package.author.username, "name": package.name, "score": package.score } \
+			for package in query.all()]
+	return jsonify(pkgs)
+
+
 @bp.route("/api/packages/<author>/<name>/")
 @is_package_page
 def package(package):

app/blueprints/api/tokens.py

@@ -69,7 +69,7 @@ def create_edit_token(username, id=None):
 		elif token.owner != user:
 			abort(403)
 
-		access_token = session.pop("token_" + str(id), None)
+		access_token = session.pop("token_" + str(token.id), None)
 
 	form = CreateAPIToken(formdata=request.form, obj=token)
 	form.package.query_factory = lambda: Package.query.filter_by(author=user).all()
@@ -80,13 +80,14 @@ def create_edit_token(username, id=None):
 			token.owner = user
 			token.access_token = randomString(32)
 
-			# Store token so it can be shown in the edit page
-			session["token_" + str(token.id)] = token.access_token
-
 		form.populate_obj(token)
 		db.session.add(token)
 		db.session.commit() # save
 
+		if is_new:
+			# Store token so it can be shown in the edit page
+			session["token_" + str(token.id)] = token.access_token
+
 		return redirect(url_for("api.create_edit_token", username=username, id=token.id))
 
 	return render_template("api/create_edit_token.html", user=user, form=form, token=token, access_token=access_token)
@@ -102,8 +103,6 @@ def reset_token(username, id):
 	if not user.checkPerm(current_user, Permission.CREATE_TOKEN):
 		abort(403)
 
-	is_new = id is None
-
 	token = APIToken.query.get(id)
 	if token is None:
 		abort(404)

app/blueprints/github/__init__.py

@@ -20,7 +20,7 @@ bp = Blueprint("github", __name__)
 
 from flask import redirect, url_for, request, flash, abort, render_template, jsonify, current_app
 from flask_user import current_user, login_required
-from sqlalchemy import func
+from sqlalchemy import func, or_, and_
 from flask_github import GitHub
 from app import github, csrf
 from app.models import db, User, APIToken, Package, Permission
@@ -92,10 +92,13 @@ def webhook():
 	github_url = "github.com/" + json["repository"]["full_name"]
 	package = Package.query.filter(Package.repo.like("%{}%".format(github_url))).first()
 	if package is None:
-		return error(400, "Unknown package")
+		return error(400, "Could not find package, did you set the VCS repo in CDB correctly?")
 
 	# Get all tokens for package
-	possible_tokens = APIToken.query.filter_by(package=package).all()
+	tokens_query = APIToken.query.filter(or_(APIToken.package==package,
+			and_(APIToken.package==None, APIToken.owner==package.author)))
+
+	possible_tokens = tokens_query.all()
 	actual_token = None
 
 	#
@@ -118,7 +121,7 @@ def webhook():
 			break
 
 	if actual_token is None:
-		return error(403, "Invalid authentication")
+		return error(403, "Invalid authentication, couldn't validate API token")
 
 	if not package.checkPerm(actual_token.owner, Permission.APPROVE_RELEASE):
 		return error(403, "Only trusted members can use webhooks")

app/blueprints/users/claim.py

@@ -34,15 +34,16 @@ def claim():
 		if user and user.rank.atLeast(UserRank.NEW_MEMBER):
 			flash("User has already been claimed", "danger")
 			return redirect(url_for("users.claim"))
-		elif user is None and method == "github":
-			flash("Unable to get Github username for user", "danger")
-			return redirect(url_for("users.claim"))
-		elif user is None:
-			flash("Unable to find that user", "danger")
+		elif method == "github":
+			if user is None or user.github_username is None:
+				flash("Unable to get Github username for user", "danger")
+				return redirect(url_for("users.claim"))
+			else:
+				return redirect(url_for("github.start"))
+		elif user is None and request.method == "POST":
+			flash("Unable to find user", "danger")
 			return redirect(url_for("users.claim"))
 
-		if user is not None and method == "github":
-			return redirect(url_for("github.start"))
 
 	token = None
 	if "forum_token" in session:
@@ -70,8 +71,17 @@ def claim():
 			sig = None
 			try:
 				profile = getProfile("https://forum.minetest.net", username)
-				sig = profile.signature
-			except IOError:
+				sig = profile.signature if profile else None
+			except IOError as e:
+				if hasattr(e, 'message'):
+					message = e.message
+				else:
+					message = str(e)
+
+				flash("Error whilst attempting to access forums: " + message, "danger")
+				return redirect(url_for("users.claim", username=username))
+
+			if profile is None:
 				flash("Unable to get forum signature - does the user exist?", "danger")
 				return redirect(url_for("users.claim", username=username))
 

app/blueprints/users/profile.py

@@ -16,7 +16,7 @@
 
 
 from flask import *
-from flask_user import *
+from flask_user import signals, current_user, user_manager
 from flask_login import login_user, logout_user
 from app.markdown import render_markdown
 from . import bp
@@ -192,7 +192,7 @@ def set_password():
 
 			# Send 'password_changed' email
 			if user_manager.USER_ENABLE_EMAIL and current_user.email:
-				emails.send_password_changed_email(current_user)
+				user_manager.email_manager.send_password_changed_email(current_user)
 
 			# Send password_changed signal
 			signals.user_changed_password.send(current_app._get_current_object(), user=current_user)

app/flatpages/help.md

@@ -4,5 +4,6 @@ title: Help
 * [Ranks and Permissions](ranks_permissions)
 * [Content Ratings and Flags](content_flags)
 * [Reporting Content](reporting)
+* [Top Packages Algorithm](top_packages)
 * [API](api)
 * [Creating Releases using Webhooks](release_webhooks)

app/flatpages/help/api.md

@@ -21,6 +21,7 @@ You can use the `/api/whoami` to check authentication.
 ### Packages
 
 * GET `/api/packages/` - See [Package Queries](#package-queries)
+* GET `/api/scores/` - See [Package Queries](#package-queries)
 * GET `/api/packages/<username>/<name>/`
 
 ### Releases

app/flatpages/help/top_packages.md

@@ -0,0 +1,39 @@
+title: Top Packages Algorithm
+
+## Pseudo rolling average
+
+Every package loses 5% of its score every day.
+
+An open source package will gain 1 score for each unique download,
+whereas a non-free package will only gain 0.1 score.
+
+A package currently only gains score through downloads.
+In the future, a package will also gain score through reviews.
+
+## Seed using a legacy heuristic
+
+The scoring system was seeded (ie: the scores were initially set to) 20% of an
+arbitrary legacy heuristic that was previously used to rank packages.
+
+This legacy heuristic is as follows:
+
+	forum_score = views / max(years_since_creation, 2 weeks) + 80*clamp(months, 0.5, 6)
+	forum_bonus = views + posts
+
+	multiplier = 1
+	if no screenshot:
+		multiplier *= 0.8
+	if not foss:
+		multiplier *= 0.1
+
+	score = multiplier * (max(downloads, forum_score * 0.6) + forum_bonus)
+
+As said, this legacy score is no longer used when ranking mods.
+It was only used to provide an initial score for the rolling average,
+which was 20% of the above value.
+
+## Transparency and Feedback
+
+You can see all scores using the [scores REST API](/api/scores/).
+
+Consider [suggesting improvements](https://github.com/minetest/contentdb/issues/new?assignees=&labels=Policy&template=policy.md&title=).

app/maillogger.py

@@ -41,7 +41,7 @@ class FlaskMailHTMLFormatter(logging.Formatter):
 		formatted_exception = logging.Handler.formatException(self, exc_info)
 		return FlaskMailHTMLFormatter.pre_template % ("Exception information", formatted_exception)
 	def formatStack(self, stack_info):
-		return FlaskMailHTMLFormatter.pre_template % ("<h1>Stack information</h1><pre>%s</pre>", stack_info)
+		return FlaskMailHTMLFormatter.pre_template % ("<h1>Stack information</h1><pre><code>%s</code></pre>", stack_info)
 
 
 # see: https://github.com/python/cpython/blob/3.6/Lib/logging/__init__.py (class Handler)
@@ -100,7 +100,7 @@ Message:
 <tr>		<th>Time:</th><td>%(asctime)s</td></tr>
 </table>
 <h2>Message</h2>
-<pre>%(message)s</pre>"""
+<pre><code>%(message)s</code></pre>"""
 
 	import logging
 	mail_handler = FlaskMailHandler(mailer, subject_template)

app/models.py

@@ -214,6 +214,9 @@ class User(db.Model, UserMixin):
 			.filter(Thread.created_at > hour_ago).count() < 2
 
 	def __eq__(self, other):
+		if other is None:
+			return False
+
 		if not self.is_authenticated or not other.is_authenticated:
 			return False
 
@@ -656,7 +659,7 @@ class Package(db.Model):
 		if perm == Permission.MAKE_RELEASE or perm == Permission.ADD_SCREENSHOTS:
 			return isOwner or user.rank.atLeast(UserRank.EDITOR)
 
-		if perm == Permission.EDIT_PACKAGE or perm == Permission.APPROVE_CHANGES:
+		if perm == Permission.EDIT_PACKAGE or perm == Permission.APPROVE_CHANGES or perm == Permission.APPROVE_RELEASE:
 			if isOwner:
 				return user.rank.atLeast(UserRank.MEMBER if self.approved else UserRank.NEW_MEMBER)
 			else:
@@ -670,7 +673,7 @@ class Package(db.Model):
 		elif perm == Permission.APPROVE_NEW or perm == Permission.CHANGE_AUTHOR:
 			return user.rank.atLeast(UserRank.EDITOR)
 
-		elif perm == Permission.APPROVE_RELEASE or perm == Permission.APPROVE_SCREENSHOT:
+		elif perm == Permission.APPROVE_SCREENSHOT:
 			return user.rank.atLeast(UserRank.TRUSTED_MEMBER if isOwner else UserRank.EDITOR)
 
 		# Moderators can delete packages

app/tasks/phpbbparser.py

@@ -74,7 +74,14 @@ def __extract_signature(soup):
 def getProfile(url, username):
 	url = url + "/memberlist.php?mode=viewprofile&un=" + urlEncodeNonAscii(username)
 
-	contents = urllib.request.urlopen(url).read().decode("utf-8")
+	req = urllib.request.urlopen(url, timeout=5)
+	if req.getcode() == 404:
+		return None
+
+	if req.getcode() != 200:
+		raise IOError(req.getcode())
+
+	contents = req.read().decode("utf-8")
 	soup = BeautifulSoup(contents, "lxml")
 	if soup is None:
 		return None