Fix API auth crash and add more error messages

Fixes #190

README.md

@@ -1,7 +1,7 @@
 # Content Database
+[![Build status](https://gitlab.com/minetest/contentdb/badges/master/pipeline.svg)](https://gitlab.com/minetest/contentdb/pipelines)
 
-Content database for Minetest mods, games, and more.
-
+Content database for Minetest mods, games, and more.\
 Developed by rubenwardy, license GPLv3.0+.
 
 ## How-tos

app/__init__.py

@@ -60,7 +60,6 @@ if not app.debug and app.config["MAIL_UTILS_ERROR_SEND_TO"]:
 from .markdown import init_app
 init_app(app)
 
-
 # @babel.localeselector
 # def get_locale():
 # 	return request.accept_languages.best_match(app.config['LANGUAGES'].keys())

app/blueprints/api/auth.py

@@ -16,6 +16,7 @@
 
 from flask import request, make_response, jsonify, abort
 from app.models import APIToken
+from .support import error
 from functools import wraps
 
 def is_api_authd(f):
@@ -29,13 +30,13 @@ def is_api_authd(f):
 		elif value[0:7].lower() == "bearer ":
 			access_token = value[7:]
 			if len(access_token) < 10:
-				abort(400)
+				error(400, "API token is too short")
 
 			token = APIToken.query.filter_by(access_token=access_token).first()
 			if token is None:
-				abort(403)
+				error(403, "Unknown API token")
 		else:
-			abort(403)
+			abort(403, "Unsupported authentication method")
 
 		return f(token=token, *args, **kwargs)
 

app/blueprints/api/endpoints.py

@@ -28,6 +28,9 @@ from app.querybuilder import QueryBuilder
 
 @bp.route("/api/packages/")
 def packages():
+	import sys
+	print("\n\n############", file=sys.stderr)
+
 	qb    = QueryBuilder(request.args)
 	query = qb.buildPackageQuery()
 	ver   = qb.getMinetestVersion()
@@ -37,6 +40,16 @@ def packages():
 	return jsonify(pkgs)
 
 
+@bp.route("/api/scores/")
+def package_scores():
+	qb    = QueryBuilder(request.args)
+	query = qb.buildPackageQuery()
+
+	pkgs = [{ "author": package.author.username, "name": package.name, "score": package.score } \
+			for package in query.all()]
+	return jsonify(pkgs)
+
+
 @bp.route("/api/packages/<author>/<name>/")
 @is_package_page
 def package(package):
@@ -130,16 +143,21 @@ def markdown():
 @is_package_page
 @is_api_authd
 def create_release(token, package):
+	if not token:
+		error(401, "Authentication needed")
+
+	if not package.checkPerm(token.owner, Permission.APPROVE_RELEASE):
+		error(403, "You do not have the permission to approve releases")
+
 	json = request.json
 	if json is None:
-		return error(400, "JSON post data is required")
+		error(400, "JSON post data is required")
 
 	for option in ["method", "title", "ref"]:
 		if json.get(option) is None:
-			return error(400, option + " is required in the POST data")
-
+			error(400, option + " is required in the POST data")
 
 	if json["method"].lower() != "git":
-		return error(400, "Release-creation methods other than git are not supported")
+		error(400, "Release-creation methods other than git are not supported")
 
 	return handleCreateRelease(token, package, json["title"], json["ref"])

app/blueprints/api/support.py

@@ -1,12 +1,12 @@
 from app.models import PackageRelease, db, Permission
 from app.tasks.importtasks import makeVCSRelease
 from celery import uuid
-from flask import jsonify, make_response, url_for
+from flask import jsonify, abort, url_for
 import datetime
 
 
 def error(status, message):
-	return make_response(jsonify({ "success": False, "error": message }), status)
+	abort(status, jsonify({ "success": False, "error": message }))
 
 
 def handleCreateRelease(token, package, title, ref):

app/blueprints/api/tokens.py

@@ -69,7 +69,7 @@ def create_edit_token(username, id=None):
 		elif token.owner != user:
 			abort(403)
 
-		access_token = session.pop("token_" + str(id), None)
+		access_token = session.pop("token_" + str(token.id), None)
 
 	form = CreateAPIToken(formdata=request.form, obj=token)
 	form.package.query_factory = lambda: Package.query.filter_by(author=user).all()
@@ -80,13 +80,14 @@ def create_edit_token(username, id=None):
 			token.owner = user
 			token.access_token = randomString(32)
 
-			# Store token so it can be shown in the edit page
-			session["token_" + str(token.id)] = token.access_token
-
 		form.populate_obj(token)
 		db.session.add(token)
 		db.session.commit() # save
 
+		if is_new:
+			# Store token so it can be shown in the edit page
+			session["token_" + str(token.id)] = token.access_token
+
 		return redirect(url_for("api.create_edit_token", username=username, id=token.id))
 
 	return render_template("api/create_edit_token.html", user=user, form=form, token=token, access_token=access_token)
@@ -102,8 +103,6 @@ def reset_token(username, id):
 	if not user.checkPerm(current_user, Permission.CREATE_TOKEN):
 		abort(403)
 
-	is_new = id is None
-
 	token = APIToken.query.get(id)
 	if token is None:
 		abort(404)

app/blueprints/github/__init__.py

@@ -20,7 +20,7 @@ bp = Blueprint("github", __name__)
 
 from flask import redirect, url_for, request, flash, abort, render_template, jsonify, current_app
 from flask_user import current_user, login_required
-from sqlalchemy import func
+from sqlalchemy import func, or_, and_
 from flask_github import GitHub
 from app import github, csrf
 from app.models import db, User, APIToken, Package, Permission
@@ -92,10 +92,13 @@ def webhook():
 	github_url = "github.com/" + json["repository"]["full_name"]
 	package = Package.query.filter(Package.repo.like("%{}%".format(github_url))).first()
 	if package is None:
-		return error(400, "Unknown package")
+		return error(400, "Could not find package, did you set the VCS repo in CDB correctly?")
 
 	# Get all tokens for package
-	possible_tokens = APIToken.query.filter_by(package=package).all()
+	tokens_query = APIToken.query.filter(or_(APIToken.package==package,
+			and_(APIToken.package==None, APIToken.owner==package.author)))
+
+	possible_tokens = tokens_query.all()
 	actual_token = None
 
 	#
@@ -118,10 +121,10 @@ def webhook():
 			break
 
 	if actual_token is None:
-		return error(403, "Invalid authentication")
+		return error(403, "Invalid authentication, couldn't validate API token")
 
 	if not package.checkPerm(actual_token.owner, Permission.APPROVE_RELEASE):
-		return error(403, "Only trusted members can use webhooks")
+		return error(403, "You do not have the permission to approve releases")
 
 	#
 	# Check event

app/blueprints/gitlab/__init__.py

@@ -44,7 +44,7 @@ def webhook():
 		return error(403, "Invalid authentication")
 
 	if not package.checkPerm(token.owner, Permission.APPROVE_RELEASE):
-		return error(403, "Only trusted members can use webhooks")
+		return error(403, "You do not have the permission to approve releases")
 
 	#
 	# Check event

app/blueprints/homepage/__init__.py

@@ -4,17 +4,23 @@ bp = Blueprint("homepage", __name__)
 
 from app.models import *
 import flask_menu as menu
+from sqlalchemy.orm import joinedload
 from sqlalchemy.sql.expression import func
 
 @bp.route("/")
 @menu.register_menu(bp, ".", "Home")
 def home():
+	def join(query):
+		return query.options( \
+			joinedload(Package.license), \
+			joinedload(Package.media_license))
+
 	query   = Package.query.filter_by(approved=True, soft_deleted=False)
 	count   = query.count()
-	new     = query.order_by(db.desc(Package.created_at)).limit(8).all()
-	pop_mod = query.filter_by(type=PackageType.MOD).order_by(db.desc(Package.score)).limit(8).all()
-	pop_gam = query.filter_by(type=PackageType.GAME).order_by(db.desc(Package.score)).limit(4).all()
-	pop_txp = query.filter_by(type=PackageType.TXP).order_by(db.desc(Package.score)).limit(4).all()
+	new     = join(query.order_by(db.desc(Package.created_at))).limit(8).all()
+	pop_mod = join(query.filter_by(type=PackageType.MOD).order_by(db.desc(Package.score))).limit(8).all()
+	pop_gam = join(query.filter_by(type=PackageType.GAME).order_by(db.desc(Package.score))).limit(4).all()
+	pop_txp = join(query.filter_by(type=PackageType.TXP).order_by(db.desc(Package.score))).limit(4).all()
 	downloads_result = db.session.query(func.sum(PackageRelease.downloads)).one_or_none()
 	downloads = 0 if not downloads_result or not downloads_result[0] else downloads_result[0]
 	return render_template("index.html", count=count, downloads=downloads, \

app/blueprints/metrics/__init__.py

@@ -0,0 +1,74 @@
+# Content DB
+# Copyright (C) 2020  rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+from flask import Blueprint, make_response
+from app.models import Package, PackageRelease, db, User, UserRank
+from sqlalchemy.sql.expression import func
+
+bp = Blueprint("metrics", __name__)
+
+def generate_metrics(full=False):
+	def write_single_stat(name, help, type, value):
+		fmt = "# HELP {name} {help}\n# TYPE {name} {type}\n{name} {value}\n\n"
+
+		return fmt.format(name=name, help=help, type=type, value=value)
+
+	def gen_labels(labels):
+		pieces = [key + "=" + str(val) for key, val in labels.items()]
+		return (",").join(pieces)
+
+
+	def write_array_stat(name, help, type, data):
+		ret = ("# HELP {name} {help}\n# TYPE {name} {type}\n") \
+				.format(name=name, help=help, type=type)
+
+		for entry in data:
+			assert(len(entry) == 2)
+			ret += ("{name}{{{labels}}} {value}\n") \
+					.format(name=name, labels=gen_labels(entry[0]), value=entry[1])
+
+		return ret + "\n"
+
+
+	downloads_result = db.session.query(func.sum(PackageRelease.downloads)).one_or_none()
+	downloads = 0 if not downloads_result or not downloads_result[0] else downloads_result[0]
+
+	packages = Package.query.filter_by(approved=True, soft_deleted=False).count()
+	users = User.query.filter(User.rank != UserRank.NOT_JOINED).count()
+
+	ret = ""
+	ret += write_single_stat("contentdb_packages", "Total packages", "counter", packages)
+	ret += write_single_stat("contentdb_users", "Number of registered users", "counter", users)
+	ret += write_single_stat("contentdb_downloads", "Total downloads", "counter", downloads)
+
+	if full:
+		scores = Package.query.join(User).with_entities(User.username, Package.name, Package.score) \
+			.filter(Package.approved==True, Package.soft_deleted==False).all()
+
+		ret += write_array_stat("contentdb_package_score", "Package score", "gauge", \
+			[({ "author": score[0], "name": score[1] }, score[2])  for score in scores])
+	else:
+		score_result = db.session.query(func.sum(Package.score)).one_or_none()
+		score = 0 if not score_result or not score_result[0] else score_result[0]
+		ret += write_single_stat("contentdb_score", "Total package score", "gauge", score)
+
+	return ret
+
+@bp.route("/metrics")
+def metrics():
+    response = make_response(generate_metrics(), 200)
+    response.mimetype = "text/plain"
+    return response

app/blueprints/packages/packages.py

@@ -30,7 +30,8 @@ from flask_wtf import FlaskForm
 from wtforms import *
 from wtforms.validators import *
 from wtforms.ext.sqlalchemy.fields import QuerySelectField, QuerySelectMultipleField
-from sqlalchemy import or_
+from sqlalchemy import or_, func
+from sqlalchemy.orm import joinedload, subqueryload
 
 
 @menu.register_menu(bp, ".mods", "Mods", order=11, endpoint_arguments_constructor=lambda: { 'type': 'mod' })
@@ -43,6 +44,11 @@ def list_all():
 	query = qb.buildPackageQuery()
 	title = qb.title
 
+	query = query.options( \
+			joinedload(Package.license), \
+			joinedload(Package.media_license), \
+			subqueryload(Package.tags))
+
 	if qb.lucky:
 		package = query.first()
 		if package:
@@ -64,6 +70,14 @@ def list_all():
 	prev_url = url_for("packages.list_all", type=type_name, q=search, page=query.prev_num) \
 			if query.has_prev else None
 
+	authors = []
+	if search:
+		authors = User.query \
+			.filter(or_(*[func.lower(User.username) == name.lower().strip() for name in search.split(" ")])) \
+			.all()
+
+		authors = [(author.username, search.lower().replace(author.username.lower(), "")) for author in authors]
+
 	topics = None
 	if qb.search and not query.has_next:
 		qb.show_discarded = True
@@ -73,6 +87,7 @@ def list_all():
 	return render_template("packages/list.html", \
 			title=title, packages=query.items, topics=topics, \
 			query=search, tags=tags, type=type_name, \
+			authors = authors, \
 			next_url=next_url, prev_url=prev_url, page=page, page_max=query.pages, packages_count=query.total)
 
 
@@ -164,22 +179,17 @@ def download(package):
 			flash("No download available.", "danger")
 			return redirect(package.getDetailsURL())
 	else:
-		PackageRelease.query.filter_by(id=release.id).update({
-				"downloads": PackageRelease.downloads + 1
-			})
-		db.session.commit()
-
-		return redirect(release.url, code=302)
+		return redirect(release.getDownloadURL(), code=302)
 
 
 class PackageForm(FlaskForm):
-	name          = StringField("Name (Technical)", [InputRequired(), Length(1, 20), Regexp("^[a-z0-9_]+$", 0, "Lower case letters (a-z), digits (0-9), and underscores (_) only")])
-	title         = StringField("Title (Human-readable)", [InputRequired(), Length(3, 50)])
+	name          = StringField("Name (Technical)", [InputRequired(), Length(1, 100), Regexp("^[a-z0-9_]+$", 0, "Lower case letters (a-z), digits (0-9), and underscores (_) only")])
+	title         = StringField("Title (Human-readable)", [InputRequired(), Length(3, 100)])
 	short_desc     = StringField("Short Description (Plaintext)", [InputRequired(), Length(1,200)])
 	desc          = TextAreaField("Long Description (Markdown)", [Optional(), Length(0,10000)])
 	type          = SelectField("Type", [InputRequired()], choices=PackageType.choices(), coerce=PackageType.coerce, default=PackageType.MOD)
-	license       = QuerySelectField("License", [InputRequired()], query_factory=lambda: License.query.order_by(db.asc(License.name)), get_pk=lambda a: a.id, get_label=lambda a: a.name)
-	media_license = QuerySelectField("Media License", [InputRequired()], query_factory=lambda: License.query.order_by(db.asc(License.name)), get_pk=lambda a: a.id, get_label=lambda a: a.name)
+	license       = QuerySelectField("License", [DataRequired()], allow_blank=True, query_factory=lambda: License.query.order_by(db.asc(License.name)), get_pk=lambda a: a.id, get_label=lambda a: a.name)
+	media_license = QuerySelectField("Media License", [DataRequired()], allow_blank=True, query_factory=lambda: License.query.order_by(db.asc(License.name)), get_pk=lambda a: a.id, get_label=lambda a: a.name)
 	provides_str  = StringField("Provides (mods included in package)", [Optional()])
 	tags          = QuerySelectMultipleField('Tags', query_factory=lambda: Tag.query.order_by(db.asc(Tag.name)), get_pk=lambda a: a.id, get_label=lambda a: a.title)
 	harddep_str   = StringField("Hard Dependencies", [Optional()])
@@ -227,6 +237,8 @@ def create_edit(author=None, name=None):
 			form.title.data  = request.args.get("title")
 			form.repo.data   = request.args.get("repo")
 			form.forums.data = request.args.get("forums")
+			form.license.data = None
+			form.media_license.data = None
 		else:
 			form.harddep_str.data  = ",".join([str(x) for x in package.getSortedHardDependencies() ])
 			form.softdep_str.data  = ",".join([str(x) for x in package.getSortedOptionalDependencies() ])

app/blueprints/users/claim.py

@@ -34,15 +34,16 @@ def claim():
 		if user and user.rank.atLeast(UserRank.NEW_MEMBER):
 			flash("User has already been claimed", "danger")
 			return redirect(url_for("users.claim"))
-		elif user is None and method == "github":
-			flash("Unable to get Github username for user", "danger")
-			return redirect(url_for("users.claim"))
-		elif user is None:
-			flash("Unable to find that user", "danger")
+		elif method == "github":
+			if user is None or user.github_username is None:
+				flash("Unable to get Github username for user", "danger")
+				return redirect(url_for("users.claim"))
+			else:
+				return redirect(url_for("github.start"))
+		elif user is None and request.method == "POST":
+			flash("Unable to find user", "danger")
 			return redirect(url_for("users.claim"))
 
-		if user is not None and method == "github":
-			return redirect(url_for("github.start"))
 
 	token = None
 	if "forum_token" in session:
@@ -70,8 +71,17 @@ def claim():
 			sig = None
 			try:
 				profile = getProfile("https://forum.minetest.net", username)
-				sig = profile.signature
-			except IOError:
+				sig = profile.signature if profile else None
+			except IOError as e:
+				if hasattr(e, 'message'):
+					message = e.message
+				else:
+					message = str(e)
+
+				flash("Error whilst attempting to access forums: " + message, "danger")
+				return redirect(url_for("users.claim", username=username))
+
+			if profile is None:
 				flash("Unable to get forum signature - does the user exist?", "danger")
 				return redirect(url_for("users.claim", username=username))
 

app/blueprints/users/profile.py

@@ -16,7 +16,7 @@
 
 
 from flask import *
-from flask_user import *
+from flask_user import signals, current_user, user_manager
 from flask_login import login_user, logout_user
 from app.markdown import render_markdown
 from . import bp
@@ -63,6 +63,8 @@ def profile(username):
 			# Copy form fields to user_profile fields
 			if user.checkPerm(current_user, Permission.CHANGE_DNAME):
 				user.display_name = form["display_name"].data
+
+			if user.checkPerm(current_user, Permission.CHANGE_PROFILE_URLS):
 				user.website_url  = form["website_url"].data
 				user.donate_url   = form["donate_url"].data
 
@@ -190,7 +192,7 @@ def set_password():
 
 			# Send 'password_changed' email
 			if user_manager.USER_ENABLE_EMAIL and current_user.email:
-				emails.send_password_changed_email(current_user)
+				user_manager.email_manager.send_password_changed_email(current_user)
 
 			# Send password_changed signal
 			signals.user_changed_password.send(current_app._get_current_object(), user=current_user)

app/flatpages/help.md

@@ -1,8 +1,15 @@
 title: Help
 
+## Content
+
 * [Package Tags](package_tags)
 * [Ranks and Permissions](ranks_permissions)
 * [Content Ratings and Flags](content_flags)
 * [Reporting Content](reporting)
+* [Top Packages Algorithm](top_packages)
+
+## Developers
+
 * [API](api)
+* [Metrics](metrics)
 * [Creating Releases using Webhooks](release_webhooks)

app/flatpages/help/api.md

@@ -9,6 +9,8 @@ Authentication is done using Bearer tokens:
 
 You can use the `/api/whoami` to check authentication.
 
+Tokens can be attained by visiting "API Tokens" on your profile page.
+
 ## Endpoints
 
 ### Misc
@@ -16,11 +18,12 @@ You can use the `/api/whoami` to check authentication.
 * GET `/api/whoami/` - Json dictionary with the following keys:
 	* `is_authenticated` - True on successful API authentication
 	* `username` - Username of the user authenticated as, null otherwise.
-	* 403 will be thrown on unsupported authentication type, invalid access token, or other errors.
+	* 4xx status codes will be thrown on unsupported authentication type, invalid access token, or other errors.
 
 ### Packages
 
 * GET `/api/packages/` - See [Package Queries](#package-queries)
+* GET `/api/scores/` - See [Package Queries](#package-queries)
 * GET `/api/packages/<username>/<name>/`
 
 ### Releases

app/flatpages/help/metrics.md

@@ -0,0 +1,16 @@
+title: Prometheus Metrics
+
+## What is Prometheus?
+
+[Prometheus](https://prometheus.io) is an "open-source monitoring system with a
+dimensional data model, flexible query language, efficient time series database
+and modern alerting approach".
+
+Prometheus Metrics can be accessed at [/metrics](/metrics).
+
+## Metrics
+
+* `contentdb_packages` - Total packages (counter).
+* `contentdb_users` - Number of registered users (counter).
+* `contentdb_downloads` - Total downloads (counter).
+* `contentdb_score` - Total package score (gauge).

app/flatpages/help/release_webhooks.md

@@ -17,10 +17,6 @@ The process is as follows:
 3. The git host posts a webhook notification to ContentDB, using the API token assigned to it.
 4. ContentDB checks the API token and issues a new release.
 
-<p class="alert alert-info">
-	This feature is in beta, and is only available for Trusted Members.
-</p>
-
 ## Setting up
 
 ### GitHub (automatic)

app/flatpages/help/top_packages.md

@@ -0,0 +1,49 @@
+title: Top Packages Algorithm
+
+## Score
+
+A package's score is currently equal to a pseudo rolling average of downloads.
+In the future, a package will also gain score through reviews.
+
+## Pseudo rolling average of downloads
+
+Every package loses 5% of its score every day.
+
+An open source package will gain 1 score for each unique download,
+whereas a non-free package will only gain 0.1 score.
+
+This metric aims to be roughly equivalent to the average downloads.
+
+## Seeded using a legacy heuristic
+
+The scoring system was seeded (ie: the scores were initially set to) 20% of an
+arbitrary legacy heuristic that was previously used to rank packages.
+
+This legacy heuristic is as follows:
+
+	forum_score = views / max(years_since_creation, 2 weeks) + 80*clamp(months, 0.5, 6)
+	forum_bonus = views + posts
+
+	multiplier = 1
+	if no screenshot:
+		multiplier *= 0.8
+	if not foss:
+		multiplier *= 0.1
+
+	score = multiplier * (max(downloads, forum_score * 0.6) + forum_bonus)
+
+As said, this legacy score is no longer used when ranking mods.
+It was only used to provide an initial score for the rolling average,
+which was 20% of the above value.
+
+## Manual adjustments
+
+The admin occasionally reduces all packages by a set percentage to speed up
+convergence. Convergence is when
+
+## Transparency and Feedback
+
+You can see all scores using the [scores REST API](/api/scores/), or by
+using the [Prometheus metrics](/help/metrics/) endpoint.
+
+Consider [suggesting improvements](https://github.com/minetest/contentdb/issues/new?assignees=&labels=Policy&template=policy.md&title=).

app/flatpages/policy_and_guidance.md

@@ -76,7 +76,7 @@ to change the name of the package, or your package won't be accepted.
 
 We reserve the right to issue exceptions for this where we feel necessary.
 
-### 3.2 Mod Forks and Reimplementations
+### 3.2. Mod Forks and Reimplementations
 
 An exception to the above is that mods are allowed to have the same name as a
 mod if its a fork of that mod (or a close reimplementation). In real terms, it
@@ -88,7 +88,7 @@ reimplementation of the mod that owns the name.
 
 ## 4. Licenses
 
-### 4.1 Allowed Licenses
+### 4.1. Allowed Licenses
 
 Please ensure that you correctly credit any resources (code, assets, or otherwise)
 that you have used in your package.
@@ -106,7 +106,7 @@ get around to adding it.
 Please note that the definitions of "free" and "non-free" is the same as that
 of the [Free Software Foundation](https://www.gnu.org/philosophy/free-sw.en.html).
 
-### 4.2 Recommended Licenses
+### 4.2. Recommended Licenses
 
 It is highly recommended that you use a free and open source software license.
 FOSS licenses result in a sharing community and will increase the number of potential users your package has.

app/maillogger.py

@@ -41,7 +41,7 @@ class FlaskMailHTMLFormatter(logging.Formatter):
 		formatted_exception = logging.Handler.formatException(self, exc_info)
 		return FlaskMailHTMLFormatter.pre_template % ("Exception information", formatted_exception)
 	def formatStack(self, stack_info):
-		return FlaskMailHTMLFormatter.pre_template % ("<h1>Stack information</h1><pre>%s</pre>", stack_info)
+		return FlaskMailHTMLFormatter.pre_template % ("<h1>Stack information</h1><pre><code>%s</code></pre>", stack_info)
 
 
 # see: https://github.com/python/cpython/blob/3.6/Lib/logging/__init__.py (class Handler)
@@ -100,7 +100,7 @@ Message:
 <tr>		<th>Time:</th><td>%(asctime)s</td></tr>
 </table>
 <h2>Message</h2>
-<pre>%(message)s</pre>"""
+<pre><code>%(message)s</code></pre>"""
 
 	import logging
 	mail_handler = FlaskMailHandler(mailer, subject_template)

app/models.py

@@ -93,6 +93,7 @@ class Permission(enum.Enum):
 	UNAPPROVE_PACKAGE  = "UNAPPROVE_PACKAGE"
 	TOPIC_DISCARD      = "TOPIC_DISCARD"
 	CREATE_TOKEN       = "CREATE_TOKEN"
+	CHANGE_PROFILE_URLS = "CHANGE_PROFILE_URLS"
 
 	# Only return true if the permission is valid for *all* contexts
 	# See Package.checkPerm for package-specific contexts
@@ -146,7 +147,7 @@ class User(db.Model, UserMixin):
 	notifications = db.relationship("Notification", primaryjoin="User.id==Notification.user_id")
 
 	# causednotifs  = db.relationship("Notification", backref="causer", lazy="dynamic")
-	packages      = db.relationship("Package", backref="author", lazy="dynamic")
+	packages      = db.relationship("Package", backref=db.backref("author", lazy="joined"), lazy="dynamic")
 	requests      = db.relationship("EditRequest", backref="author", lazy="dynamic")
 	threads       = db.relationship("Thread", backref="author", lazy="dynamic")
 	tokens        = db.relationship("APIToken", backref="owner", lazy="dynamic")
@@ -192,7 +193,7 @@ class User(db.Model, UserMixin):
 			return user.rank.atLeast(UserRank.EDITOR)
 		elif perm == Permission.CHANGE_RANK or perm == Permission.CHANGE_DNAME:
 			return user.rank.atLeast(UserRank.MODERATOR)
-		elif perm == Permission.CHANGE_EMAIL:
+		elif perm == Permission.CHANGE_EMAIL or perm == Permission.CHANGE_PROFILE_URLS:
 			return user == self or (user.rank.atLeast(UserRank.MODERATOR) and user.rank.atLeast(self.rank))
 		elif perm == Permission.CREATE_TOKEN:
 			if user == self:
@@ -213,6 +214,9 @@ class User(db.Model, UserMixin):
 			.filter(Thread.created_at > hour_ago).count() < 2
 
 	def __eq__(self, other):
+		if other is None:
+			return False
+
 		if not self.is_authenticated or not other.is_authenticated:
 			return False
 
@@ -433,12 +437,12 @@ class Package(db.Model):
 	forums       = db.Column(db.Integer,     nullable=True)
 
 	provides = db.relationship("MetaPackage", \
-			secondary=provides, lazy="subquery", order_by=db.asc("name"), \
+			secondary=provides, lazy="select", order_by=db.asc("name"), \
 			backref=db.backref("packages", lazy="dynamic", order_by=db.desc("score")))
 
 	dependencies = db.relationship("Dependency", backref="depender", lazy="dynamic", foreign_keys=[Dependency.depender_id])
 
-	tags = db.relationship("Tag", secondary=tags, lazy="subquery",
+	tags = db.relationship("Tag", secondary=tags, lazy="select",
 			backref=db.backref("packages", lazy=True))
 
 	releases = db.relationship("PackageRelease", backref="package",
@@ -574,13 +578,25 @@ class Package(db.Model):
 		screenshot = self.screenshots.filter_by(approved=True).order_by(db.asc(PackageScreenshot.id)).first()
 		return screenshot.getThumbnailURL(level) if screenshot is not None else None
 
-	def getMainScreenshotURL(self):
+	def getMainScreenshotURL(self, absolute=False):
 		screenshot = self.screenshots.filter_by(approved=True).order_by(db.asc(PackageScreenshot.id)).first()
-		return screenshot.url if screenshot is not None else None
+		if screenshot is None:
+			return None
 
-	def getDetailsURL(self):
-		return url_for("packages.view",
-				author=self.author.username, name=self.name)
+		if absolute:
+			from app.utils import abs_url
+			return abs_url(screenshot.url)
+		else:
+			return screenshot.url
+
+	def getDetailsURL(self, absolute=False):
+		if absolute:
+			from app.utils import abs_url_for
+			return abs_url_for("packages.view",
+					author=self.author.username, name=self.name)
+		else:
+			return url_for("packages.view",
+					author=self.author.username, name=self.name)
 
 	def getEditURL(self):
 		return url_for("packages.create_edit",
@@ -655,7 +671,7 @@ class Package(db.Model):
 		if perm == Permission.MAKE_RELEASE or perm == Permission.ADD_SCREENSHOTS:
 			return isOwner or user.rank.atLeast(UserRank.EDITOR)
 
-		if perm == Permission.EDIT_PACKAGE or perm == Permission.APPROVE_CHANGES:
+		if perm == Permission.EDIT_PACKAGE or perm == Permission.APPROVE_CHANGES or perm == Permission.APPROVE_RELEASE:
 			if isOwner:
 				return user.rank.atLeast(UserRank.MEMBER if self.approved else UserRank.NEW_MEMBER)
 			else:
@@ -669,7 +685,7 @@ class Package(db.Model):
 		elif perm == Permission.APPROVE_NEW or perm == Permission.CHANGE_AUTHOR:
 			return user.rank.atLeast(UserRank.EDITOR)
 
-		elif perm == Permission.APPROVE_RELEASE or perm == Permission.APPROVE_SCREENSHOT:
+		elif perm == Permission.APPROVE_SCREENSHOT:
 			return user.rank.atLeast(UserRank.TRUSTED_MEMBER if isOwner else UserRank.EDITOR)
 
 		# Moderators can delete packages
@@ -831,12 +847,11 @@ class PackageRelease(db.Model):
 				name=self.package.name,
 				id=self.id)
 
-
 	def __init__(self):
 		self.releaseDate = datetime.datetime.now()
 
 	def approve(self, user):
-		if self.package.approved and \
+		if self.package.approved or \
 				not self.package.checkPerm(user, Permission.APPROVE_RELEASE):
 			return False
 
@@ -1131,3 +1146,8 @@ class ForumTopic(db.Model):
 
 # Setup Flask-User
 user_manager = UserManager(app, db, User)
+
+if app.config.get("LOG_SQL"):
+	import logging
+	logging.basicConfig()
+	logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)

app/querybuilder.py

@@ -1,5 +1,6 @@
-from .models import db, PackageType, Package, ForumTopic, License, MinetestRelease, PackageRelease
-from .utils import isNo, isYes
+from .models import db, PackageType, Package, ForumTopic, License, MinetestRelease, PackageRelease, User, Tag
+from .models import tags as Tags
+from .utils import isNo, isYes, get_int_or_abort
 from sqlalchemy.sql.expression import func
 from flask import abort
 from sqlalchemy import or_
@@ -19,18 +20,30 @@ class QueryBuilder:
 		if len(types) > 0:
 			title = ", ".join([type.value + "s" for type in types])
 
+		# Get tags types
+		tags = args.getlist("tag")
+		tags = [Tag.query.filter_by(name=tname).first() for tname in tags]
+		tags = [tag for tag in tags if tag is not None]
+
+		# Hid
 		hide_flags = args.getlist("hide")
 
 		self.title  = title
 		self.types  = types
-		self.search = args.get("q")
+		self.tags   = tags
+
 		self.random = "random" in args
 		self.lucky  = "lucky" in args
 		self.hide_nonfree = "nonfree" in hide_flags
 		self.limit  = 1 if self.lucky else None
 		self.order_by  = args.get("sort")
 		self.order_dir = args.get("order") or "desc"
+
+		# Filters
+
+		self.search = args.get("q")
 		self.protocol_version = args.get("protocol_version")
+		self.author = args.get("author")
 
 		self.show_discarded = isYes(args.get("show_discarded"))
 		self.show_added = args.get("show_added")
@@ -48,7 +61,7 @@ class QueryBuilder:
 		if not self.protocol_version:
 			return None
 
-		self.protocol_version = int(self.protocol_version)
+		self.protocol_version = get_int_or_abort(self.protocol_version)
 		version = MinetestRelease.query.filter(MinetestRelease.protocol>=self.protocol_version).first()
 		if version is not None:
 			return version.id
@@ -84,6 +97,16 @@ class QueryBuilder:
 
 			query = query.order_by(to_order)
 
+		if self.author:
+			author = User.query.filter_by(username=self.author).first()
+			if not author:
+				abort(404)
+
+			query = query.filter_by(author=author)
+
+		for tag in self.tags:
+			query = query.filter(Package.tags.any(Tag.id == tag.id))
+
 		if self.hide_nonfree:
 			query = query.filter(Package.license.has(License.is_foss == True))
 			query = query.filter(Package.media_license.has(License.is_foss == True))
@@ -116,7 +139,6 @@ class QueryBuilder:
 			query = query.order_by(db.desc(ForumTopic.views))
 		elif self.order_by == "date":
 			query = query.order_by(db.asc(ForumTopic.created_at))
-			sort_by = "date"
 
 		if self.search:
 			query = query.filter(ForumTopic.title.ilike('%' + self.search + '%'))

app/scss/custom.scss

@@ -3,12 +3,54 @@
 @import "packagegrid.scss";
 @import "comments.scss";
 
+h1 {
+	font-size: 2em;
+	font-weight: bold;
+	margin: 0 0 0.5em;
+	letter-spacing: .05em
+}
+
+h2 {
+	font-size: 1.8em;
+	font-weight: bold;
+	color: white;
+	margin: 1.5em 0 1em;
+	letter-spacing: .05em;
+	padding: 0 0 0.5em 0;
+	border-bottom: 1px solid #444;
+}
+
+h3 {
+	font-size: 1.3em;
+	font-weight: bold;
+	color: white;
+	margin: 1.5em 0 1em;
+	letter-spacing: .05em
+}
+
+p, .content li {
+	-moz-osx-font-smoothing: grayscale;
+	-webkit-font-smoothing: antialiased !important;
+	-moz-font-smoothing: antialiased !important;
+	text-rendering: optimizelegibility !important;
+	letter-spacing: .03em;
+	line-height: 1.6em;
+}
+
+pre code {
+	display: block;
+	border: 1px solid rgba(255, 255, 255, 0.1);
+	background: rgba(255, 255, 255, 0.03);
+	padding: 0.75rem 1.25rem;
+	border-radius: 0.25rem;
+}
+
 .dropdown-menu {
 	margin-top: 0;
 }
 
 .dropdown:hover .dropdown-menu {
-   display: block;
+	display: block;
 }
 
 .nav-link > img {
@@ -57,8 +99,15 @@
 	text-decoration: none;
 }
 
-.card .table {
-	margin-bottom: 0;
+.card {
+	.card-header {
+		margin: 0;
+		font-size: 100%;
+		font-weight: normal;
+	}
+	.table {
+		margin-bottom: 0;
+	}
 }
 
 .btn-download {

app/scss/packagegrid.scss

@@ -38,35 +38,43 @@ li.d-flex {
 	bottom: 0;
 	left: 0;
 	padding: 1em;
+
+	h3 {
+		color: white;
+		font-size: 120%;
+		font-weight: bold;
+		margin: 0;
+		padding: 0;
+	}
+
+	small {
+		color: #ddd;
+		font-size: 75%;
+		font-weight: bold;
+	}
+
+	p {
+		display: none;
+		color: #ddd;
+		font-weight: normal;
+	}
 }
 
-.packagegridinfo h3 {
-	color: white;
-	font-size: 120%;
-	font-weight: bold;
-}
+.packagetile a:hover  {
+	.packagegridinfo {
+		top: 0;
+	}
 
-.packagegridinfo small {
-	color: #ddd;
-	font-size: 75%;
-	font-weight: bold;
-}
+	h3 {
+		margin-bottom: 0.5em;
+	}
 
-.packagegridinfo p {
-	display: none;
-	color: #ddd;
-	font-weight: normal;
-}
+	p {
+		display: block;
+	}
 
-.packagetile a:hover .packagegridinfo {
-	top: 0;
-}
-
-.packagetile a:hover p {
-	display: block;
-}
-
-.packagetile a:hover .packagegridscrub {
-	top: 0;
-	background: rgba(0,0,0,0.8);
+	.packagegridscrub {
+		top: 0;
+		background: rgba(0,0,0,0.8);
+	}
 }

app/tasks/importtasks.py

@@ -162,6 +162,9 @@ def cloneRepo(urlstr, ref=None, recursive=False):
 			origin.fetch()
 			origin.pull(ref)
 
+			for submodule in repo.submodules:
+				submodule.update(init=True)
+
 		return gitDir, repo
 
 	except GitCommandError as e:

app/tasks/phpbbparser.py

@@ -74,7 +74,14 @@ def __extract_signature(soup):
 def getProfile(url, username):
 	url = url + "/memberlist.php?mode=viewprofile&un=" + urlEncodeNonAscii(username)
 
-	contents = urllib.request.urlopen(url).read().decode("utf-8")
+	req = urllib.request.urlopen(url, timeout=5)
+	if req.getcode() == 404:
+		return None
+
+	if req.getcode() != 200:
+		raise IOError(req.getcode())
+
+	contents = req.read().decode("utf-8")
 	soup = BeautifulSoup(contents, "lxml")
 	if soup is None:
 		return None

app/tasks/pkgtasks.py

@@ -15,9 +15,10 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 
-from app.models import Package
+from app.models import Package, db
 from app.tasks import celery
 
 @celery.task()
 def updatePackageScores():
-	Package.query.update({ "score": Package.score * 0.8 })
+	Package.query.update({ "score": Package.score * 0.95 })
+	db.session.commit()

app/template_filters.py

@@ -1,10 +1,16 @@
 from . import app
+from .utils import abs_url_for
 from urllib.parse import urlparse
 
 @app.context_processor
 def inject_debug():
     return dict(debug=app.debug)
 
+
+@app.context_processor
+def inject_functions():
+    return dict(abs_url_for=abs_url_for)
+
 @app.template_filter()
 def throw(err):
 	raise Exception(err)

app/templates/base.html

@@ -7,7 +7,7 @@
 	<meta name="viewport" content="width=device-width, initial-scale=1">
 	<title>{% block title %}title{% endblock %} - {{ config.USER_APP_NAME }}</title>
 	<link rel="stylesheet" type="text/css" href="/static/bootstrap.css">
-	<link rel="stylesheet" type="text/css" href="/static/custom.css?v=10">
+	<link rel="stylesheet" type="text/css" href="/static/custom.css?v=11">
 	<link rel="search" type="application/opensearchdescription+xml" href="/static/opensearch.xml" title="ContentDB" />
 	<link rel="shortcut icon" href="/favicon-16.png" sizes="16x16">
 	<link rel="icon" href="/favicon-128.png" sizes="128x128">
@@ -129,7 +129,7 @@
 	{% endblock %}
 
 	<footer class="container footer-copyright my-5 page-footer font-small text-center">
-		ContentDB &copy; 2018-9 to <a href="https://rubenwardy.com/">rubenwardy</a> |
+		ContentDB &copy; 2018-20 to <a href="https://rubenwardy.com/">rubenwardy</a> |
 		<a href="https://github.com/minetest/contentdb">GitHub</a> |
 		<a href="{{ url_for('flatpage', path='help') }}">{{ _("Help") }}</a> |
 		<a href="{{ url_for('flatpage', path='policy_and_guidance') }}">{{ _("Policy and Guidance") }}</a> |

app/templates/flatpage.html

@@ -4,8 +4,10 @@
 {{ page['title'] }}
 {% endblock %}
 
-{% block content %}
-	{% if not page["no_h1"] %}<h1>{{ page['title'] }}</h1>{% endif %}
+{% block container %}
+	<main class="container mt-4 content">
+		{% if not page["no_h1"] %}<h1>{{ page['title'] }}</h1>{% endif %}
 
-	{{ page.html | safe }}
+		{{ page.html | safe }}
+	</main>
 {% endblock %}

app/templates/packages/list.html

@@ -5,6 +5,21 @@
 {% endblock %}
 
 {% block content %}
+	{% if authors %}
+		<p class="alert alert-primary">
+			Did you mean to search for packages by
+
+			{% for author in authors %}
+				<a href="{{ url_for('packages.list_all', type=type, author=author[0], q=author[1]) }}">{{ author[0] }}</a>
+				{% if not loop.last %}
+    				,
+				{% endif %}
+			{% endfor %}
+			?
+		</p>
+	{% endif %}
+
+
 	{% from "macros/packagegridtile.html" import render_pkggrid %}
 	{{ render_pkggrid(packages) }}
 

app/templates/packages/view.html

@@ -6,6 +6,16 @@
 	{{ package.title }}
 {% endblock %}
 
+{% block headextra %}
+	<meta name="og:title" content="{{ package.title }}"/>
+	<meta name="og:description" content="{{ package.short_desc }}"/>
+	<meta name="description" content="{{ package.short_desc }}"/>
+	<meta name="og:url" content="{{ package.getDetailsURL(absolute=True) }}"/>
+	{% if package.getMainScreenshotURL() %}
+	<meta name="og:image" content="{{ package.getMainScreenshotURL(absolute=True) }}"/>
+	{% endif %}
+{% endblock %}
+
 {% block container %}
 	{% if not package.license.is_foss and not package.media_license.is_foss and package.type != package.type.TXP  %}
 		{% set package_warning="Non-free code and media" %}
@@ -38,7 +48,8 @@
 					</span>
 				{% endif %}
 				{% for t in package.tags %}
-					<span class="badge badge-primary">{{ t.title }}</span>
+					<a class="badge badge-primary"
+						href="{{ url_for('packages.list_all', tag=t.name) }}">{{ t.title }}</a>
 				{% endfor %}
 			</p>
 

app/templates/users/profile.html

@@ -164,6 +164,9 @@
 
 					{% if user.checkPerm(current_user, "CHANGE_DNAME") %}
 						{{ render_field(form.display_name, tabindex=230) }}
+					{% endif %}
+
+					{% if user.checkPerm(current_user, "CHANGE_PROFILE_URLS") %}
 						{{ render_field(form.website_url, tabindex=232) }}
 						{{ render_field(form.donate_url, tabindex=233) }}
 					{% endif %}
@@ -177,7 +180,9 @@
 						{{ render_field(form.rank, tabindex=250) }}
 					{% endif %}
 
-					{{ render_submit_field(form.submit, tabindex=280) }}
+					<p>
+						{{ render_submit_field(form.submit, tabindex=280) }}
+					</p>
 				</form>
 			</div>
 		</div>

app/utils.py

@@ -18,16 +18,18 @@
 from flask import request, flash, abort, redirect
 from flask_user import *
 from flask_login import login_user, logout_user
-from app.models import *
-from app import app
+from .models import *
+from . import app
 import random, string, os, imghdr
+from urllib.parse import urljoin
 
-
-@app.template_filter()
 def abs_url_for(path, **kwargs):
 	scheme = "https" if app.config["BASE_URL"][:5] == "https" else "http"
 	return url_for(path, _external=True, _scheme=scheme, **kwargs)
 
+def abs_url(path):
+	return urljoin(app.config["BASE_URL"], path)
+
 def get_int_or_abort(v, default=None):
 	try:
 		return int(v or default)

config.example.cfg

@@ -32,6 +32,9 @@ MAIL_UTILS_ERROR_SEND_TO = [""]
 UPLOAD_DIR = "/var/cdb/uploads/"
 THUMBNAIL_DIR = "/var/cdb/thumbnails/"
 
+TEMPLATES_AUTO_RELOAD = False
+LOG_SQL = False
+
 LANGUAGES = {
     'en': 'English',
 }

docker-compose.yml

@@ -60,3 +60,12 @@ services:
       - 5124:5124
     depends_on:
       - redis
+
+  exporter:
+    image: ovalmoney/celery-exporter
+    env_file:
+      - config.env
+    ports:
+      - 5125:9540
+    depends_on:
+      - redis

requirements.txt

@@ -17,7 +17,7 @@ beautifulsoup4~=4.6
 celery~=4.4
 kombu~=4.6
 GitPython~=3.0
-git-archive-all~=1.20
+git-archive-all~=1.21
 lxml~=4.2
 pillow~=7.0
 pyScss~=1.3