Fix email sign up

Fixes #190

app/__init__.py

@@ -21,7 +21,7 @@ from flask_gravatar import Gravatar
 import flask_menu as menu
 from flask_mail import Mail
 from flask_github import GitHub
-from flask_wtf.csrf import CsrfProtect
+from flask_wtf.csrf import CSRFProtect
 from flask_flatpages import FlatPages
 from flask_babel import Babel
 import os, redis
@@ -35,7 +35,7 @@ r = redis.Redis.from_url(app.config["REDIS_URL"])
 
 menu.Menu(app=app)
 github = GitHub(app)
-csrf = CsrfProtect(app)
+csrf = CSRFProtect(app)
 mail = Mail(app)
 pages = FlatPages(app)
 babel = Babel(app)
@@ -60,7 +60,6 @@ if not app.debug and app.config["MAIL_UTILS_ERROR_SEND_TO"]:
 from .markdown import init_app
 init_app(app)
 
-
 # @babel.localeselector
 # def get_locale():
 # 	return request.accept_languages.best_match(app.config['LANGUAGES'].keys())
@@ -87,7 +86,7 @@ def flatpage(path):
 def check_for_ban():
 	if current_user.is_authenticated:
 		if current_user.rank == models.UserRank.BANNED:
-			flash("You have been banned.", "error")
+			flash("You have been banned.", "danger")
 			logout_user()
 			return redirect(url_for('user.login'))
 		elif current_user.rank == models.UserRank.NOT_JOINED:

app/blueprints/admin/admin.py

@@ -71,7 +71,7 @@ def admin_page():
 		elif action == "restore":
 			package = Package.query.get(request.form["package"])
 			if package is None:
-				flash("Unknown package", "error")
+				flash("Unknown package", "danger")
 			else:
 				package.soft_deleted = False
 				db.session.commit()
@@ -115,7 +115,7 @@ def admin_page():
 				db.session.commit()
 
 		else:
-			flash("Unknown action: " + action, "error")
+			flash("Unknown action: " + action, "danger")
 
 	deleted_packages = Package.query.filter_by(soft_deleted=True).all()
 	return render_template("admin/list.html", deleted_packages=deleted_packages)
@@ -132,11 +132,11 @@ def switch_user():
 	if request.method == "POST" and form.validate():
 		user = User.query.filter_by(username=form["username"].data).first()
 		if user is None:
-			flash("Unable to find user", "error")
+			flash("Unable to find user", "danger")
 		elif loginUser(user):
 			return redirect(url_for("users.profile", username=current_user.username))
 		else:
-			flash("Unable to login as user", "error")
+			flash("Unable to login as user", "danger")
 
 
 	# Process GET or invalid POST

app/blueprints/api/endpoints.py

@@ -19,6 +19,7 @@ from flask import *
 from flask_user import *
 from . import bp
 from .auth import is_api_authd
+from .support import error, handleCreateRelease
 from app import csrf
 from app.models import *
 from app.utils import is_package_page
@@ -71,6 +72,13 @@ def package_dependencies(package):
 	return jsonify(ret)
 
 
+@bp.route("/api/packages/<author>/<name>/releases/")
+@is_package_page
+def list_releases(package):
+	releases = package.releases.filter_by(approved=True).all()
+	return jsonify([ rel.getAsDictionary() for rel in releases ])
+
+
 @bp.route("/api/topics/")
 def topics():
 	qb     = QueryBuilder(request.args)
@@ -113,5 +121,25 @@ def whoami(token):
 
 @bp.route("/api/markdown/", methods=["POST"])
 @csrf.exempt
-def clean_markdown():
+def markdown():
 	return render_markdown(request.data.decode("utf-8"))
+
+
+@bp.route("/api/packages/<author>/<name>/releases/new/", methods=["POST"])
+@csrf.exempt
+@is_package_page
+@is_api_authd
+def create_release(token, package):
+	json = request.json
+	if json is None:
+		return error(400, "JSON post data is required")
+
+	for option in ["method", "title", "ref"]:
+		if json.get(option) is None:
+			return error(400, option + " is required in the POST data")
+
+
+	if json["method"].lower() != "git":
+		return error(400, "Release-creation methods other than git are not supported")
+
+	return handleCreateRelease(token, package, json["title"], json["ref"])

app/blueprints/api/support.py

@@ -0,0 +1,40 @@
+from app.models import PackageRelease, db, Permission
+from app.tasks.importtasks import makeVCSRelease
+from celery import uuid
+from flask import jsonify, make_response, url_for
+import datetime
+
+
+def error(status, message):
+	return make_response(jsonify({ "success": False, "error": message }), status)
+
+
+def handleCreateRelease(token, package, title, ref):
+	if not token.canOperateOnPackage(package):
+		return error(403, "API token does not have access to the package")
+
+	if not package.checkPerm(token.owner, Permission.MAKE_RELEASE):
+		return error(403, "Permission denied. Missing MAKE_RELEASE permission")
+
+	five_minutes_ago = datetime.datetime.now() - datetime.timedelta(minutes=5)
+	count = package.releases.filter(PackageRelease.releaseDate > five_minutes_ago).count()
+	if count >= 2:
+		return error(429, "Too many requests, please wait before trying again")
+
+	rel = PackageRelease()
+	rel.package = package
+	rel.title   = title
+	rel.url     = ""
+	rel.task_id = uuid()
+	rel.min_rel = None
+	rel.max_rel = None
+	db.session.add(rel)
+	db.session.commit()
+
+	makeVCSRelease.apply_async((rel.id, ref), task_id=rel.task_id)
+
+	return jsonify({
+		"success": True,
+		"task": url_for("tasks.check", id=rel.task_id),
+		"release": rel.getAsDictionary()
+	})

app/blueprints/api/tokens.py

@@ -29,6 +29,8 @@ from wtforms.ext.sqlalchemy.fields import QuerySelectField
 
 class CreateAPIToken(FlaskForm):
 	name	     = StringField("Name", [InputRequired(), Length(1, 30)])
+	package      = QuerySelectField("Limit to package", allow_blank=True, \
+			get_pk=lambda a: a.id, get_label=lambda a: a.title)
 	submit	     = SubmitField("Save")
 
 
@@ -67,9 +69,11 @@ def create_edit_token(username, id=None):
 		elif token.owner != user:
 			abort(403)
 
-		access_token = session.pop("token_" + str(id), None)
+		access_token = session.pop("token_" + str(token.id), None)
 
 	form = CreateAPIToken(formdata=request.form, obj=token)
+	form.package.query_factory = lambda: Package.query.filter_by(author=user).all()
+
 	if request.method == "POST" and form.validate():
 		if is_new:
 			token = APIToken()
@@ -78,11 +82,11 @@ def create_edit_token(username, id=None):
 
 		form.populate_obj(token)
 		db.session.add(token)
-
 		db.session.commit() # save
 
-		# Store token so it can be shown in the edit page
-		session["token_" + str(token.id)] = token.access_token
+		if is_new:
+			# Store token so it can be shown in the edit page
+			session["token_" + str(token.id)] = token.access_token
 
 		return redirect(url_for("api.create_edit_token", username=username, id=token.id))
 
@@ -99,8 +103,6 @@ def reset_token(username, id):
 	if not user.checkPerm(current_user, Permission.CREATE_TOKEN):
 		abort(403)
 
-	is_new = id is None
-
 	token = APIToken.query.get(id)
 	if token is None:
 		abort(404)

app/blueprints/github/__init__.py

@@ -0,0 +1,273 @@
+# Content DB
+# Copyright (C) 2018  rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+from flask import Blueprint
+
+bp = Blueprint("github", __name__)
+
+from flask import redirect, url_for, request, flash, abort, render_template, jsonify, current_app
+from flask_user import current_user, login_required
+from sqlalchemy import func, or_, and_
+from flask_github import GitHub
+from app import github, csrf
+from app.models import db, User, APIToken, Package, Permission
+from app.utils import loginUser, randomString, abs_url_for
+from app.blueprints.api.support import error, handleCreateRelease
+import hmac, requests, json
+
+from flask_wtf import FlaskForm
+from wtforms import SelectField, SubmitField
+
+@bp.route("/github/start/")
+def start():
+	return github.authorize("", redirect_uri=abs_url_for("github.callback"))
+
+@bp.route("/github/view/")
+def view_permissions():
+	url = "https://github.com/settings/connections/applications/" + \
+			current_app.config["GITHUB_CLIENT_ID"]
+	return redirect(url)
+
+@bp.route("/github/callback/")
+@github.authorized_handler
+def callback(oauth_token):
+	next_url = request.args.get("next")
+	if oauth_token is None:
+		flash("Authorization failed [err=gh-oauth-login-failed]", "danger")
+		return redirect(url_for("user.login"))
+
+	# Get Github username
+	url = "https://api.github.com/user"
+	r = requests.get(url, headers={"Authorization": "token " + oauth_token})
+	username = r.json()["login"]
+
+	# Get user by github username
+	userByGithub = User.query.filter(func.lower(User.github_username) == func.lower(username)).first()
+
+	# If logged in, connect
+	if current_user and current_user.is_authenticated:
+		if userByGithub is None:
+			current_user.github_username = username
+			db.session.commit()
+			flash("Linked github to account", "success")
+			return redirect(url_for("homepage.home"))
+		else:
+			flash("Github account is already associated with another user", "danger")
+			return redirect(url_for("homepage.home"))
+
+	# If not logged in, log in
+	else:
+		if userByGithub is None:
+			flash("Unable to find an account for that Github user", "danger")
+			return redirect(url_for("users.claim"))
+		elif loginUser(userByGithub):
+			if not current_user.hasPassword():
+				return redirect(next_url or url_for("users.set_password", optional=True))
+			else:
+				return redirect(next_url or url_for("homepage.home"))
+		else:
+			flash("Authorization failed [err=gh-login-failed]", "danger")
+			return redirect(url_for("user.login"))
+
+
+@bp.route("/github/webhook/", methods=["POST"])
+@csrf.exempt
+def webhook():
+	json = request.json
+
+	# Get package
+	github_url = "github.com/" + json["repository"]["full_name"]
+	package = Package.query.filter(Package.repo.like("%{}%".format(github_url))).first()
+	if package is None:
+		return error(400, "Could not find package, did you set the VCS repo in CDB correctly?")
+
+	# Get all tokens for package
+	tokens_query = APIToken.query.filter(or_(APIToken.package==package,
+			and_(APIToken.package==None, APIToken.owner==package.author)))
+
+	possible_tokens = tokens_query.all()
+	actual_token = None
+
+	#
+	# Check signature
+	#
+
+	header_signature = request.headers.get('X-Hub-Signature')
+	if header_signature is None:
+		return error(403, "Expected payload signature")
+
+	sha_name, signature = header_signature.split('=')
+	if sha_name != 'sha1':
+		return error(403, "Expected SHA1 payload signature")
+
+	for token in possible_tokens:
+		mac = hmac.new(token.access_token.encode("utf-8"), msg=request.data, digestmod='sha1')
+
+		if hmac.compare_digest(str(mac.hexdigest()), signature):
+			actual_token = token
+			break
+
+	if actual_token is None:
+		return error(403, "Invalid authentication, couldn't validate API token")
+
+	if not package.checkPerm(actual_token.owner, Permission.APPROVE_RELEASE):
+		return error(403, "Only trusted members can use webhooks")
+
+	#
+	# Check event
+	#
+
+	event = request.headers.get("X-GitHub-Event")
+	if event == "push":
+		ref = json["after"]
+		title = json["head_commit"]["message"].partition("\n")[0]
+	elif event == "create" and json["ref_type"] == "tag":
+		ref = json["ref"]
+		title = ref
+	elif event == "ping":
+		return jsonify({ "success": True, "message": "Ping successful" })
+	else:
+		return error(400, "Unsupported event. Only 'push', `create:tag`, and 'ping' are supported.")
+
+	#
+	# Perform release
+	#
+
+	return handleCreateRelease(actual_token, package, title, ref)
+
+
+class SetupWebhookForm(FlaskForm):
+	event   = SelectField("Event Type", choices=[('create', 'New tag or GitHub release'), ('push', 'Push')])
+	submit  = SubmitField("Save")
+
+
+@bp.route("/github/callback/webhook/")
+@github.authorized_handler
+def callback_webhook(oauth_token=None):
+	pid = request.args.get("pid")
+	if pid is None:
+		abort(404)
+
+	current_user.github_access_token = oauth_token
+	db.session.commit()
+
+	return redirect(url_for("github.setup_webhook", pid=pid))
+
+
+@bp.route("/github/webhook/new/", methods=["GET", "POST"])
+@login_required
+def setup_webhook():
+	pid = request.args.get("pid")
+	if pid is None:
+		abort(404)
+
+	package = Package.query.get(pid)
+	if package is None:
+		abort(404)
+
+	if not package.checkPerm(current_user, Permission.APPROVE_RELEASE):
+		flash("Only trusted members can use webhooks", "danger")
+		return redirect(package.getDetailsURL())
+
+	gh_user, gh_repo = package.getGitHubFullName()
+	if gh_user is None or gh_repo is None:
+		flash("Unable to get Github full name from repo address", "danger")
+		return redirect(package.getDetailsURL())
+
+	if current_user.github_access_token is None:
+		return github.authorize("write:repo_hook", \
+			redirect_uri=abs_url_for("github.callback_webhook", pid=pid))
+
+	form = SetupWebhookForm(formdata=request.form)
+	if request.method == "POST" and form.validate():
+		token = APIToken()
+		token.name = "GitHub Webhook for " + package.title
+		token.owner = current_user
+		token.access_token = randomString(32)
+		token.package = package
+
+		event = form.event.data
+		if event != "push" and event != "create":
+			abort(500)
+
+		if handleMakeWebhook(gh_user, gh_repo, package, \
+				current_user.github_access_token, event, token):
+			flash("Successfully created webhook", "success")
+			return redirect(package.getDetailsURL())
+		else:
+			return redirect(url_for("github.setup_webhook", pid=package.id))
+
+	return render_template("github/setup_webhook.html", \
+		form=form, package=package)
+
+
+def handleMakeWebhook(gh_user, gh_repo, package, oauth, event, token):
+	url = "https://api.github.com/repos/{}/{}/hooks".format(gh_user, gh_repo)
+	headers = {
+		"Authorization": "token " + oauth
+	}
+	data = {
+		"name": "web",
+		"active": True,
+		"events": [event],
+		"config": {
+			"url": abs_url_for("github.webhook"),
+			"content_type": "json",
+			"secret": token.access_token
+		},
+	}
+
+	# First check that the webhook doesn't already exist
+	r = requests.get(url, headers=headers)
+
+	if r.status_code == 401 or r.status_code == 403:
+		current_user.github_access_token = None
+		db.session.commit()
+		return False
+
+	if r.status_code != 200:
+		flash("Failed to create webhook, received response from Github " +
+			str(r.status_code) + ": " +
+			str(r.json().get("message")), "danger")
+		return False
+
+	for hook in r.json():
+		if hook.get("config") and hook["config"].get("url") and \
+				hook["config"]["url"] == data["config"]["url"]:
+			flash("Failed to create webhook, as it already exists", "danger")
+			return False
+
+
+	# Create it
+	r = requests.post(url, headers=headers, data=json.dumps(data))
+
+	if r.status_code == 201:
+		db.session.add(token)
+		db.session.commit()
+
+		return True
+
+	elif r.status_code == 401 or r.status_code == 403:
+		current_user.github_access_token = None
+		db.session.commit()
+
+		return False
+
+	else:
+		flash("Failed to create webhook, received response from Github " +
+			str(r.status_code) + ": " +
+			str(r.json().get("message")), "danger")
+		return False

app/blueprints/gitlab/__init__.py

@@ -0,0 +1,67 @@
+# ContentDB
+# Copyright (C) 2020  rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+from flask import Blueprint, request
+
+bp = Blueprint("gitlab", __name__)
+
+from app import csrf
+from app.models import Package, APIToken, Permission
+from app.blueprints.api.support import error, handleCreateRelease
+
+
+@bp.route("/gitlab/webhook/", methods=["POST"])
+@csrf.exempt
+def webhook():
+	json = request.json
+
+	# Get package
+	gitlab_url = "gitlab.com/{}/{}".format(json["project"]["namespace"], json["project"]["name"])
+	package = Package.query.filter(Package.repo.like("%{}%".format(gitlab_url))).first()
+	if package is None:
+		return error(400, "Unknown package")
+
+	# Get all tokens for package
+	secret = request.headers.get("X-Gitlab-Token")
+	if secret is None:
+		return error(403, "Token required")
+
+	token = APIToken.query.filter_by(access_token=secret).first()
+	if secret is None:
+		return error(403, "Invalid authentication")
+
+	if not package.checkPerm(token.owner, Permission.APPROVE_RELEASE):
+		return error(403, "Only trusted members can use webhooks")
+
+	#
+	# Check event
+	#
+
+	event = json["event_name"]
+	if event == "push":
+		ref = json["after"]
+		title = ref[:5]
+	elif event == "tag_push":
+		ref = json["ref"]
+		title = ref.replace("refs/tags/", "")
+	else:
+		return error(400, "Unsupported event. Only 'push' and 'tag_push' are supported.")
+
+	#
+	# Perform release
+	#
+
+	return handleCreateRelease(token, package, title, ref)

app/blueprints/packages/editrequests.py

@@ -50,7 +50,7 @@ def create_edit_editrequest_page(package, id=None):
 			abort(403)
 
 		if erequest.status != 0:
-			flash("Can't edit EditRequest, it has already been merged or rejected", "error")
+			flash("Can't edit EditRequest, it has already been merged or rejected", "danger")
 			return redirect(erequest.getURL())
 
 		edited_package = Package(package)
@@ -127,7 +127,7 @@ def view_editrequest_page(package, id):
 @is_package_page
 def approve_editrequest_page(package, id):
 	if not package.checkPerm(current_user, Permission.APPROVE_CHANGES):
-		flash("You don't have permission to do that.", "error")
+		flash("You don't have permission to do that.", "danger")
 		return redirect(package.getDetailsURL())
 
 	erequest = EditRequest.query.get(id)
@@ -135,7 +135,7 @@ def approve_editrequest_page(package, id):
 		abort(404)
 
 	if erequest.status != 0:
-		flash("Edit request has already been resolved", "error")
+		flash("Edit request has already been resolved", "danger")
 
 	else:
 		erequest.status = 1
@@ -152,7 +152,7 @@ def approve_editrequest_page(package, id):
 @is_package_page
 def reject_editrequest_page(package, id):
 	if not package.checkPerm(current_user, Permission.APPROVE_CHANGES):
-		flash("You don't have permission to do that.", "error")
+		flash("You don't have permission to do that.", "danger")
 		return redirect(package.getDetailsURL())
 
 	erequest = EditRequest.query.get(id)
@@ -160,7 +160,7 @@ def reject_editrequest_page(package, id):
 		abort(404)
 
 	if erequest.status != 0:
-		flash("Edit request has already been resolved", "error")
+		flash("Edit request has already been resolved", "danger")
 
 	else:
 		erequest.status = 2

app/blueprints/packages/packages.py

@@ -30,7 +30,7 @@ from flask_wtf import FlaskForm
 from wtforms import *
 from wtforms.validators import *
 from wtforms.ext.sqlalchemy.fields import QuerySelectField, QuerySelectMultipleField
-from sqlalchemy import or_
+from sqlalchemy import or_, func
 
 
 @menu.register_menu(bp, ".mods", "Mods", order=11, endpoint_arguments_constructor=lambda: { 'type': 'mod' })
@@ -64,14 +64,24 @@ def list_all():
 	prev_url = url_for("packages.list_all", type=type_name, q=search, page=query.prev_num) \
 			if query.has_prev else None
 
+	authors = []
+	if search:
+		authors = User.query \
+			.filter(or_(*[func.lower(User.username) == name.lower().strip() for name in search.split(" ")])) \
+			.all()
+
+		authors = [(author.username, search.lower().replace(author.username.lower(), "")) for author in authors]
+
 	topics = None
 	if qb.search and not query.has_next:
+		qb.show_discarded = True
 		topics = qb.buildTopicQuery().all()
 
 	tags = Tag.query.all()
 	return render_template("packages/list.html", \
 			title=title, packages=query.items, topics=topics, \
 			query=search, tags=tags, type=type_name, \
+			authors = authors, \
 			next_url=next_url, prev_url=prev_url, page=page, page_max=query.pages, packages_count=query.total)
 
 
@@ -160,25 +170,20 @@ def download(package):
 				not "text/html" in request.accept_mimetypes:
 			return "", 204
 		else:
-			flash("No download available.", "error")
+			flash("No download available.", "danger")
 			return redirect(package.getDetailsURL())
 	else:
-		PackageRelease.query.filter_by(id=release.id).update({
-				"downloads": PackageRelease.downloads + 1
-			})
-		db.session.commit()
-
-		return redirect(release.url, code=302)
+		return redirect(release.getDownloadURL(), code=302)
 
 
 class PackageForm(FlaskForm):
-	name          = StringField("Name (Technical)", [InputRequired(), Length(1, 20), Regexp("^[a-z0-9_]", 0, "Lower case letters (a-z), digits (0-9), and underscores (_) only")])
-	title         = StringField("Title (Human-readable)", [InputRequired(), Length(3, 50)])
+	name          = StringField("Name (Technical)", [InputRequired(), Length(1, 100), Regexp("^[a-z0-9_]+$", 0, "Lower case letters (a-z), digits (0-9), and underscores (_) only")])
+	title         = StringField("Title (Human-readable)", [InputRequired(), Length(3, 100)])
 	short_desc     = StringField("Short Description (Plaintext)", [InputRequired(), Length(1,200)])
 	desc          = TextAreaField("Long Description (Markdown)", [Optional(), Length(0,10000)])
 	type          = SelectField("Type", [InputRequired()], choices=PackageType.choices(), coerce=PackageType.coerce, default=PackageType.MOD)
-	license       = QuerySelectField("License", [InputRequired()], query_factory=lambda: License.query.order_by(db.asc(License.name)), get_pk=lambda a: a.id, get_label=lambda a: a.name)
-	media_license = QuerySelectField("Media License", [InputRequired()], query_factory=lambda: License.query.order_by(db.asc(License.name)), get_pk=lambda a: a.id, get_label=lambda a: a.name)
+	license       = QuerySelectField("License", [DataRequired()], allow_blank=True, query_factory=lambda: License.query.order_by(db.asc(License.name)), get_pk=lambda a: a.id, get_label=lambda a: a.name)
+	media_license = QuerySelectField("Media License", [DataRequired()], allow_blank=True, query_factory=lambda: License.query.order_by(db.asc(License.name)), get_pk=lambda a: a.id, get_label=lambda a: a.name)
 	provides_str  = StringField("Provides (mods included in package)", [Optional()])
 	tags          = QuerySelectMultipleField('Tags', query_factory=lambda: Tag.query.order_by(db.asc(Tag.name)), get_pk=lambda a: a.id, get_label=lambda a: a.title)
 	harddep_str   = StringField("Hard Dependencies", [Optional()])
@@ -203,11 +208,11 @@ def create_edit(author=None, name=None):
 		else:
 			author = User.query.filter_by(username=author).first()
 			if author is None:
-				flash("Unable to find that user", "error")
+				flash("Unable to find that user", "danger")
 				return redirect(url_for("packages.create_edit"))
 
 			if not author.checkPerm(current_user, Permission.CHANGE_AUTHOR):
-				flash("Permission denied", "error")
+				flash("Permission denied", "danger")
 				return redirect(url_for("packages.create_edit"))
 
 	else:
@@ -226,6 +231,8 @@ def create_edit(author=None, name=None):
 			form.title.data  = request.args.get("title")
 			form.repo.data   = request.args.get("repo")
 			form.forums.data = request.args.get("forums")
+			form.license.data = None
+			form.media_license.data = None
 		else:
 			form.harddep_str.data  = ",".join([str(x) for x in package.getSortedHardDependencies() ])
 			form.softdep_str.data  = ",".join([str(x) for x in package.getSortedOptionalDependencies() ])
@@ -239,7 +246,7 @@ def create_edit(author=None, name=None):
 				if package.soft_deleted:
 					Package.query.filter_by(name=form["name"].data, author_id=author.id).delete()
 				else:
-					flash("Package already exists!", "error")
+					flash("Package already exists!", "danger")
 					return redirect(url_for("packages.create_edit"))
 
 			package = Package()
@@ -312,10 +319,10 @@ def create_edit(author=None, name=None):
 @is_package_page
 def approve(package):
 	if not package.checkPerm(current_user, Permission.APPROVE_NEW):
-		flash("You don't have permission to do that.", "error")
+		flash("You don't have permission to do that.", "danger")
 
 	elif package.approved:
-		flash("Package has already been approved", "error")
+		flash("Package has already been approved", "danger")
 
 	else:
 		package.approved = True
@@ -340,7 +347,7 @@ def remove(package):
 
 	if "delete" in request.form:
 		if not package.checkPerm(current_user, Permission.DELETE_PACKAGE):
-			flash("You don't have permission to do that.", "error")
+			flash("You don't have permission to do that.", "danger")
 			return redirect(package.getDetailsURL())
 
 		package.soft_deleted = True
@@ -355,7 +362,7 @@ def remove(package):
 		return redirect(url)
 	elif "unapprove" in request.form:
 		if not package.checkPerm(current_user, Permission.UNAPPROVE_PACKAGE):
-			flash("You don't have permission to do that.", "error")
+			flash("You don't have permission to do that.", "danger")
 			return redirect(package.getDetailsURL())
 
 		package.approved = False

app/blueprints/threads/__init__.py

@@ -142,7 +142,7 @@ def new():
 	if "pid" in request.args:
 		package = Package.query.get(int(request.args.get("pid")))
 		if package is None:
-			flash("Unable to find that package!", "error")
+			flash("Unable to find that package!", "danger")
 
 	# Don't allow making orphan threads on approved packages for now
 	if package is None:
@@ -156,12 +156,12 @@ def new():
 
 	# Check that user can make the thread
 	if not package.checkPerm(current_user, Permission.CREATE_THREAD):
-		flash("Unable to create thread!", "error")
+		flash("Unable to create thread!", "danger")
 		return redirect(url_for("homepage.home"))
 
 	# Only allow creating one thread when not approved
 	elif is_review_thread and package.review_thread is not None:
-		flash("A review thread already exists!", "error")
+		flash("A review thread already exists!", "danger")
 		return redirect(url_for("threads.view", id=package.review_thread.id))
 
 	elif not current_user.canOpenThreadRL():

app/blueprints/users/__init__.py

@@ -2,4 +2,4 @@ from flask import Blueprint
 
 bp = Blueprint("users", __name__)
 
-from . import githublogin, profile
+from . import profile, claim

app/blueprints/users/claim.py

@@ -0,0 +1,98 @@
+# Content DB
+# Copyright (C) 2018  rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+
+from . import bp
+from flask import redirect, render_template, session, request, flash, url_for
+from flask_user import current_user
+from app.models import db, User, UserRank
+from app.utils import randomString, loginUser, rank_required
+from app.tasks.forumtasks import checkForumAccount
+from app.tasks.phpbbparser import getProfile
+
+@bp.route("/user/claim/", methods=["GET", "POST"])
+def claim():
+	username = request.args.get("username")
+	if username is None:
+		username = ""
+	else:
+		method = request.args.get("method")
+		user = User.query.filter_by(forums_username=username).first()
+		if user and user.rank.atLeast(UserRank.NEW_MEMBER):
+			flash("User has already been claimed", "danger")
+			return redirect(url_for("users.claim"))
+		elif user is None and method == "github":
+			flash("Unable to get Github username for user", "danger")
+			return redirect(url_for("users.claim"))
+		elif user is None:
+			flash("Unable to find that user", "danger")
+			return redirect(url_for("users.claim"))
+
+		if user is not None and method == "github":
+			return redirect(url_for("github.start"))
+
+	token = None
+	if "forum_token" in session:
+		token = session["forum_token"]
+	else:
+		token = randomString(32)
+		session["forum_token"] = token
+
+	if request.method == "POST":
+		ctype	= request.form.get("claim_type")
+		username = request.form.get("username")
+
+		if username is None or len(username.strip()) < 2:
+			flash("Invalid username", "danger")
+		elif ctype == "github":
+			task = checkForumAccount.delay(username)
+			return redirect(url_for("tasks.check", id=task.id, r=url_for("users.claim", username=username, method="github")))
+		elif ctype == "forum":
+			user = User.query.filter_by(forums_username=username).first()
+			if user is not None and user.rank.atLeast(UserRank.NEW_MEMBER):
+				flash("That user has already been claimed!", "danger")
+				return redirect(url_for("users.claim"))
+
+			# Get signature
+			sig = None
+			try:
+				profile = getProfile("https://forum.minetest.net", username)
+				sig = profile.signature
+			except IOError:
+				flash("Unable to get forum signature - does the user exist?", "danger")
+				return redirect(url_for("users.claim", username=username))
+
+			# Look for key
+			if token in sig:
+				if user is None:
+					user = User(username)
+					user.forums_username = username
+					db.session.add(user)
+					db.session.commit()
+
+				if loginUser(user):
+					return redirect(url_for("users.set_password"))
+				else:
+					flash("Unable to login as user", "danger")
+					return redirect(url_for("users.claim", username=username))
+
+			else:
+				flash("Could not find the key in your signature!", "danger")
+				return redirect(url_for("users.claim", username=username))
+		else:
+			flash("Unknown claim type", "danger")
+
+	return render_template("users/claim.html", username=username, key=token)

app/blueprints/users/githublogin.py

@@ -1,74 +0,0 @@
-# Content DB
-# Copyright (C) 2018  rubenwardy
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-
-from flask import *
-from flask_user import *
-from flask_login import login_user, logout_user
-from sqlalchemy import func
-import flask_menu as menu
-from flask_github import GitHub
-from . import bp
-from app import github
-from app.models import *
-from app.utils import loginUser
-
-@bp.route("/user/github/start/")
-def github_signin():
-	return github.authorize("")
-
-@bp.route("/user/github/callback/")
-@github.authorized_handler
-def github_authorized(oauth_token):
-	next_url = request.args.get("next")
-	if oauth_token is None:
-		flash("Authorization failed [err=gh-oauth-login-failed]", "danger")
-		return redirect(url_for("user.login"))
-
-	import requests
-
-	# Get Github username
-	url = "https://api.github.com/user"
-	r = requests.get(url, headers={"Authorization": "token " + oauth_token})
-	username = r.json()["login"]
-
-	# Get user by github username
-	userByGithub = User.query.filter(func.lower(User.github_username) == func.lower(username)).first()
-
-	# If logged in, connect
-	if current_user and current_user.is_authenticated:
-		if userByGithub is None:
-			current_user.github_username = username
-			db.session.commit()
-			flash("Linked github to account", "success")
-			return redirect(url_for("homepage.home"))
-		else:
-			flash("Github account is already associated with another user", "danger")
-			return redirect(url_for("homepage.home"))
-
-	# If not logged in, log in
-	else:
-		if userByGithub is None:
-			flash("Unable to find an account for that Github user", "error")
-			return redirect(url_for("users.claim"))
-		elif loginUser(userByGithub):
-			if not current_user.hasPassword():
-				return redirect(next_url or url_for("users.set_password", optional=True))
-			else:
-				return redirect(next_url or url_for("homepage.home"))
-		else:
-			flash("Authorization failed [err=gh-login-failed]", "danger")
-			return redirect(url_for("user.login"))

app/blueprints/users/profile.py

@@ -63,6 +63,8 @@ def profile(username):
 			# Copy form fields to user_profile fields
 			if user.checkPerm(current_user, Permission.CHANGE_DNAME):
 				user.display_name = form["display_name"].data
+
+			if user.checkPerm(current_user, Permission.CHANGE_PROFILE_URLS):
 				user.website_url  = form["website_url"].data
 				user.donate_url   = form["donate_url"].data
 
@@ -71,7 +73,7 @@ def profile(username):
 				if current_user.rank.atLeast(newRank):
 					user.rank = form["rank"].data
 				else:
-					flash("Can't promote a user to a rank higher than yourself!", "error")
+					flash("Can't promote a user to a rank higher than yourself!", "danger")
 
 			if user.checkPerm(current_user, Permission.CHANGE_EMAIL):
 				newEmail = form["email"].data
@@ -147,7 +149,7 @@ def send_email(username):
 	next_url = url_for("users.profile", username=user.username)
 
 	if user.email is None:
-		flash("User has no email address!", "error")
+		flash("User has no email address!", "danger")
 		return redirect(next_url)
 
 	form = SendEmailForm(request.form)
@@ -214,91 +216,17 @@ def set_password():
 			else:
 				return redirect(url_for("user.login"))
 		else:
-			flash("Passwords do not match", "error")
+			flash("Passwords do not match", "danger")
 
 	return render_template("users/set_password.html", form=form, optional=request.args.get("optional"))
 
 
-@bp.route("/user/claim/", methods=["GET", "POST"])
-def claim():
-	username = request.args.get("username")
-	if username is None:
-		username = ""
-	else:
-		method = request.args.get("method")
-		user = User.query.filter_by(forums_username=username).first()
-		if user and user.rank.atLeast(UserRank.NEW_MEMBER):
-			flash("User has already been claimed", "error")
-			return redirect(url_for("users.claim"))
-		elif user is None and method == "github":
-			flash("Unable to get Github username for user", "error")
-			return redirect(url_for("users.claim"))
-		elif user is None:
-			flash("Unable to find that user", "error")
-			return redirect(url_for("users.claim"))
-
-		if user is not None and method == "github":
-			return redirect(url_for("users.github_signin"))
-
-	token = None
-	if "forum_token" in session:
-		token = session["forum_token"]
-	else:
-		token = randomString(32)
-		session["forum_token"] = token
-
-	if request.method == "POST":
-		ctype	= request.form.get("claim_type")
-		username = request.form.get("username")
-
-		if username is None or len(username.strip()) < 2:
-			flash("Invalid username", "error")
-		elif ctype == "github":
-			task = checkForumAccount.delay(username)
-			return redirect(url_for("tasks.check", id=task.id, r=url_for("users.claim", username=username, method="github")))
-		elif ctype == "forum":
-			user = User.query.filter_by(forums_username=username).first()
-			if user is not None and user.rank.atLeast(UserRank.NEW_MEMBER):
-				flash("That user has already been claimed!", "error")
-				return redirect(url_for("users.claim"))
-
-			# Get signature
-			sig = None
-			try:
-				profile = getProfile("https://forum.minetest.net", username)
-				sig = profile.signature
-			except IOError:
-				flash("Unable to get forum signature - does the user exist?", "error")
-				return redirect(url_for("users.claim", username=username))
-
-			# Look for key
-			if token in sig:
-				if user is None:
-					user = User(username)
-					user.forums_username = username
-					db.session.add(user)
-					db.session.commit()
-
-				if loginUser(user):
-					return redirect(url_for("users.set_password"))
-				else:
-					flash("Unable to login as user", "error")
-					return redirect(url_for("users.claim", username=username))
-
-			else:
-				flash("Could not find the key in your signature!", "error")
-				return redirect(url_for("users.claim", username=username))
-		else:
-			flash("Unknown claim type", "error")
-
-	return render_template("users/claim.html", username=username, key=token)
-
 @bp.route("/users/verify/")
 def verify_email():
 	token = request.args.get("token")
 	ver = UserEmailVerification.query.filter_by(token=token).first()
 	if ver is None:
-		flash("Unknown verification token!", "error")
+		flash("Unknown verification token!", "danger")
 	else:
 		ver.user.email = ver.email
 		db.session.delete(ver)

app/flatpages/help.md

@@ -5,3 +5,4 @@ title: Help
 * [Content Ratings and Flags](content_flags)
 * [Reporting Content](reporting)
 * [API](api)
+* [Creating Releases using Webhooks](release_webhooks)

app/flatpages/help/api.md

@@ -23,6 +23,19 @@ You can use the `/api/whoami` to check authentication.
 * GET `/api/packages/` - See [Package Queries](#package-queries)
 * GET `/api/packages/<username>/<name>/`
 
+### Releases
+
+* GET `/api/packages/<username>/<name>/releases/`
+* POST `/api/packages/<username>/<name>/releases/new/`
+	* Requires authentication.
+	* `title`: human-readable name of the release.
+	* `method`: Release-creation method, only `git` is supported.
+	* `min_protocol`: (Optional) minimum Minetest protocol version. See [Minetest](#minetest).
+	* `min_protocol`: (Optional) maximum Minetest protocol version. See [Minetest](#minetest).
+	* If `git` release-creation method:
+		* `ref` - git reference, eg: `master`.
+
+
 ### Topics
 
 * GET `/api/topics/` - Supports [Package Queries](#package-queries), and the following two options:

app/flatpages/help/release_webhooks.md

@@ -0,0 +1,81 @@
+title: Creating Releases using Webhooks
+
+## What does this mean?
+
+A webhook is a notification from one service to another. Put simply, a webhook
+is used to notify ContentDB that the git repository has changed.
+
+ContentDB offers the ability to automatically create releases using webhooks
+from either Github or Gitlab. If you're not using either of those services,
+you can also use the [API](../api) to create releases.
+
+The process is as follows:
+
+1. The user creates an API Token and a webhook to use it. This can be done automatically
+   for Github.
+2. The user pushes a commit to the git host (Gitlab or Github).
+3. The git host posts a webhook notification to ContentDB, using the API token assigned to it.
+4. ContentDB checks the API token and issues a new release.
+
+<p class="alert alert-info">
+	This feature is in beta, and is only available for Trusted Members.
+</p>
+
+## Setting up
+
+### GitHub (automatic)
+
+1. Go to your package's page.
+2. Make sure that the repository URL is set to a Github repository.
+   Only github.com is supported.
+3. Go to "Releases" > "+", and click "Setup webhook" at the top of the create release
+   page.
+   If you do not see this, either the repository isn't using Github or you do
+   not have permission to use webhook releases (ie: you're not a Trusted Member).
+4. Grant ContentDB the ability to manage Webhooks.
+5. Set the event to either "New tag or Github Release" (highly recommended) or "Push".
+
+   N.B.: GitHub uses tags to power GitHub Releases, meaning that creating a webhook
+   on "New tag" will sync GitHub and ContentDB releases.
+
+### GitHub (manual)
+
+1. Create an API Token by visiting your profile and clicking "API Tokens: Manage".
+2. Copy the access token that was generated.
+3. Go to the repository's settings > Webhooks > Add Webhook.
+4. Set the payload URL to `https://content.minetest.net/github/webhook/`
+5. Set the content type to JSON.
+6. Set the secret to the access token that you copied.
+7. Set the events
+  * If you want a rolling release, choose "just the push event".
+  * Or if you want a stable release cycle based on tags,
+    choose "Let me select" > Branch or tag creation.
+
+### GitLab (manual)
+
+1. Create an API Token by visiting your profile and clicking "API Tokens: Manage".
+2. Copy the access token that was generated.
+3. Go to the repository's settings > Integrations.
+4. Set the URL to `https://content.minetest.net/gitlab/webhook/`
+6. Set the secret token to the access token that you copied.
+7. Set the events
+    * If you want a rolling release, choose "Push events".
+    * Or if you want a stable release cycle based on tags,
+      choose "Tag push events".
+
+## Configuring
+
+### Setting minimum and maximum Minetest versions
+
+<p class="alert alert-info">
+	This feature is unimplemented.
+</p>
+
+1. Open up the conf file for the package.
+   This will be `game.conf`, `mod.conf`, `modpack.conf`, or `texture_pack.conf`
+   depending on the content type.
+2. Set `min_protocol` and `max_protocol` to the respective protocol numbers
+   of the Minetest versions.
+     * 0.4 = 32
+     * 5.0 = 37
+     * 5.1 = 38

app/models.py

@@ -93,6 +93,7 @@ class Permission(enum.Enum):
 	UNAPPROVE_PACKAGE  = "UNAPPROVE_PACKAGE"
 	TOPIC_DISCARD      = "TOPIC_DISCARD"
 	CREATE_TOKEN       = "CREATE_TOKEN"
+	CHANGE_PROFILE_URLS = "CHANGE_PROFILE_URLS"
 
 	# Only return true if the permission is valid for *all* contexts
 	# See Package.checkPerm for package-specific contexts
@@ -109,6 +110,9 @@ class Permission(enum.Enum):
 		else:
 			raise Exception("Non-global permission checked globally. Use Package.checkPerm or User.checkPerm instead.")
 
+def display_name_default(context):
+    return context.get_current_parameters()["username"]
+
 class User(db.Model, UserMixin):
 	id           = db.Column(db.Integer, primary_key=True)
 
@@ -123,6 +127,9 @@ class User(db.Model, UserMixin):
 	github_username = db.Column(db.String(50, collation="NOCASE"), nullable=True, unique=True)
 	forums_username = db.Column(db.String(50, collation="NOCASE"), nullable=True, unique=True)
 
+	# Access token for webhook setup
+	github_access_token = db.Column(db.String(50), nullable=True, server_default=None)
+
 	# User email information
 	email         = db.Column(db.String(255), nullable=True, unique=True)
 	email_confirmed_at  = db.Column(db.DateTime())
@@ -130,7 +137,7 @@ class User(db.Model, UserMixin):
 	# User information
 	profile_pic   = db.Column(db.String(255), nullable=True, server_default=None)
 	active        = db.Column("is_active", db.Boolean, nullable=False, server_default="0")
-	display_name  = db.Column(db.String(100), nullable=False, server_default="")
+	display_name  = db.Column(db.String(100), nullable=False, default=display_name_default)
 
 	# Links
 	website_url   = db.Column(db.String(255), nullable=True, default=None)
@@ -146,7 +153,7 @@ class User(db.Model, UserMixin):
 	tokens        = db.relationship("APIToken", backref="owner", lazy="dynamic")
 	replies       = db.relationship("ThreadReply", backref="author", lazy="dynamic")
 
-	def __init__(self, username, active=False, email=None, password=""):
+	def __init__(self, username=None, active=False, email=None, password=""):
 		self.username = username
 		self.email_confirmed_at = datetime.datetime.now() - datetime.timedelta(days=6000)
 		self.display_name = username
@@ -186,7 +193,7 @@ class User(db.Model, UserMixin):
 			return user.rank.atLeast(UserRank.EDITOR)
 		elif perm == Permission.CHANGE_RANK or perm == Permission.CHANGE_DNAME:
 			return user.rank.atLeast(UserRank.MODERATOR)
-		elif perm == Permission.CHANGE_EMAIL:
+		elif perm == Permission.CHANGE_EMAIL or perm == Permission.CHANGE_PROFILE_URLS:
 			return user == self or (user.rank.atLeast(UserRank.MODERATOR) and user.rank.atLeast(self.rank))
 		elif perm == Permission.CREATE_TOKEN:
 			if user == self:
@@ -207,6 +214,9 @@ class User(db.Model, UserMixin):
 			.filter(Thread.created_at > hour_ago).count() < 2
 
 	def __eq__(self, other):
+		if other is None:
+			return False
+
 		if not self.is_authenticated or not other.is_authenticated:
 			return False
 
@@ -458,6 +468,31 @@ class Package(db.Model):
 	def getIsFOSS(self):
 		return self.license.is_foss and self.media_license.is_foss
 
+	def getIsOnGitHub(self):
+		if self.repo is None:
+			return False
+
+		url = urlparse(self.repo)
+		return url.netloc == "github.com"
+
+	def getGitHubFullName(self):
+		if self.repo is None:
+			return None
+
+		url = urlparse(self.repo)
+		if url.netloc != "github.com":
+			return None
+
+		import re
+		m = re.search(r"^\/([^\/]+)\/([^\/]+)\/?$", url.path)
+		if m is None:
+			return
+
+		user = m.group(1)
+		repo = m.group(2).replace(".git", "")
+
+		return (user,repo)
+
 	def getSortedDependencies(self, is_hard=None):
 		query = self.dependencies
 		if is_hard is not None:
@@ -519,7 +554,7 @@ class Package(db.Model):
 			"short_description": self.short_desc,
 			"desc": self.desc,
 			"type": self.type.toName(),
-			"created_at": self.created_at,
+			"created_at": self.created_at.isoformat(),
 
 			"license": self.license.name,
 			"media_license": self.media_license.name,
@@ -770,6 +805,18 @@ class PackageRelease(db.Model):
 	# If the release is approved, then the task_id must be null and the url must be present
 	CK_approval_valid = db.CheckConstraint("not approved OR (task_id IS NULL AND (url = '') IS NOT FALSE)")
 
+	def getAsDictionary(self):
+		return {
+			"id": self.id,
+			"title": self.title,
+			"url": self.url if self.url != "" else None,
+			"release_date": self.releaseDate.isoformat(),
+			"commit": self.commit_hash,
+			"downloads": self.downloads,
+			"min_protocol": self.min_rel and self.min_rel.protocol,
+			"max_protocol": self.max_rel and self.max_rel.protocol
+		}
+
 	def getEditURL(self):
 		return url_for("packages.edit_release",
 				author=self.package.author.username,
@@ -861,12 +908,21 @@ class PackageScreenshot(db.Model):
 class APIToken(db.Model):
 	id           = db.Column(db.Integer, primary_key=True)
 	access_token = db.Column(db.String(34), unique=True)
+
 	name         = db.Column(db.String(100), nullable=False)
 	owner_id     = db.Column(db.Integer, db.ForeignKey("user.id"), nullable=False)
+	# owner is created using backref
+
 	created_at   = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow)
 
+	package_id = db.Column(db.Integer, db.ForeignKey("package.id"), nullable=True)
+	package    = db.relationship("Package", foreign_keys=[package_id])
+
 	def canOperateOnPackage(self, package):
-		return packages.count() == 0 or package in packages
+		if self.package and self.package != package:
+			return False
+
+		return package.author == self.owner
 
 
 class EditRequest(db.Model):

app/querybuilder.py

@@ -1,4 +1,5 @@
-from .models import db, PackageType, Package, ForumTopic, License, MinetestRelease, PackageRelease
+from .models import db, PackageType, Package, ForumTopic, License, MinetestRelease, PackageRelease, User, Tag
+from .models import tags as Tags
 from .utils import isNo, isYes
 from sqlalchemy.sql.expression import func
 from flask import abort
@@ -19,18 +20,30 @@ class QueryBuilder:
 		if len(types) > 0:
 			title = ", ".join([type.value + "s" for type in types])
 
+		# Get tags types
+		tags = args.getlist("tag")
+		tags = [Tag.query.filter_by(name=tname).first() for tname in tags]
+		tags = [tag for tag in tags if tag is not None]
+
+		# Hid
 		hide_flags = args.getlist("hide")
 
 		self.title  = title
 		self.types  = types
-		self.search = args.get("q")
+		self.tags   = tags
+
 		self.random = "random" in args
 		self.lucky  = "lucky" in args
 		self.hide_nonfree = "nonfree" in hide_flags
 		self.limit  = 1 if self.lucky else None
 		self.order_by  = args.get("sort")
 		self.order_dir = args.get("order") or "desc"
+
+		# Filters
+
+		self.search = args.get("q")
 		self.protocol_version = args.get("protocol_version")
+		self.author = args.get("author")
 
 		self.show_discarded = isYes(args.get("show_discarded"))
 		self.show_added = args.get("show_added")
@@ -84,6 +97,16 @@ class QueryBuilder:
 
 			query = query.order_by(to_order)
 
+		if self.author:
+			author = User.query.filter_by(username=self.author).first()
+			if not author:
+				abort(404)
+
+			query = query.filter_by(author=author)
+
+		for tag in self.tags:
+			query = query.filter(Package.tags.any(Tag.id == tag.id))
+
 		if self.hide_nonfree:
 			query = query.filter(Package.license.has(License.is_foss == True))
 			query = query.filter(Package.media_license.has(License.is_foss == True))

app/tasks/emails.py

@@ -19,6 +19,7 @@ from flask import render_template, url_for
 from flask_mail import Message
 from app import mail
 from app.tasks import celery
+from app.utils import abs_url_for
 
 @celery.task()
 def sendVerifyEmail(newEmail, token):
@@ -34,7 +35,7 @@ def sendVerifyEmail(newEmail, token):
 			If this was you, then please click this link to verify the address:
 
 			{}
-		""".format(url_for('users.verify_email', token=token, _external=True))
+		""".format(abs_url_for('users.verify_email', token=token))
 
 	msg.html = render_template("emails/verify.html", token=token)
 	mail.send(msg)

app/tasks/importtasks.py

@@ -156,8 +156,14 @@ def cloneRepo(urlstr, ref=None, recursive=False):
 			repo = git.Repo.clone_from(gitUrl, gitDir, \
 					progress=None, env=None, depth=1, recursive=recursive, kill_after_timeout=15)
 		else:
-			repo = git.Repo.clone_from(gitUrl, gitDir, \
-					progress=None, env=None, depth=1, recursive=recursive, kill_after_timeout=15, b=ref)
+			repo = git.Repo.init(gitDir)
+			origin = repo.create_remote("origin", url=gitUrl)
+			assert origin.exists()
+			origin.fetch()
+			origin.pull(ref)
+
+			for submodule in repo.submodules:
+				submodule.update(init=True)
 
 		return gitDir, repo
 

app/tasks/pkgtasks.py

@@ -15,9 +15,10 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 
-from app.models import Package
+from app.models import Package, db
 from app.tasks import celery
 
 @celery.task()
 def updatePackageScores():
-	Package.query.update({ "score": Package.score * 0.8 })
+	Package.query.update({ "score": Package.score * 0.95 })
+	db.session.commit()

app/template_filters.py

@@ -1,10 +1,16 @@
 from . import app
+from .utils import abs_url_for
 from urllib.parse import urlparse
 
 @app.context_processor
 def inject_debug():
     return dict(debug=app.debug)
 
+
+@app.context_processor
+def inject_functions():
+    return dict(abs_url_for=abs_url_for)
+
 @app.template_filter()
 def throw(err):
 	raise Exception(err)

app/templates/api/create_edit_token.html

@@ -21,7 +21,8 @@
 	<h1 class="mt-0">{{ self.title() }}</h1>
 
 	<div class="alert alert-warning">
-		{{ _("Use carefully, as you may be held responsible for any damage caused by rogue scripts") }}
+		{{ _("API Tokens allow scripts to act on your behalf.") }}
+		{{ _("Be careful with what/whom you share tokens with, as you are responsible for your account's actions.") }}
 	</div>
 
 	{% if token %}
@@ -47,6 +48,7 @@
 		{{ form.hidden_tag() }}
 
 		{{ render_field(form.name, placeholder="Human readable") }}
+		{{ render_field(form.package) }}
 
 		{{ render_submit_field(form.submit) }}
 	</form>

app/templates/base.html

@@ -111,7 +111,7 @@
 						<li class="alert alert-{{category}} container">
 							<span class="icon_message"></span>
 
-							{{ message|safe }}
+							{{ message }}
 
 							<div style="clear: both;"></div>
 						</li>
@@ -129,7 +129,7 @@
 	{% endblock %}
 
 	<footer class="container footer-copyright my-5 page-footer font-small text-center">
-		ContentDB &copy; 2018-9 to <a href="https://rubenwardy.com/">rubenwardy</a> |
+		ContentDB &copy; 2018-20 to <a href="https://rubenwardy.com/">rubenwardy</a> |
 		<a href="https://github.com/minetest/contentdb">GitHub</a> |
 		<a href="{{ url_for('flatpage', path='help') }}">{{ _("Help") }}</a> |
 		<a href="{{ url_for('flatpage', path='policy_and_guidance') }}">{{ _("Policy and Guidance") }}</a> |

app/templates/emails/verify.html

@@ -16,12 +16,12 @@
 	If this was you, then please click this link to verify the address:
 </p>
 
-<a class="btn" href="{{ url_for('users.verify_email', token=token, _external=True) }}">
+<a class="btn" href="{{ abs_url_for('users.verify_email', token=token) }}">
 	Confirm Email Address
 </a>
 
 <p style="font-size: 80%;">
-	Or paste this into your browser: {{ url_for('users.verify_email', token=token, _external=True) }}
+	Or paste this into your browser: {{ abs_url_for('users.verify_email', token=token) }}
 <p>
 
 {% endblock %}

app/templates/flask_user/login.html

@@ -60,7 +60,7 @@ Sign in
 			{% from "flask_user/_macros.html" import render_field, render_checkbox_field, render_submit_field %}
 			<h2 class="card-header">{%trans%}Sign in with Github{%endtrans%}</h2>
 			<div class="card-body">
-				<a class="btn btn-primary" href="{{ url_for('users.github_signin') }}">GitHub</a>
+				<a class="btn btn-primary" href="{{ url_for('github.start') }}">GitHub</a>
 			</div>
 		</div>
 	</div>

app/templates/github/setup_webhook.html

@@ -0,0 +1,29 @@
+{% extends "base.html" %}
+
+{% block title %}
+	{{ _("Setup GitHub webhook") }}
+{% endblock %}
+
+{% from "macros/forms.html" import render_field, render_submit_field, render_radio_field %}
+
+{% block content %}
+	<h1 class="mt-0">{{ self.title() }}</h1>
+
+	<div class="alert alert-info">
+		{{ _("You can delete the webhook at any time by going into Settings > Webhooks on the repository.") }}
+	</div>
+
+	<form method="POST" action="" enctype="multipart/form-data">
+		{{ form.hidden_tag() }}
+
+		{{ render_field(form.event) }}
+
+		{{ render_submit_field(form.submit) }}
+	</form>
+
+	<p class="mt-4">
+		You will need admin access to the repository.
+		When setting up hooks on an organisation,
+		<a href="{{ url_for('github.view_permissions') }}">make sure that you have granted access</a>.
+	</p>
+{% endblock %}

app/templates/macros/topics.html

@@ -54,6 +54,7 @@
 		<li{% if topic.wip %} class="wiptopic"{% endif %}>
 			<a href="https://forum.minetest.net/viewtopic.php?t={{ topic.topic_id}}">{{ topic.title }}</a>
 			{% if topic.wip %}[WIP]{% endif %}
+			{% if topic.discarded %}[Old]{% endif %}
 			{% if topic.name %}[{{ topic.name }}]{% endif %}
 			{% if show_author %}
 				by <a href="{{ url_for('users.profile', username=topic.author.username) }}">{{ topic.author.display_name }}</a>

app/templates/packages/create_edit.html

@@ -50,7 +50,7 @@
 			{% if package and package.approved and not package.checkPerm(current_user, "CHANGE_NAME") %}
 				{{ render_field(form.name, class_="pkg_meta col-sm-3", readonly=True) }}
 			{% else %}
-				{{ render_field(form.name, class_="pkg_meta col-sm-3") }}
+				{{ render_field(form.name, class_="pkg_meta col-sm-3", pattern="[a-z0-9_]+", title=_("Lower case letters (a-z), digits (0-9), and underscores (_) only")) }}
 			{% endif %}
 			</div>
 			{{ render_field(form.short_desc, class_="pkg_meta") }}

app/templates/packages/list.html

@@ -5,6 +5,21 @@
 {% endblock %}
 
 {% block content %}
+	{% if authors %}
+		<p class="alert alert-primary">
+			Did you mean to search for packages by
+
+			{% for author in authors %}
+				<a href="{{ url_for('packages.list_all', type=type, author=author[0], q=author[1]) }}">{{ author[0] }}</a>
+				{% if not loop.last %}
+    				,
+				{% endif %}
+			{% endfor %}
+			?
+		</p>
+	{% endif %}
+
+
 	{% from "macros/packagegridtile.html" import render_pkggrid %}
 	{{ render_pkggrid(packages) }}
 

app/templates/packages/release_new.html

@@ -5,6 +5,24 @@
 {% endblock %}
 
 {% block content %}
+	<p class="alert alert-info mb-4">
+		<a class="float-right btn btn-sm btn-info" href="{{ url_for('flatpage', path='help/release_webhooks') }}">{{ _("Learn more") }}</a>
+		{% if package.author == current_user and package.checkPerm(current_user, "APPROVE_RELEASE") and package.getIsOnGitHub() %}
+			<a class="float-right btn btn-sm btn-info mr-2" href="{{ url_for('github.setup_webhook', pid=package.id) }}">{{ _("Setup webhook") }}</a>
+			<i class="fas fa-info mr-2"></i>
+
+			{{ _("Set up a webhook on GitHub to create releases automatically.") }}
+		{% elif package.repo %}
+			<i class="fas fa-info mr-2"></i>
+
+			{{ _("You can create releases automatically when you push commits or tags to your repository.") }}
+		{% else %}
+			<i class="fas fa-info mr-2"></i>
+
+			{{ _("Using git will allow you to create releases automatically when you push code or tags.") }}
+		{% endif %}
+	</p>
+
 	{% from "macros/forms.html" import render_field, render_submit_field, render_radio_field %}
 	<form method="POST" action="" enctype="multipart/form-data">
 		{{ form.hidden_tag() }}

app/templates/packages/view.html

@@ -38,7 +38,8 @@
 					</span>
 				{% endif %}
 				{% for t in package.tags %}
-					<span class="badge badge-primary">{{ t.title }}</span>
+					<a class="badge badge-primary"
+						href="{{ url_for('packages.list_all', tag=t.name) }}">{{ t.title }}</a>
 				{% endfor %}
 			</p>
 

app/templates/users/profile.html

@@ -57,13 +57,14 @@
 								{% if user.github_username %}
 									<a href="https://github.com/{{ user.github_username }}">GitHub</a>
 								{% elif user == current_user %}
-									<a href="{{ url_for('users.github_signin') }}">Link Github</a>
+									<a href="{{ url_for('github.start') }}">Link Github</a>
 								{% endif %}
 
 								{% if user.website_url %}
 									| <a href="{{ user.website_url }}" rel="nofollow">Website</a>
 								{% endif %}
 
+
 								{% if user == current_user %}
 									<br>
 									<small class="text-muted">
@@ -73,6 +74,16 @@
 								{% endif %}
 							</td>
 						</tr>
+
+						{% if user == current_user and user.github_username %}
+							<tr>
+								<td>Privacy:</td>
+								<td>
+									<a href="{{ url_for('github.view_permissions') }}">View ContentDB's GitHub Permissions</a>
+								</td>
+							</tr>
+						{% endif %}
+
 						{% if current_user.is_authenticated and current_user.rank.atLeast(current_user.rank.MODERATOR) %}
 						<tr>
 							<td>Admin</td>
@@ -115,7 +126,7 @@
 										</a>
 									{% endif %}
 								</td>
-							</tr>
+						</tr>
 							<tr>
 								<td>Password:</td>
 								<td>
@@ -153,6 +164,9 @@
 
 					{% if user.checkPerm(current_user, "CHANGE_DNAME") %}
 						{{ render_field(form.display_name, tabindex=230) }}
+					{% endif %}
+
+					{% if user.checkPerm(current_user, "CHANGE_PROFILE_URLS") %}
 						{{ render_field(form.website_url, tabindex=232) }}
 						{{ render_field(form.donate_url, tabindex=233) }}
 					{% endif %}
@@ -166,7 +180,9 @@
 						{{ render_field(form.rank, tabindex=250) }}
 					{% endif %}
 
-					{{ render_submit_field(form.submit, tabindex=280) }}
+					<p>
+						{{ render_submit_field(form.submit, tabindex=280) }}
+					</p>
 				</form>
 			</div>
 		</div>

app/utils.py

@@ -18,10 +18,14 @@
 from flask import request, flash, abort, redirect
 from flask_user import *
 from flask_login import login_user, logout_user
-from app.models import *
-from app import app
+from .models import *
+from . import app
 import random, string, os, imghdr
 
+def abs_url_for(path, **kwargs):
+	scheme = "https" if app.config["BASE_URL"][:5] == "https" else "http"
+	return url_for(path, _external=True, _scheme=scheme, **kwargs)
+
 def get_int_or_abort(v, default=None):
 	try:
 		return int(v or default)
@@ -48,7 +52,7 @@ def randomString(n):
 
 def doFileUpload(file, fileType, fileTypeDesc):
 	if not file or file is None or file.filename == "":
-		flash("No selected file", "error")
+		flash("No selected file", "danger")
 		return None, None
 
 	assert os.path.isdir(app.config["UPLOAD_DIR"]), "UPLOAD_DIR must exist"
@@ -114,7 +118,7 @@ def loginUser(user):
 		return False
 
 	if user.rank == UserRank.BANNED:
-		flash("You have been banned.", "error")
+		flash("You have been banned.", "danger")
 		return False
 
 	user.active = True
@@ -125,7 +129,7 @@ def loginUser(user):
 
 	# Check if user account has been disabled
 	if not _call_or_get(user.is_active):
-		flash("Your account has not been enabled.", "error")
+		flash("Your account has not been enabled.", "danger")
 		return False
 
 	# Use Flask-Login to sign in user

config.example.cfg

@@ -6,6 +6,7 @@ SECRET_KEY = ""
 WTF_CSRF_SECRET_KEY = ""
 
 SQLALCHEMY_DATABASE_URI = "sqlite:///../db.sqlite"
+SQLALCHEMY_TRACK_MODIFICATIONS = False
 
 GITHUB_CLIENT_ID = ""
 GITHUB_CLIENT_SECRET = ""

migrations/versions/7a48dbd05780_.py

@@ -0,0 +1,24 @@
+"""empty message
+
+Revision ID: 7a48dbd05780
+Revises: df66c78e6791
+Create Date: 2020-01-24 21:52:49.744404
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '7a48dbd05780'
+down_revision = 'df66c78e6791'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column('user', sa.Column('github_access_token', sa.String(length=50), nullable=True, server_default=None))
+
+
+def downgrade():
+    op.drop_column('user', 'github_access_token')

migrations/versions/df66c78e6791_.py

@@ -0,0 +1,26 @@
+"""empty message
+
+Revision ID: df66c78e6791
+Revises: a0f6c8743362
+Create Date: 2020-01-24 18:39:58.363417
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = 'df66c78e6791'
+down_revision = 'a0f6c8743362'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column('api_token', sa.Column('package_id', sa.Integer(), nullable=True))
+    op.create_foreign_key(None, 'api_token', 'package', ['package_id'], ['id'])
+
+
+def downgrade():
+    op.drop_constraint(None, 'api_token', type_='foreignkey')
+    op.drop_column('api_token', 'package_id')

requirements.txt

@@ -17,7 +17,7 @@ beautifulsoup4~=4.6
 celery~=4.4
 kombu~=4.6
 GitPython~=3.0
-git-archive-all~=1.20
+git-archive-all~=1.21
 lxml~=4.2
 pillow~=7.0
 pyScss~=1.3