Reduce chance of accidental release deletion

Fixes #176

.dockerignore

@@ -1,3 +1,5 @@
 .git
 data
 uploads
+*.pyc
+__pycache__

.gitignore

@@ -1,5 +1,5 @@
-config.cfg
-*.env
+/config.cfg
+/*.env
 *.sqlite
 .vscode
 custom.css

.gitlab-ci.yml

@@ -0,0 +1,22 @@
+image: docker/compose
+services:
+  - docker:dind
+cache:
+  key: "$CI_COMMIT_REF_SLUG"
+  paths:
+    - /var/lib/docker
+
+# build:
+#   stage: build
+#   script:
+#     - cp utils/gitlabci/* .
+#     - docker-compose build
+
+UI_Test:
+  stage: test
+  script:
+    - cp utils/gitlabci/* .
+    - docker-compose up -d
+    - ./utils/run_migrations.sh
+    - ./utils/tests_cov.sh
+    - docker-compose down

Dockerfile

@@ -5,15 +5,18 @@ RUN groupadd -g 5123 cdb && \
 
 WORKDIR /home/cdb
 
+RUN mkdir /var/cdb
+RUN chown -R cdb:cdb /var/cdb
+
 COPY requirements.txt requirements.txt
-RUN pip install -r ./requirements.txt
+RUN pip install -r requirements.txt
 RUN pip install gunicorn
 
 COPY utils utils
-COPY config.cfg ./config.cfg
+COPY config.cfg config.cfg
 COPY migrations migrations
 COPY app app
 
-RUN mkdir /home/cdb/app/public/uploads/
-RUN chown cdb:cdb /home/cdb -R
+RUN chown -R cdb:cdb /home/cdb
+
 USER cdb

README.md

@@ -18,10 +18,46 @@ rm db.sqlite && python setup.py -t && FLASK_CONFIG=../config.cfg FLASK_APP=app/_
 
 # Create migration
 FLASK_CONFIG=../config.cfg FLASK_APP=app/__init__.py flask db migrate
-
 # Run migration
 FLASK_CONFIG=../config.cfg FLASK_APP=app/__init__.py flask db upgrade
 
 # Enter docker
 docker exec -it contentdb_app_1 bash
+
+# Hot/live reload (only works with FLASK_DEBUG=1)
+./utils/reload.sh
+
+# Cold update a running version of CDB with minimal downtime
+./utils/update.sh
+```
+
+## Database
+
+
+```mermaid
+classDiagram
+
+User "1" --> "*" Package
+User --> UserEmailVerification
+User "1" --> "*" Notification
+Package "1" --> "*" Release
+Package "1" --> "*" Dependency
+Package "1" --> "*" Tag
+Package "1" --> "*" MetaPackage : provides
+Release --> MinetestVersion
+Package --> License
+Dependency --> Package
+Dependency --> MetaPackage
+MetaPackage "1" --> "*" Package
+Package "1" --> "*" Screenshot
+Package "1" --> "*" Thread
+Thread "1" --> "*" Reply
+Thread "1" --> "*" User : watchers
+User "1" --> "*" Thread
+User "1" --> "*" Reply
+User "1" --> "*" ForumTopic
+
+User --> "0..1" EmailPreferences
+User "1" --> "*" APIToken
+APIToken --> Package
 ```

app/__init__.py

@@ -72,7 +72,7 @@ from flask_login import logout_user
 
 @app.route("/uploads/<path:path>")
 def send_upload(path):
-	return send_from_directory("public/uploads", path)
+	return send_from_directory(app.config['UPLOAD_DIR'], path)
 
 @menu.register_menu(app, ".help", "Help", order=19, endpoint_arguments_constructor=lambda: { 'path': 'help' })
 @app.route('/<path:path>/')

app/blueprints/admin/admin.py

@@ -20,8 +20,8 @@ from flask_user import *
 import flask_menu as menu
 from . import bp
 from app.models import *
-from celery import uuid
-from app.tasks.importtasks import importRepoScreenshot, importAllDependencies, makeVCSRelease
+from celery import uuid, group
+from app.tasks.importtasks import importRepoScreenshot, importAllDependencies, makeVCSRelease, checkZipRelease
 from app.tasks.forumtasks  import importTopicList, checkAllForumAccounts
 from flask_wtf import FlaskForm
 from wtforms import *
@@ -37,6 +37,21 @@ def admin_page():
 			PackageRelease.query.filter(PackageRelease.task_id != None).delete()
 			db.session.commit()
 			return redirect(url_for("admin.admin_page"))
+		elif action == "checkreleases":
+			releases = PackageRelease.query.filter(PackageRelease.url.like("/uploads/%")).all()
+
+			tasks = []
+			for release in releases:
+				zippath = release.url.replace("/uploads/", app.config["UPLOAD_DIR"])
+				tasks.append(checkZipRelease.s(release.id, zippath))
+
+			result = group(tasks).apply_async()
+
+			while not result.ready():
+				import time
+				time.sleep(0.1)
+
+			return redirect(url_for("todo.view"))
 		elif action == "importmodlist":
 			task = importTopicList.delay()
 			return redirect(url_for("tasks.check", id=task.id, r=url_for("todo.topics")))

app/blueprints/api/__init__.py

@@ -14,87 +14,8 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-
-from flask import *
-from flask_user import *
-from app.models import *
-from app.utils import is_package_page
-from app.querybuilder import QueryBuilder
+from flask import Blueprint
 
 bp = Blueprint("api", __name__)
 
-@bp.route("/api/packages/")
-def packages():
-	qb    = QueryBuilder(request.args)
-	query = qb.buildPackageQuery()
-	ver   = qb.getMinetestVersion()
-
-	pkgs = [package.getAsDictionaryShort(current_app.config["BASE_URL"], version=ver) \
-			for package in query.all()]
-	return jsonify(pkgs)
-
-
-@bp.route("/api/packages/<author>/<name>/")
-@is_package_page
-def package(package):
-	return jsonify(package.getAsDictionary(current_app.config["BASE_URL"]))
-
-
-@bp.route("/api/packages/<author>/<name>/dependencies/")
-@is_package_page
-def package_dependencies(package):
-	ret = []
-
-	for dep in package.dependencies:
-		name = None
-		fulfilled_by = None
-
-		if dep.package:
-			name = dep.package.name
-			fulfilled_by = [ dep.package.getAsDictionaryKey() ]
-
-		elif dep.meta_package:
-			name = dep.meta_package.name
-			fulfilled_by = [ pkg.getAsDictionaryKey() for pkg in dep.meta_package.packages]
-
-		else:
-			raise "Malformed dependency"
-
-		ret.append({
-			"name": name,
-			"is_optional": dep.optional,
-			"packages": fulfilled_by
-		})
-
-	return jsonify(ret)
-
-
-@bp.route("/api/topics/")
-def topics():
-	qb     = QueryBuilder(request.args)
-	query  = qb.buildTopicQuery(show_added=True)
-	return jsonify([t.getAsDictionary() for t in query.all()])
-
-
-@bp.route("/api/topic_discard/", methods=["POST"])
-@login_required
-def topic_set_discard():
-	tid = request.args.get("tid")
-	discard = request.args.get("discard")
-	if tid is None or discard is None:
-		abort(400)
-
-	topic = ForumTopic.query.get(tid)
-	if not topic.checkPerm(current_user, Permission.TOPIC_DISCARD):
-		abort(403)
-
-	topic.discarded = discard == "true"
-	db.session.commit()
-
-	return jsonify(topic.getAsDictionary())
-
-
-@bp.route("/api/minetest_versions/")
-def versions():
-	return jsonify([{ "name": rel.name, "protocol_version": rel.protocol }\
-			for rel in MinetestRelease.query.all() if rel.getActual() is not None])
+from . import tokens, endpoints

app/blueprints/api/auth.py

@@ -0,0 +1,42 @@
+# Content DB
+# Copyright (C) 2019  rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+from flask import request, make_response, jsonify, abort
+from app.models import APIToken
+from functools import wraps
+
+def is_api_authd(f):
+	@wraps(f)
+	def decorated_function(*args, **kwargs):
+		token = None
+
+		value = request.headers.get("authorization")
+		if value is None:
+			pass
+		elif value[0:7].lower() == "bearer ":
+			access_token = value[7:]
+			if len(access_token) < 10:
+				abort(400)
+
+			token = APIToken.query.filter_by(access_token=access_token).first()
+			if token is None:
+				abort(403)
+		else:
+			abort(403)
+
+		return f(token=token, *args, **kwargs)
+
+	return decorated_function

app/blueprints/api/endpoints.py

@@ -0,0 +1,109 @@
+# Content DB
+# Copyright (C) 2018  rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+
+from flask import *
+from flask_user import *
+from . import bp
+from .auth import is_api_authd
+from app.models import *
+from app.utils import is_package_page
+from app.querybuilder import QueryBuilder
+
+@bp.route("/api/packages/")
+def packages():
+	qb    = QueryBuilder(request.args)
+	query = qb.buildPackageQuery()
+	ver   = qb.getMinetestVersion()
+
+	pkgs = [package.getAsDictionaryShort(current_app.config["BASE_URL"], version=ver) \
+			for package in query.all()]
+	return jsonify(pkgs)
+
+
+@bp.route("/api/packages/<author>/<name>/")
+@is_package_page
+def package(package):
+	return jsonify(package.getAsDictionary(current_app.config["BASE_URL"]))
+
+
+@bp.route("/api/packages/<author>/<name>/dependencies/")
+@is_package_page
+def package_dependencies(package):
+	ret = []
+
+	for dep in package.dependencies:
+		name = None
+		fulfilled_by = None
+
+		if dep.package:
+			name = dep.package.name
+			fulfilled_by = [ dep.package.getAsDictionaryKey() ]
+
+		elif dep.meta_package:
+			name = dep.meta_package.name
+			fulfilled_by = [ pkg.getAsDictionaryKey() for pkg in dep.meta_package.packages]
+
+		else:
+			raise "Malformed dependency"
+
+		ret.append({
+			"name": name,
+			"is_optional": dep.optional,
+			"packages": fulfilled_by
+		})
+
+	return jsonify(ret)
+
+
+@bp.route("/api/topics/")
+def topics():
+	qb     = QueryBuilder(request.args)
+	query  = qb.buildTopicQuery(show_added=True)
+	return jsonify([t.getAsDictionary() for t in query.all()])
+
+
+@bp.route("/api/topic_discard/", methods=["POST"])
+@login_required
+def topic_set_discard():
+	tid = request.args.get("tid")
+	discard = request.args.get("discard")
+	if tid is None or discard is None:
+		abort(400)
+
+	topic = ForumTopic.query.get(tid)
+	if not topic.checkPerm(current_user, Permission.TOPIC_DISCARD):
+		abort(403)
+
+	topic.discarded = discard == "true"
+	db.session.commit()
+
+	return jsonify(topic.getAsDictionary())
+
+
+@bp.route("/api/minetest_versions/")
+def versions():
+	return jsonify([{ "name": rel.name, "protocol_version": rel.protocol }\
+			for rel in MinetestRelease.query.all() if rel.getActual() is not None])
+
+
+@bp.route("/api/whoami/")
+@is_api_authd
+def whoami(token):
+	if token is None:
+		return jsonify({ "is_authenticated": False, "username": None })
+	else:
+		return jsonify({ "is_authenticated": True, "username": token.owner.username })

app/blueprints/api/tokens.py

@@ -0,0 +1,141 @@
+# Content DB
+# Copyright (C) 2018  rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+
+from flask import render_template, redirect, request, session, url_for, abort
+from flask_user import login_required, current_user
+from . import bp
+from app.models import db, User, APIToken, Package, Permission
+from app.utils import randomString
+from app.querybuilder import QueryBuilder
+
+from flask_wtf import FlaskForm
+from wtforms import *
+from wtforms.validators import *
+from wtforms.ext.sqlalchemy.fields import QuerySelectField
+
+class CreateAPIToken(FlaskForm):
+	name	     = StringField("Name", [InputRequired(), Length(1, 30)])
+	submit	     = SubmitField("Save")
+
+
+@bp.route("/users/<username>/tokens/")
+@login_required
+def list_tokens(username):
+	user = User.query.filter_by(username=username).first()
+	if user is None:
+		abort(404)
+
+	if not user.checkPerm(current_user, Permission.CREATE_TOKEN):
+		abort(403)
+
+	return render_template("api/list_tokens.html", user=user)
+
+
+@bp.route("/users/<username>/tokens/new/", methods=["GET", "POST"])
+@bp.route("/users/<username>/tokens/<int:id>/edit/", methods=["GET", "POST"])
+@login_required
+def create_edit_token(username, id=None):
+	user = User.query.filter_by(username=username).first()
+	if user is None:
+		abort(404)
+
+	if not user.checkPerm(current_user, Permission.CREATE_TOKEN):
+		abort(403)
+
+	is_new = id is None
+
+	token = None
+	access_token = None
+	if not is_new:
+		token = APIToken.query.get(id)
+		if token is None:
+			abort(404)
+		elif token.owner != user:
+			abort(403)
+
+		access_token = session.pop("token_" + str(id), None)
+
+	form = CreateAPIToken(formdata=request.form, obj=token)
+	if request.method == "POST" and form.validate():
+		if is_new:
+			token = APIToken()
+			token.owner = user
+			token.access_token = randomString(32)
+
+		form.populate_obj(token)
+		db.session.add(token)
+
+		db.session.commit() # save
+
+		# Store token so it can be shown in the edit page
+		session["token_" + str(token.id)] = token.access_token
+
+		return redirect(url_for("api.create_edit_token", username=username, id=token.id))
+
+	return render_template("api/create_edit_token.html", user=user, form=form, token=token, access_token=access_token)
+
+
+@bp.route("/users/<username>/tokens/<int:id>/reset/", methods=["POST"])
+@login_required
+def reset_token(username, id):
+	user = User.query.filter_by(username=username).first()
+	if user is None:
+		abort(404)
+
+	if not user.checkPerm(current_user, Permission.CREATE_TOKEN):
+		abort(403)
+
+	is_new = id is None
+
+	token = APIToken.query.get(id)
+	if token is None:
+		abort(404)
+	elif token.owner != user:
+		abort(403)
+
+	token.access_token = randomString(32)
+
+	db.session.commit() # save
+
+	# Store token so it can be shown in the edit page
+	session["token_" + str(token.id)] = token.access_token
+
+	return redirect(url_for("api.create_edit_token", username=username, id=token.id))
+
+
+@bp.route("/users/<username>/tokens/<int:id>/delete/", methods=["POST"])
+@login_required
+def delete_token(username, id):
+	user = User.query.filter_by(username=username).first()
+	if user is None:
+		abort(404)
+
+	if not user.checkPerm(current_user, Permission.CREATE_TOKEN):
+		abort(403)
+
+	is_new = id is None
+
+	token = APIToken.query.get(id)
+	if token is None:
+		abort(404)
+	elif token.owner != user:
+		abort(403)
+
+	db.session.delete(token)
+	db.session.commit()
+
+	return redirect(url_for("api.list_tokens", username=username))

app/blueprints/homepage/__init__.py

@@ -15,6 +15,7 @@ def home():
 	pop_mod = query.filter_by(type=PackageType.MOD).order_by(db.desc(Package.score)).limit(8).all()
 	pop_gam = query.filter_by(type=PackageType.GAME).order_by(db.desc(Package.score)).limit(4).all()
 	pop_txp = query.filter_by(type=PackageType.TXP).order_by(db.desc(Package.score)).limit(4).all()
-	downloads = db.session.query(func.sum(PackageRelease.downloads)).first()[0]
+	downloads_result = db.session.query(func.sum(PackageRelease.downloads)).one_or_none()
+	downloads = 0 if not downloads_result or not downloads_result[0] else downloads_result[0]
 	return render_template("index.html", count=count, downloads=downloads, \
 			new=new, pop_mod=pop_mod, pop_txp=pop_txp, pop_gam=pop_gam)

app/blueprints/notifications/__init__.py

@@ -15,7 +15,7 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 
-from flask import Blueprint, render_template, redirect
+from flask import Blueprint, render_template, redirect, url_for
 from flask_user import current_user, login_required
 from app.models import db
 

app/blueprints/packages/releases.py

@@ -22,7 +22,7 @@ from . import bp
 
 from app.rediscache import has_key, set_key, make_download_key
 from app.models import *
-from app.tasks.importtasks import makeVCSRelease
+from app.tasks.importtasks import makeVCSRelease, checkZipRelease
 from app.utils import *
 
 from celery import uuid
@@ -98,22 +98,25 @@ def create_release(package):
 
 			return redirect(url_for("tasks.check", id=rel.task_id, r=rel.getEditURL()))
 		else:
-			uploadedPath = doFileUpload(form.fileUpload.data, "zip", "a zip file")
-			if uploadedPath is not None:
+			uploadedUrl, uploadedPath = doFileUpload(form.fileUpload.data, "zip", "a zip file")
+			if uploadedUrl is not None:
 				rel = PackageRelease()
 				rel.package = package
 				rel.title = form["title"].data
-				rel.url = uploadedPath
+				rel.url = uploadedUrl
+				rel.task_id = uuid()
 				rel.min_rel = form["min_rel"].data.getActual()
 				rel.max_rel = form["max_rel"].data.getActual()
-				rel.approve(current_user)
 				db.session.add(rel)
 				db.session.commit()
 
+				checkZipRelease.apply_async((rel.id, uploadedPath), task_id=rel.task_id)
+
 				msg = "{}: Release {} created".format(package.title, rel.title)
 				triggerNotif(package.author, current_user, msg, rel.getEditURL())
 				db.session.commit()
-				return redirect(package.getDetailsURL())
+
+				return redirect(url_for("tasks.check", id=rel.task_id, r=rel.getEditURL()))
 
 	return render_template("packages/release_new.html", package=package, form=form)
 
@@ -163,6 +166,11 @@ def edit_release(package, id):
 
 	# Initial form class from post data and default data
 	form = EditPackageReleaseForm(formdata=request.form, obj=release)
+
+	# HACK: fix bug in wtforms
+	if request.method == "GET":
+		form.approved.data = release.approved
+
 	if request.method == "POST" and form.validate():
 		wasApproved = release.approved
 		if canEdit:
@@ -225,3 +233,20 @@ def bulk_change_release(package):
 		return redirect(package.getDetailsURL())
 
 	return render_template("packages/release_bulk_change.html", package=package, form=form)
+
+
+@bp.route("/packages/<author>/<name>/releases/<id>/delete/", methods=["POST"])
+@login_required
+@is_package_page
+def delete_release(package, id):
+	release = PackageRelease.query.get(id)
+	if release is None or release.package != package:
+		abort(404)
+
+	if not release.checkPerm(current_user, Permission.DELETE_RELEASE):
+		return redirect(release.getEditURL())
+
+	db.session.delete(release)
+	db.session.commit()
+
+	return redirect(package.getDetailsURL())

app/blueprints/packages/screenshots.py

@@ -50,13 +50,13 @@ def create_screenshot(package, id=None):
 	# Initial form class from post data and default data
 	form = CreateScreenshotForm()
 	if request.method == "POST" and form.validate():
-		uploadedPath = doFileUpload(form.fileUpload.data, "image",
+		uploadedUrl, uploadedPath = doFileUpload(form.fileUpload.data, "image",
 				"a PNG or JPG image file")
-		if uploadedPath is not None:
+		if uploadedUrl is not None:
 			ss = PackageScreenshot()
 			ss.package  = package
 			ss.title    = form["title"].data or "Untitled"
-			ss.url      = uploadedPath
+			ss.url      = uploadedUrl
 			ss.approved = package.checkPerm(current_user, Permission.APPROVE_SCREENSHOT)
 			db.session.add(ss)
 

app/blueprints/thumbnails/__init__.py

@@ -25,10 +25,10 @@ from PIL import Image
 ALLOWED_RESOLUTIONS=[(100,67), (270,180), (350,233)]
 
 def mkdir(path):
+	assert path != "" and path is not None
 	if not os.path.isdir(path):
 		os.mkdir(path)
 
-mkdir("app/public/thumbnails/")
 
 def resize_and_crop(img_path, modified_path, size):
 	img = Image.open(img_path)
@@ -65,10 +65,15 @@ def make_thumbnail(img, level):
 
 	w, h = ALLOWED_RESOLUTIONS[level - 1]
 
-	mkdir("app/public/thumbnails/{:d}/".format(level))
+	upload_dir = current_app.config["UPLOAD_DIR"]
+	thumbnail_dir = current_app.config["THUMBNAIL_DIR"]
+	mkdir(thumbnail_dir)
 
-	cache_filepath  = "public/thumbnails/{:d}/{}".format(level, img)
-	source_filepath = "public/uploads/" + img
+	output_dir = os.path.join(thumbnail_dir, str(level))
+	mkdir(output_dir)
 
-	resize_and_crop("app/" + source_filepath, "app/" + cache_filepath, (w, h))
+	cache_filepath  = os.path.join(output_dir, img)
+	source_filepath = os.path.join(upload_dir, img)
+
+	resize_and_crop(source_filepath, cache_filepath, (w, h))
 	return send_file(cache_filepath)

app/blueprints/users/githublogin.py

@@ -65,7 +65,7 @@ def github_authorized(oauth_token):
 			flash("Unable to find an account for that Github user", "error")
 			return redirect(url_for("users.claim"))
 		elif loginUser(userByGithub):
-			if current_user.password is None:
+			if not current_user.hasPassword():
 				return redirect(next_url or url_for("users.set_password", optional=True))
 			else:
 				return redirect(next_url or url_for("homepage.home"))

app/blueprints/users/profile.py

@@ -170,7 +170,7 @@ class SetPasswordForm(FlaskForm):
 @bp.route("/user/set-password/", methods=["GET", "POST"])
 @login_required
 def set_password():
-	if current_user.password is not None:
+	if current_user.hasPassword():
 		return redirect(url_for("user.change_password"))
 
 	form = SetPasswordForm(request.form)
@@ -185,10 +185,11 @@ def set_password():
 			hashed_password = user_manager.hash_password(form.password.data)
 
 			# Change password
-			user_manager.update_password(current_user, hashed_password)
+			current_user.password = hashed_password
+			db.session.commit()
 
 			# Send 'password_changed' email
-			if user_manager.enable_email and user_manager.send_password_changed_email and current_user.email:
+			if user_manager.USER_ENABLE_EMAIL and current_user.email:
 				emails.send_password_changed_email(current_user)
 
 			# Send password_changed signal
@@ -211,7 +212,7 @@ def set_password():
 				task = sendVerifyEmail.delay(newEmail, token)
 				return redirect(url_for("tasks.check", id=task.id, r=url_for("users.profile", username=current_user.username)))
 			else:
-				return redirect(url_for("users.profile", username=current_user.username))
+				return redirect(url_for("user.login"))
 		else:
 			flash("Passwords do not match", "error")
 

app/default_data.py

@@ -0,0 +1,354 @@
+from .models import *
+from .utils import make_flask_user_password
+
+
+def populate(session):
+	admin_user = User("rubenwardy")
+	admin_user.active = True
+	admin_user.password = make_flask_user_password("tuckfrump")
+	admin_user.github_username = "rubenwardy"
+	admin_user.forums_username = "rubenwardy"
+	admin_user.rank = UserRank.ADMIN
+	session.add(admin_user)
+
+	session.add(MinetestRelease("None", 0))
+	session.add(MinetestRelease("0.4.16/17", 32))
+	session.add(MinetestRelease("5.0", 37))
+	session.add(MinetestRelease("5.1", 38))
+
+	tags = {}
+	for tag in ["Inventory", "Mapgen", "Building", \
+			"Mobs and NPCs", "Tools", "Player effects", \
+			"Environment", "Transport", "Maintenance", "Plants and farming", \
+			"PvP", "PvE", "Survival", "Creative", "Puzzle", "Multiplayer", "Singleplayer"]:
+		row = Tag(tag)
+		tags[row.name] = row
+		session.add(row)
+
+	licenses = {}
+	for license in ["GPLv2.1", "GPLv3", "LGPLv2.1", "LGPLv3", "AGPLv2.1", "AGPLv3",
+					"Apache", "BSD 3-Clause", "BSD 2-Clause", "CC0", "CC-BY-SA",
+					"CC-BY", "MIT", "ZLib", "Other (Free)"]:
+		row = License(license)
+		licenses[row.name] = row
+		session.add(row)
+
+	for license in ["CC-BY-NC-SA", "Other (Non-free)"]:
+		row = License(license, False)
+		licenses[row.name] = row
+		session.add(row)
+
+
+def populate_test_data(session):
+	licenses = { x.name : x for x in License.query.all() }
+	tags = { x.name : x for x in Tag.query.all() }
+	admin_user = User.query.filter_by(rank=UserRank.ADMIN).first()
+	v4 = MinetestRelease.query.filter_by(protocol=32).first()
+	v50 = MinetestRelease.query.filter_by(protocol=37).first()
+	v51 = MinetestRelease.query.filter_by(protocol=38).first()
+
+	ez = User("Shara")
+	ez.github_username = "Ezhh"
+	ez.forums_username = "Shara"
+	ez.rank = UserRank.EDITOR
+	session.add(ez)
+
+	not1 = Notification(admin_user, ez, "Awards approved", "/packages/rubenwardy/awards/")
+	session.add(not1)
+
+	jeija = User("Jeija")
+	jeija.github_username = "Jeija"
+	jeija.forums_username = "Jeija"
+	session.add(jeija)
+
+
+	mod = Package()
+	mod.approved = True
+	mod.name = "alpha"
+	mod.title = "Alpha Test"
+	mod.license = licenses["MIT"]
+	mod.media_license = licenses["MIT"]
+	mod.type = PackageType.MOD
+	mod.author = admin_user
+	mod.tags.append(tags["mapgen"])
+	mod.tags.append(tags["environment"])
+	mod.repo = "https://github.com/ezhh/other_worlds"
+	mod.issueTracker = "https://github.com/ezhh/other_worlds/issues"
+	mod.forums = 16015
+	mod.short_desc = "The content library should not be used yet as it is still in alpha"
+	mod.desc = "This is the long desc"
+	session.add(mod)
+
+	rel = PackageRelease()
+	rel.package = mod
+	rel.title = "v1.0.0"
+	rel.url = "https://github.com/ezhh/handholds/archive/master.zip"
+	rel.approved = True
+	session.add(rel)
+
+	mod1 = Package()
+	mod1.approved = True
+	mod1.name = "awards"
+	mod1.title = "Awards"
+	mod1.license = licenses["LGPLv2.1"]
+	mod1.media_license = licenses["MIT"]
+	mod1.type = PackageType.MOD
+	mod1.author = admin_user
+	mod1.tags.append(tags["player_effects"])
+	mod1.repo = "https://github.com/rubenwardy/awards"
+	mod1.issueTracker = "https://github.com/rubenwardy/awards/issues"
+	mod1.forums = 4870
+	mod1.short_desc = "Adds achievements and an API to register new ones."
+	mod1.desc = """
+Majority of awards are back ported from Calinou's old fork in Carbone, under same license.
+
+```
+awards.register_achievement("award_mesefind",{
+    title = "First Mese Find",
+    description = "Found some Mese!",
+    trigger = {
+        type   = "dig",          -- award is given when
+        node   = "default:mese", -- this type of node has been dug
+        target = 1,              -- this number of times
+    },
+})
+```
+"""
+
+	rel = PackageRelease()
+	rel.package = mod1
+	rel.min_rel = v51
+	rel.title = "v1.0.0"
+	rel.url = "https://github.com/rubenwardy/awards/archive/master.zip"
+	rel.approved = True
+	session.add(rel)
+
+	mod2 = Package()
+	mod2.approved = True
+	mod2.name = "mesecons"
+	mod2.title = "Mesecons"
+	mod2.tags.append(tags["tools"])
+	mod2.type = PackageType.MOD
+	mod2.license = licenses["LGPLv3"]
+	mod2.media_license = licenses["MIT"]
+	mod2.author = jeija
+	mod2.repo = "https://github.com/minetest-mods/mesecons/"
+	mod2.issueTracker = "https://github.com/minetest-mods/mesecons/issues"
+	mod2.forums = 628
+	mod2.short_desc = "Mesecons adds everything digital, from all kinds of sensors, switches, solar panels, detectors, pistons, lamps, sound blocks to advanced digital circuitry like logic gates and programmable blocks."
+	mod2.desc = """
+MESECONS by Jeija and contributors
+
+Mezzee-what?
+------------
+[Mesecons](http://mesecons.net/)! They're yellow, they're conductive, and they'll add a whole new dimension to Minetest's gameplay.
+
+Mesecons is a mod for [Minetest](http://minetest.net/) that implements a ton of items related to digital circuitry, such as wires, buttons, lights, and even programmable controllers. Among other things, there are also pistons, solar panels, pressure plates, and note blocks.
+
+Mesecons has a similar goal to Redstone in Minecraft, but works in its own way, with different rules and mechanics.
+
+OK, I want in.
+--------------
+Go get it!
+
+[DOWNLOAD IT NOW](https://github.com/minetest-mods/mesecons/archive/master.zip)
+
+Now go ahead and install it like any other Minetest mod. Don't know how? Check out [the wonderful page about it](http://wiki.minetest.com/wiki/Mods) over at the Minetest Wiki. For your convenience, here's a quick summary:
+
+1. If Mesecons is still in a ZIP file, extract the folder inside to somewhere on the computer.
+2. Make sure that when you open the folder, you can directly find `README.md` in the listing. If you just see another folder, move that folder up one level and delete the old one.
+3. Open up the Minetest mods folder - usually `/mods/`. If you see the `minetest` or folder inside of that, that is your mod folder instead.
+4. Copy the Mesecons folder into the mods folder.
+
+Don't like some parts of Mesecons? Open up the Mesecons folder and delete the subfolder containing the mod you don't want. If you didn't want movestones, for example, all you have to do is delete the `mesecons_movestones` folder and they will no longer be available.
+
+There are no dependencies - it will work right after installing!
+
+How do I use this thing?
+------------------------
+How about a [quick overview video](https://www.youtube.com/watch?v=6kmeQj6iW5k)?
+
+Or maybe a [comprehensive reference](http://mesecons.net/items.html) is your style?
+
+An overview for the very newest of new beginners? How does [this one](http://uberi.mesecons.net/projects/MeseconsBasics/index.html) look?
+
+Want to get more into building? Why not check out the [Mesecons Laboratory](http://uberi.mesecons.net/), a website dedicated to advanced Mesecons builders?
+
+Want to contribute to Mesecons itself? Check out the [source code](https://github.com/minetest-mods/mesecons)!
+
+Who wrote it anyways?
+---------------------
+These awesome people made Mesecons possible!
+
+| Contributor     | Contribution                     |
+| --------------- | -------------------------------- |
+| Hawk777         | Code for VoxelManip caching      |
+| Jat15           | Various tweaks.                  |
+| Jeija           | **Main developer! Everything.**  |
+| Jordach         | Noteblock sounds.                |
+| khonkhortistan  | Code, recipes, textures.         |
+| Kotolegokot     | Nodeboxes for items.             |
+| minerd247       | Textures.                        |
+| Nore/Novatux    | Code.                            |
+| RealBadAngel    | Fixes, improvements.             |
+| sfan5           | Code, recipes, textures.         |
+| suzenako        | Piston sounds.                   |
+| Uberi/Temperest | Code, textures, documentation.   |
+| VanessaE        | Code, recipes, textures, design. |
+| Whiskers75      | Logic gates implementation.      |
+
+There are also a whole bunch of other people helping with everything from code to testing and feedback. Mesecons would also not be possible without their help!
+
+Alright, how can I use it?
+--------------------------
+All textures in this project are licensed under the CC-BY-SA 3.0 (Creative Commons Attribution-ShareAlike 3.0 Generic). That means you can distribute and remix them as much as you want to, under the condition that you give credit to the authors and the project, and that if you remix and release them, they must be under the same or similar license to this one.
+
+All code in this project is licensed under the LGPL version 3 or later. That means you have unlimited freedom to distribute and modify the work however you see fit, provided that if you decide to distribute it or any modified versions of it, you must also use the same license. The LGPL also grants the additional freedom to write extensions for the software and distribute them without the extensions being subject to the terms of the LGPL, although the software itself retains its license.
+
+No warranty is provided, express or implied, for any part of the project.
+
+"""
+
+	session.add(mod1)
+	session.add(mod2)
+
+	mod = Package()
+	mod.approved = True
+	mod.name = "handholds"
+	mod.title = "Handholds"
+	mod.license = licenses["MIT"]
+	mod.media_license = licenses["MIT"]
+	mod.type = PackageType.MOD
+	mod.author = ez
+	mod.tags.append(tags["player_effects"])
+	mod.repo = "https://github.com/ezhh/handholds"
+	mod.issueTracker = "https://github.com/ezhh/handholds/issues"
+	mod.forums = 17069
+	mod.short_desc = "Adds hand holds and climbing thingies"
+	mod.desc = "This is the long desc"
+	session.add(mod)
+
+	rel = PackageRelease()
+	rel.package = mod
+	rel.title = "v1.0.0"
+	rel.max_rel = v4
+	rel.url = "https://github.com/ezhh/handholds/archive/master.zip"
+	rel.approved = True
+	session.add(rel)
+
+	mod = Package()
+	mod.approved = True
+	mod.name = "other_worlds"
+	mod.title = "Other Worlds"
+	mod.license = licenses["MIT"]
+	mod.media_license = licenses["MIT"]
+	mod.type = PackageType.MOD
+	mod.author = ez
+	mod.tags.append(tags["mapgen"])
+	mod.tags.append(tags["environment"])
+	mod.repo = "https://github.com/ezhh/other_worlds"
+	mod.issueTracker = "https://github.com/ezhh/other_worlds/issues"
+	mod.forums = 16015
+	mod.short_desc = "Adds space with asteroids and comets"
+	mod.desc = "This is the long desc"
+	session.add(mod)
+
+	mod = Package()
+	mod.approved = True
+	mod.name = "food"
+	mod.title = "Food"
+	mod.license = licenses["LGPLv2.1"]
+	mod.media_license = licenses["MIT"]
+	mod.type = PackageType.MOD
+	mod.author = admin_user
+	mod.tags.append(tags["player_effects"])
+	mod.repo = "https://github.com/rubenwardy/food/"
+	mod.issueTracker = "https://github.com/rubenwardy/food/issues/"
+	mod.forums = 2960
+	mod.short_desc = "Adds lots of food and an API to manage ingredients"
+	mod.desc = "This is the long desc"
+	session.add(mod)
+
+	mod = Package()
+	mod.approved = True
+	mod.name = "food_sweet"
+	mod.title = "Sweet Foods"
+	mod.license = licenses["CC0"]
+	mod.media_license = licenses["MIT"]
+	mod.type = PackageType.MOD
+	mod.author = admin_user
+	mod.tags.append(tags["player_effects"])
+	mod.repo = "https://github.com/rubenwardy/food_sweet/"
+	mod.issueTracker = "https://github.com/rubenwardy/food_sweet/issues/"
+	mod.forums = 9039
+	mod.short_desc = "Adds sweet food"
+	mod.desc = "This is the long desc"
+	food_sweet = mod
+	session.add(mod)
+
+	game1 = Package()
+	game1.approved = True
+	game1.name = "capturetheflag"
+	game1.title = "Capture The Flag"
+	game1.type = PackageType.GAME
+	game1.license = licenses["LGPLv2.1"]
+	game1.media_license = licenses["MIT"]
+	game1.author = admin_user
+	game1.tags.append(tags["pvp"])
+	game1.tags.append(tags["survival"])
+	game1.tags.append(tags["multiplayer"])
+	game1.repo = "https://github.com/rubenwardy/capturetheflag"
+	game1.issueTracker = "https://github.com/rubenwardy/capturetheflag/issues"
+	game1.forums = 12835
+	game1.short_desc = "Two teams battle to snatch and return the enemy's flag, before the enemy takes their own!"
+	game1.desc = """
+As seen on the Capture the Flag server (minetest.rubenwardy.com:30000)
+
+Uses the CTF PvP Engine.
+"""
+
+	session.add(game1)
+
+	rel = PackageRelease()
+	rel.package = game1
+	rel.title = "v1.0.0"
+	rel.url = "https://github.com/rubenwardy/capturetheflag/archive/master.zip"
+	rel.approved = True
+	session.add(rel)
+
+
+	mod = Package()
+	mod.approved = True
+	mod.name = "pixelbox"
+	mod.title = "PixelBOX Reloaded"
+	mod.license = licenses["CC0"]
+	mod.media_license = licenses["MIT"]
+	mod.type = PackageType.TXP
+	mod.author = admin_user
+	mod.forums = 14132
+	mod.short_desc = "This is an update of the original PixelBOX texture pack by the brillant artist Gambit"
+	mod.desc = "This is the long desc"
+	session.add(mod)
+
+	rel = PackageRelease()
+	rel.package = mod
+	rel.title = "v1.0.0"
+	rel.url = "http://mamadou3.free.fr/Minetest/PixelBOX.zip"
+	rel.approved = True
+	session.add(rel)
+
+	session.commit()
+
+	metas = {}
+	for package in Package.query.filter_by(type=PackageType.MOD).all():
+		meta = None
+		try:
+			meta = metas[package.name]
+		except KeyError:
+			meta = MetaPackage(package.name)
+			session.add(meta)
+			metas[package.name] = meta
+		package.provides.append(meta)
+
+	dep = Dependency(food_sweet, meta=metas["food"])
+	session.add(dep)

app/flatpages/help.md

@@ -4,3 +4,4 @@ title: Help
 * [Ranks and Permissions](ranks_permissions)
 * [Content Ratings and Flags](content_flags)
 * [Reporting Content](reporting)
+* [API](api)

app/flatpages/help/api.md

@@ -0,0 +1,51 @@
+title: API
+
+## Authentication
+
+Not all endpoints require authentication.
+Authentication is done using Bearer tokens:
+
+	Authorization: Bearer YOURTOKEN
+
+You can use the `/api/whoami` to check authentication.
+
+## Endpoints
+
+### Misc
+
+* GET `/api/whoami/` - Json dictionary with the following keys:
+	* `is_authenticated` - True on successful API authentication
+	* `username` - Username of the user authenticated as, null otherwise.
+	* 403 will be thrown on unsupported authentication type, invalid access token, or other errors.
+
+### Packages
+
+* GET `/api/packages/` - See [Package Queries](#package-queries)
+* GET `/api/packages/<username>/<name>/`
+
+### Topics
+
+* GET `/api/topics/` - Supports [Package Queries](#package-queries), and the following two options:
+    * `show_added` - Show topics which exist as packages, default true.
+	* `show_discarded` - Show topics which have been marked as outdated, default false.
+
+### Minetest
+
+* GET `/api/minetest_versions/`
+
+
+## Package Queries
+
+Example:
+
+	/api/packages/?type=mod&type=game&q=mobs+fun&hide=nonfree&hide=gore
+
+Supported query parameters:
+
+* `type` - Package types (`mod`, `game`, `txp`).
+* `q` - Query string
+* `random` - When present, enable random ordering and ignore `sort`.
+* `hide` - Hide content based on [Content Flags](content_flags).
+* `sort` - Sort by (`name`, `views`, `date`, `score`).
+* `order` - Sort ascending (`Asc`) or descending (`desc`).
+* `protocol_version` - Only show packages supported by this Minetest protocol version.

app/flatpages/help/ranks_permissions.md

@@ -219,6 +219,21 @@ title: Ranks and Permissions
 			<th>✓</th> <!-- admin -->
 			<th>✓</th>
 		</tr>
+		<tr>
+			<td>Create Token</td>
+			<th></th> <!-- new -->
+			<th></th>
+			<th>✓</th> <!-- member -->
+			<th></th>
+			<th>✓</th> <!-- trusted member -->
+			<th></th>
+			<th>✓</th> <!-- editor -->
+			<th></th>
+			<th>✓</th> <!-- moderator -->
+			<th>✓<sup>2</sup></th>
+			<th>✓</th> <!-- admin -->
+			<th>✓</th>
+		</tr>
 		<tr>
 			<td>Set Rank</td>
 			<th></th> <!-- new -->

app/models.py

@@ -23,9 +23,8 @@ from urllib.parse import urlparse
 from flask import Flask, url_for
 from flask_sqlalchemy import SQLAlchemy, BaseQuery
 from flask_migrate import Migrate
-from flask_user import login_required, UserManager, UserMixin, SQLAlchemyAdapter
-from sqlalchemy import func
-from sqlalchemy.orm import validates
+from flask_user import login_required, UserManager, UserMixin
+from sqlalchemy import func, CheckConstraint
 from sqlalchemy_searchable import SearchQueryMixin
 from sqlalchemy_utils.types import TSVectorType
 from sqlalchemy_searchable import make_searchable
@@ -79,6 +78,7 @@ class Permission(enum.Enum):
 	CHANGE_AUTHOR      = "CHANGE_AUTHOR"
 	CHANGE_NAME        = "CHANGE_NAME"
 	MAKE_RELEASE       = "MAKE_RELEASE"
+	DELETE_RELEASE     = "DELETE_RELEASE"
 	ADD_SCREENSHOTS    = "ADD_SCREENSHOTS"
 	APPROVE_SCREENSHOT = "APPROVE_SCREENSHOT"
 	APPROVE_RELEASE    = "APPROVE_RELEASE"
@@ -92,6 +92,7 @@ class Permission(enum.Enum):
 	CREATE_THREAD      = "CREATE_THREAD"
 	UNAPPROVE_PACKAGE  = "UNAPPROVE_PACKAGE"
 	TOPIC_DISCARD      = "TOPIC_DISCARD"
+	CREATE_TOKEN       = "CREATE_TOKEN"
 
 	# Only return true if the permission is valid for *all* contexts
 	# See Package.checkPerm for package-specific contexts
@@ -113,7 +114,7 @@ class User(db.Model, UserMixin):
 
 	# User authentication information
 	username     = db.Column(db.String(50, collation="NOCASE"), nullable=False, unique=True, index=True)
-	password     = db.Column(db.String(255), nullable=True)
+	password     = db.Column(db.String(255), nullable=False, server_default="")
 	reset_password_token = db.Column(db.String(100), nullable=False, server_default="")
 
 	rank         = db.Column(db.Enum(UserRank))
@@ -124,7 +125,7 @@ class User(db.Model, UserMixin):
 
 	# User email information
 	email         = db.Column(db.String(255), nullable=True, unique=True)
-	confirmed_at  = db.Column(db.DateTime())
+	email_confirmed_at  = db.Column(db.DateTime())
 
 	# User information
 	profile_pic   = db.Column(db.String(255), nullable=True, server_default=None)
@@ -142,17 +143,21 @@ class User(db.Model, UserMixin):
 	packages      = db.relationship("Package", backref="author", lazy="dynamic")
 	requests      = db.relationship("EditRequest", backref="author", lazy="dynamic")
 	threads       = db.relationship("Thread", backref="author", lazy="dynamic")
+	tokens        = db.relationship("APIToken", backref="owner", lazy="dynamic")
 	replies       = db.relationship("ThreadReply", backref="author", lazy="dynamic")
 
-	def __init__(self, username, active=False, email=None, password=None):
+	def __init__(self, username, active=False, email=None, password=""):
 		self.username = username
-		self.confirmed_at = datetime.datetime.now() - datetime.timedelta(days=6000)
+		self.email_confirmed_at = datetime.datetime.now() - datetime.timedelta(days=6000)
 		self.display_name = username
 		self.active = active
 		self.email = email
 		self.password = password
 		self.rank = UserRank.NOT_JOINED
 
+	def hasPassword(self):
+		return self.password != ""
+
 	def canAccessTodoList(self):
 		return Permission.APPROVE_NEW.check(self) or \
 				Permission.APPROVE_RELEASE.check(self) or \
@@ -183,6 +188,11 @@ class User(db.Model, UserMixin):
 			return user.rank.atLeast(UserRank.MODERATOR)
 		elif perm == Permission.CHANGE_EMAIL:
 			return user == self or (user.rank.atLeast(UserRank.MODERATOR) and user.rank.atLeast(self.rank))
+		elif perm == Permission.CREATE_TOKEN:
+			if user == self:
+				return user.rank.atLeast(UserRank.MEMBER)
+			else:
+				return user.rank.atLeast(UserRank.MODERATOR) and user.rank.atLeast(self.rank)
 		else:
 			raise Exception("Permission {} is not related to users".format(perm.name))
 
@@ -196,6 +206,13 @@ class User(db.Model, UserMixin):
 		return Thread.query.filter_by(author=self) \
 			.filter(Thread.created_at > hour_ago).count() < 2
 
+	def __eq__(self, other):
+		if not self.is_authenticated or not other.is_authenticated:
+			return False
+
+		assert self.id > 0
+		return self.id == other.id
+
 class UserEmailVerification(db.Model):
 	id      = db.Column(db.Integer, primary_key=True)
 	user_id = db.Column(db.Integer, db.ForeignKey("user.id"))
@@ -303,7 +320,7 @@ class Dependency(db.Model):
 	package         = db.relationship("Package", foreign_keys=[package_id])
 	meta_package_id = db.Column(db.Integer, db.ForeignKey("meta_package.id"), nullable=True)
 	optional        = db.Column(db.Boolean, nullable=False, default=False)
-	__table_args__  = (db.UniqueConstraint('depender_id', 'package_id', 'meta_package_id', name='_dependency_uc'), )
+	__table_args__  = (db.UniqueConstraint("depender_id", "package_id", "meta_package_id", name="_dependency_uc"), )
 
 	def __init__(self, depender=None, package=None, meta=None):
 		if depender is None:
@@ -370,14 +387,17 @@ class Package(db.Model):
 
 	# Basic details
 	author_id    = db.Column(db.Integer, db.ForeignKey("user.id"))
-	name         = db.Column(db.String(100), nullable=False)
+	name         = db.Column(db.Unicode(100), nullable=False)
 	title        = db.Column(db.Unicode(100), nullable=False)
 	short_desc   = db.Column(db.Unicode(200), nullable=False)
 	desc         = db.Column(db.UnicodeText, nullable=True)
 	type         = db.Column(db.Enum(PackageType))
 	created_at   = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow)
 
-	search_vector = db.Column(TSVectorType("title", "short_desc", "desc"))
+	name_valid = db.CheckConstraint("name ~* '^[a-z0-9_]+$'")
+
+	search_vector = db.Column(TSVectorType("name", "title", "short_desc", "desc", \
+			weights={ "name": "A", "title": "B", "short_desc": "C", "desc": "D" }))
 
 	license_id   = db.Column(db.Integer, db.ForeignKey("license.id"), nullable=False, default=1)
 	license      = db.relationship("License", foreign_keys=[license_id])
@@ -462,8 +482,7 @@ class Package(db.Model):
 			"short_description": self.short_desc,
 			"type": self.type.toName(),
 			"release": release and release.id,
-			"thumbnail": (base_url + tnurl) if tnurl is not None else None,
-			"score": round(self.score * 10) / 10
+			"thumbnail": (base_url + tnurl) if tnurl is not None else None
 		}
 
 	def getAsDictionary(self, base_url, version=None, protonum=None):
@@ -698,8 +717,9 @@ class MinetestRelease(db.Model):
 	name     = db.Column(db.String(100), unique=True, nullable=False)
 	protocol = db.Column(db.Integer, nullable=False, default=0)
 
-	def __init__(self, name=None):
+	def __init__(self, name=None, protocol=0):
 		self.name = name
+		self.protocol = protocol
 
 	def getActual(self):
 		return None if self.name == "None" else self
@@ -723,6 +743,8 @@ class PackageRelease(db.Model):
 	max_rel_id = db.Column(db.Integer, db.ForeignKey("minetest_release.id"), nullable=True, server_default=None)
 	max_rel    = db.relationship("MinetestRelease", foreign_keys=[max_rel_id])
 
+	# If the release is approved, then the task_id must be null and the url must be present
+	CK_approval_valid = db.CheckConstraint("not approved OR (task_id IS NULL AND (url = '') IS NOT FALSE)")
 
 	def getEditURL(self):
 		return url_for("packages.edit_release",
@@ -730,6 +752,12 @@ class PackageRelease(db.Model):
 				name=self.package.name,
 				id=self.id)
 
+	def getDeleteURL(self):
+		return url_for("packages.delete_release",
+				author=self.package.author.username,
+				name=self.package.name,
+				id=self.id)
+
 	def getDownloadURL(self):
 		return url_for("packages.download_release",
 				author=self.package.author.username,
@@ -745,17 +773,47 @@ class PackageRelease(db.Model):
 				not self.package.checkPerm(user, Permission.APPROVE_RELEASE):
 			return False
 
-		assert(self.task_id is None and self.url is not None and self.url != "")
+		assert self.task_id is None and self.url is not None and self.url != ""
 
 		self.approved = True
 		return True
 
+	def checkPerm(self, user, perm):
+		if not user.is_authenticated:
+			return False
 
-class PackageReview(db.Model):
-	id         = db.Column(db.Integer, primary_key=True)
-	package_id = db.Column(db.Integer, db.ForeignKey("package.id"))
-	thread_id  = db.Column(db.Integer, db.ForeignKey("thread.id"), nullable=False)
-	recommend  = db.Column(db.Boolean, nullable=False, default=True)
+		if type(perm) == str:
+			perm = Permission[perm]
+		elif type(perm) != Permission:
+			raise Exception("Unknown permission given to PackageRelease.checkPerm()")
+
+		isOwner = user == self.package.author
+
+		if perm == Permission.DELETE_RELEASE:
+			if user.rank.atLeast(UserRank.ADMIN):
+				return True
+
+			if not (isOwner or user.rank.atLeast(UserRank.EDITOR)):
+				return False
+
+			if not self.package.approved or self.task_id is not None:
+				return True
+
+			count = PackageRelease.query \
+					.filter_by(package_id=self.package_id) \
+					.filter(PackageRelease.id > self.id) \
+					.count()
+
+			return count > 0
+		else:
+			raise Exception("Permission {} is not related to releases".format(perm.name))
+
+
+# class PackageReview(db.Model):
+# 	id         = db.Column(db.Integer, primary_key=True)
+# 	package_id = db.Column(db.Integer, db.ForeignKey("package.id"))
+# 	thread_id  = db.Column(db.Integer, db.ForeignKey("thread.id"), nullable=False)
+# 	recommend  = db.Column(db.Boolean, nullable=False, default=True)
 
 
 class PackageScreenshot(db.Model):
@@ -776,6 +834,16 @@ class PackageScreenshot(db.Model):
 		return self.url.replace("/uploads/", ("/thumbnails/{:d}/").format(level))
 
 
+class APIToken(db.Model):
+	id           = db.Column(db.Integer, primary_key=True)
+	access_token = db.Column(db.String(34), unique=True)
+	name         = db.Column(db.String(100), nullable=False)
+	owner_id     = db.Column(db.Integer, db.ForeignKey("user.id"), nullable=False)
+	created_at   = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow)
+
+	def canOperateOnPackage(self, package):
+		return packages.count() == 0 or package in packages
+
 
 class EditRequest(db.Model):
 	id           = db.Column(db.Integer, primary_key=True)
@@ -986,5 +1054,4 @@ class ForumTopic(db.Model):
 
 
 # Setup Flask-User
-db_adapter = SQLAlchemyAdapter(db, User)        # Register the User model
-user_manager = UserManager(db_adapter, app)     # Initialize Flask-User
+user_manager = UserManager(app, db, User)

app/querybuilder.py

@@ -62,7 +62,7 @@ class QueryBuilder:
 			query = query.filter(Package.type.in_(self.types))
 
 		if self.search:
-			query = query.search(self.search)
+			query = query.search(self.search, sort=True)
 
 		if self.random:
 			query = query.order_by(func.random())

app/scss/custom.scss

@@ -4,7 +4,7 @@
 @import "comments.scss";
 
 .dropdown-menu {
-    margin-top: 0;
+	margin-top: 0;
 }
 
 .dropdown:hover .dropdown-menu {
@@ -16,42 +16,48 @@
 }
 
 #alerts {
-    display: block;
-    list-style: none;
-    position: fixed;
-    bottom: 0;
-    left:0;
-    right:0;
-    margin: 0;
-    padding:0;
-    z-index: 1000;
+	display: block;
+	list-style: none;
+	position: fixed;
+	bottom: 0;
+	left:0;
+	right:0;
+	margin: 0;
+	padding:0;
+	z-index: 1000;
 }
 
 #alerts li {
-    list-style: none;
+	list-style: none;
 }
 
 .jumbotron {
-    background-size: cover;
-    background-repeat: no-repeat;
-    background-position: center;
+	background-size: cover;
+	background-repeat: no-repeat;
+	background-position: center;
+
+	.btn-outline-secondary {
+		color: #eee;
+		border-color: #666;
+		background: rgba(102, 102, 102, 0.3);
+	}
 }
 
 .alert .btn {
-    text-decoration: none;
+	text-decoration: none;
 }
 
 .card .table {
-    margin-bottom: 0;
+	margin-bottom: 0;
 }
 
 .btn-download {
-    color: #fff;
-    background-color: #00b05c;
-    border-color: #00b05c;
+	color: #fff;
+	background-color: #00b05c;
+	border-color: #00b05c;
 }
 
 .btn-download:focus, .btn-download.focus {
-    -webkit-box-shadow: 0 0 0 0.2rem rgba(231, 76, 60, 0.5);
-    box-shadow: 0 0 0 0.2rem rgba(231, 76, 60, 0.5);
+	-webkit-box-shadow: 0 0 0 0.2rem rgba(231, 76, 60, 0.5);
+	box-shadow: 0 0 0 0.2rem rgba(231, 76, 60, 0.5);
 }

app/tasks/__init__.py

@@ -77,4 +77,4 @@ CELERYBEAT_SCHEDULE = {
 }
 celery.conf.beat_schedule = CELERYBEAT_SCHEDULE
 
-from . import importtasks, forumtasks, emails
+from . import importtasks, forumtasks, emails, pkgtasks

app/tasks/importtasks.py

@@ -17,15 +17,19 @@
 
 import flask, json, os, git, tempfile, shutil, gitdb
 from git import GitCommandError
+from git_archive_all import GitArchiver
 from flask_sqlalchemy import SQLAlchemy
 from urllib.error import HTTPError
 import urllib.request
 from urllib.parse import urlparse, quote_plus, urlsplit
+from zipfile import ZipFile
+
 from app import app
 from app.models import *
 from app.tasks import celery, TaskError
 from app.utils import randomString
-
+from .minetestcheck import build_tree, MinetestCheckError, ContentType
+from .minetestcheck.config import parse_conf
 
 class GithubURLMaker:
 	def __init__(self, url):
@@ -126,173 +130,22 @@ def findModInfo(author, name, link):
 
 	return None
 
-
-def parseConf(string):
-	retval = {}
-	for line in string.split("\n"):
-		idx = line.find("=")
-		if idx > 0:
-			key   = line[:idx].strip()
-			value = line[idx+1:].strip()
-			retval[key] = value
-
-	return retval
-
-
-class PackageTreeNode:
-	def __init__(self, baseDir, author=None, repo=None, name=None):
-		print("Scanning " + baseDir)
-		self.baseDir  = baseDir
-		self.author   = author
-		self.name	 = name
-		self.repo	 = repo
-		self.meta	 = None
-		self.children = []
-
-		# Detect type
-		type = None
-		is_modpack = False
-		if os.path.isfile(baseDir + "/game.conf"):
-			type = PackageType.GAME
-		elif os.path.isfile(baseDir + "/init.lua"):
-			type = PackageType.MOD
-		elif os.path.isfile(baseDir + "/modpack.txt") or \
-				os.path.isfile(baseDir + "/modpack.conf"):
-			type = PackageType.MOD
-			is_modpack = True
-		elif os.path.isdir(baseDir + "/mods"):
-			type = PackageType.GAME
-		elif os.listdir(baseDir) == []:
-			# probably a submodule
-			return
-		else:
-			raise TaskError("Unable to detect package type!")
-
-		self.type = type
-		self.readMetaFiles()
-
-		if self.type == PackageType.GAME:
-			self.addChildrenFromModDir(baseDir + "/mods")
-		elif is_modpack:
-			self.addChildrenFromModDir(baseDir)
-
-
-	def readMetaFiles(self):
-		result = {}
-
-		# .conf file
-		try:
-			with open(self.baseDir + "/mod.conf", "r") as myfile:
-				conf = parseConf(myfile.read())
-				for key in ["name", "description", "title", "depends", "optional_depends"]:
-					try:
-						result[key] = conf[key]
-					except KeyError:
-						pass
-		except IOError:
-			print("description.txt does not exist!")
-
-		# description.txt
-		if not "description" in result:
-			try:
-				with open(self.baseDir + "/description.txt", "r") as myfile:
-					result["description"] = myfile.read()
-			except IOError:
-				print("description.txt does not exist!")
-
-		# depends.txt
-		import re
-		pattern = re.compile("^([a-z0-9_]+)\??$")
-		if not "depends" in result and not "optional_depends" in result:
-			try:
-				with open(self.baseDir + "/depends.txt", "r") as myfile:
-					contents = myfile.read()
-					soft = []
-					hard = []
-					for line in contents.split("\n"):
-						line = line.strip()
-						if pattern.match(line):
-							if line[len(line) - 1] == "?":
-								soft.append( line[:-1])
-							else:
-								hard.append(line)
-
-					result["depends"] = hard
-					result["optional_depends"] = soft
-
-			except IOError:
-				print("depends.txt does not exist!")
-
-		else:
-			if "depends" in result:
-				result["depends"] = [x.strip() for x in result["depends"].split(",")]
-			if "optional_depends" in result:
-				result["optional_depends"] = [x.strip() for x in result["optional_depends"].split(",")]
-
-
-		# Calculate Title
-		if "name" in result and not "title" in result:
-			result["title"] = result["name"].replace("_", " ").title()
-
-		# Calculate short description
-		if "description" in result:
-			desc = result["description"]
-			idx = desc.find(".") + 1
-			cutIdx = min(len(desc), 200 if idx < 5 else idx)
-			result["short_description"] = desc[:cutIdx]
-
-		# Get forum ID
-		info = findModInfo(self.author, result.get("name"), self.repo)
-		if info is not None:
-			result["forumId"] = info.get("topicId")
-
-		if "name" in result:
-			self.name = result["name"]
-			del result["name"]
-
-		self.meta = result
-
-	def addChildrenFromModDir(self, dir):
-		for entry in next(os.walk(dir))[1]:
-			path = dir + "/" + entry
-			if not entry.startswith('.') and os.path.isdir(path):
-				self.children.append(PackageTreeNode(path, name=entry))
-
-
-	def fold(self, attr, key=None, acc=None):
-		if acc is None:
-			acc = set()
-
-		if self.meta is None:
-			return acc
-
-		at = getattr(self, attr)
-		value = at if key is None else at.get(key)
-
-		if isinstance(value, list):
-			acc |= set(value)
-		elif value is not None:
-			acc.add(value)
-
-		for child in self.children:
-			child.fold(attr, key, acc)
-
-		return acc
-
-	def get(self, key):
-		return self.meta.get(key)
-
 def generateGitURL(urlstr):
 	scheme, netloc, path, query, frag = urlsplit(urlstr)
 
 	return "http://:@" + netloc + path + query
 
+
+def getTempDir():
+	return os.path.join(tempfile.gettempdir(), randomString(10))
+
+
 # Clones a repo from an unvalidated URL.
 # Returns a tuple of path and repo on sucess.
 # Throws `TaskError` on failure.
 # Caller is responsible for deleting returned directory.
 def cloneRepo(urlstr, ref=None, recursive=False):
-	gitDir = tempfile.gettempdir() + "/" + randomString(10)
+	gitDir = getTempDir()
 
 	err = None
 	try:
@@ -322,7 +175,12 @@ def cloneRepo(urlstr, ref=None, recursive=False):
 @celery.task()
 def getMeta(urlstr, author):
 	gitDir, _ = cloneRepo(urlstr, recursive=True)
-	tree = PackageTreeNode(gitDir, author=author, repo=urlstr)
+
+	try:
+		tree = build_tree(gitDir, author=author, repo=urlstr)
+	except MinetestCheckError as err:
+		raise TaskError(str(err))
+
 	shutil.rmtree(gitDir)
 
 	result = {}
@@ -371,6 +229,39 @@ def makeVCSReleaseFromGithub(id, branch, release, url):
 	return release.url
 
 
+@celery.task(bind=True)
+def checkZipRelease(self, id, path):
+	release = PackageRelease.query.get(id)
+	if release is None:
+		raise TaskError("No such release!")
+	elif release.package is None:
+		raise TaskError("No package attached to release")
+
+	temp = getTempDir()
+	try:
+		with ZipFile(path, 'r') as zip_ref:
+			zip_ref.extractall(temp)
+
+		try:
+			tree = build_tree(temp, expected_type=ContentType[release.package.type.name], \
+				author=release.package.author.username, name=release.package.name)
+		except MinetestCheckError as err:
+			if "Fails validation" not in release.title:
+				release.title += " (Fails validation)"
+
+			release.task_id = self.request.id
+			release.approved = False
+			db.session.commit()
+
+			raise TaskError(str(err))
+
+		release.task_id = None
+		release.approve(release.package.author)
+		db.session.commit()
+
+	finally:
+		shutil.rmtree(temp)
+
 
 @celery.task()
 def makeVCSRelease(id, branch):
@@ -380,29 +271,37 @@ def makeVCSRelease(id, branch):
 	elif release.package is None:
 		raise TaskError("No package attached to release")
 
-	urlmaker = None
-	url = urlparse(release.package.repo)
-	if url.netloc == "github.com":
-		return makeVCSReleaseFromGithub(id, branch, release, url)
-	else:
-		gitDir, repo = cloneRepo(release.package.repo, ref=branch, recursive=True)
+	# url = urlparse(release.package.repo)
+	# if url.netloc == "github.com":
+	# 	return makeVCSReleaseFromGithub(id, branch, release, url)
 
-		try:
-			filename = randomString(10) + ".zip"
-			destPath = os.path.join("app/public/uploads", filename)
-			with open(destPath, "wb") as fp:
-				repo.archive(fp, format="zip")
+	gitDir, repo = cloneRepo(release.package.repo, ref=branch, recursive=True)
 
-			release.url         = "/uploads/" + filename
-			release.task_id     = None
-			release.commit_hash = repo.head.object.hexsha
-			release.approve(release.package.author)
-			print(release.url)
-			db.session.commit()
+	try:
+		tree = build_tree(gitDir, expected_type=ContentType[release.package.type.name], \
+			author=release.package.author.username, name=release.package.name)
+	except MinetestCheckError as err:
+		raise TaskError(str(err))
 
-			return release.url
-		finally:
-			shutil.rmtree(gitDir)
+	try:
+		filename = randomString(10) + ".zip"
+		destPath = os.path.join(app.config["UPLOAD_DIR"], filename)
+
+		assert(not os.path.isfile(destPath))
+		archiver = GitArchiver(force_sub=True, main_repo_abspath=gitDir)
+		archiver.create(destPath)
+		assert(os.path.isfile(destPath))
+
+		release.url         = "/uploads/" + filename
+		release.task_id     = None
+		release.commit_hash = repo.head.object.hexsha
+		release.approve(release.package.author)
+		print(release.url)
+		db.session.commit()
+
+		return release.url
+	finally:
+		shutil.rmtree(gitDir)
 
 @celery.task()
 def importRepoScreenshot(id):
@@ -424,7 +323,7 @@ def importRepoScreenshot(id):
 			sourcePath = gitDir + "/screenshot." + ext
 			if os.path.isfile(sourcePath):
 				filename = randomString(10) + "." + ext
-				destPath = os.path.join("app/public/uploads", filename)
+				destPath = os.path.join(app.config["UPLOAD_DIR"], filename)
 				shutil.copyfile(sourcePath, destPath)
 
 				ss = PackageScreenshot()
@@ -461,7 +360,7 @@ def getDepends(package):
 	#
 	try:
 		contents = urllib.request.urlopen(urlmaker.getModConfURL()).read().decode("utf-8")
-		conf = parseConf(contents)
+		conf = parse_conf(contents)
 		for key in ["depends", "optional_depends"]:
 			try:
 				result[key] = conf[key]

app/tasks/minetestcheck/__init__.py

@@ -0,0 +1,48 @@
+from enum import Enum
+
+class MinetestCheckError(Exception):
+	def __init__(self, value):
+		self.value = value
+	def __str__(self):
+		return repr("Error validating package: " + self.value)
+
+class ContentType(Enum):
+	UNKNOWN = "unknown"
+	MOD = "mod"
+	MODPACK = "modpack"
+	GAME = "game"
+	TXP = "texture pack"
+
+	def isModLike(self):
+		return self == ContentType.MOD or self == ContentType.MODPACK
+
+	def validate_same(self, other):
+		"""
+		Whether or not `other` is an acceptable type for this
+		"""
+		assert(other)
+
+		if self == ContentType.MOD:
+			if not other.isModLike():
+				raise MinetestCheckError("expected a mod or modpack, found " + other.value)
+
+		elif self == ContentType.TXP:
+			if other != ContentType.UNKNOWN and other != ContentType.TXP:
+				raise MinetestCheckError("expected a " + self.value + ", found a " + other.value)
+
+		elif other != self:
+			raise MinetestCheckError("expected a " + self.value + ", found a " + other.value)
+
+
+from .tree import PackageTreeNode, get_base_dir
+
+def build_tree(path, expected_type=None, author=None, repo=None, name=None):
+	path = get_base_dir(path)
+
+	root = PackageTreeNode(path, "/", author=author, repo=repo, name=name)
+	assert(root)
+
+	if expected_type:
+		expected_type.validate_same(root.type)
+
+	return root

app/tasks/minetestcheck/config.py

@@ -0,0 +1,10 @@
+def parse_conf(string):
+	retval = {}
+	for line in string.split("\n"):
+		idx = line.find("=")
+		if idx > 0:
+			key   = line[:idx].strip()
+			value = line[idx+1:].strip()
+			retval[key] = value
+
+	return retval

app/tasks/minetestcheck/tree.py

@@ -0,0 +1,162 @@
+import os
+from . import MinetestCheckError, ContentType
+from .config import parse_conf
+
+def get_base_dir(path):
+	if not os.path.isdir(path):
+		raise IOError("Expected dir")
+
+	root, subdirs, files = next(os.walk(path))
+	if len(subdirs) == 1 and len(files) == 0:
+		return get_base_dir(path + "/" + subdirs[0])
+	else:
+		return path
+
+
+def detect_type(path):
+	if os.path.isfile(path + "/game.conf"):
+		return ContentType.GAME
+	elif os.path.isfile(path + "/init.lua"):
+		return ContentType.MOD
+	elif os.path.isfile(path + "/modpack.txt") or \
+			os.path.isfile(path + "/modpack.conf"):
+		return ContentType.MODPACK
+	elif os.path.isdir(path + "/mods"):
+		return ContentType.GAME
+	elif os.path.isfile(path + "/texture_pack.conf"):
+		return ContentType.TXP
+	else:
+		return ContentType.UNKNOWN
+
+
+class PackageTreeNode:
+	def __init__(self, baseDir, relative, author=None, repo=None, name=None):
+		print(baseDir)
+		self.baseDir  = baseDir
+		self.relative = relative
+		self.author   = author
+		self.name	 = name
+		self.repo	 = repo
+		self.meta	 = None
+		self.children = []
+
+		# Detect type
+		self.type = detect_type(baseDir)
+		self.read_meta()
+
+		if self.type == ContentType.GAME:
+			if not os.path.isdir(baseDir + "/mods"):
+				raise MinetestCheckError(("game at {} does not have a mods/ folder").format(self.relative))
+			self.add_children_from_mod_dir(baseDir + "/mods")
+		elif self.type == ContentType.MODPACK:
+			self.add_children_from_mod_dir(baseDir)
+
+
+	def read_meta(self):
+		result = {}
+
+		# .conf file
+		try:
+			with open(self.baseDir + "/mod.conf", "r") as myfile:
+				conf = parse_conf(myfile.read())
+				for key in ["name", "description", "title", "depends", "optional_depends"]:
+					try:
+						result[key] = conf[key]
+					except KeyError:
+						pass
+		except IOError:
+			pass
+
+		# description.txt
+		if not "description" in result:
+			try:
+				with open(self.baseDir + "/description.txt", "r") as myfile:
+					result["description"] = myfile.read()
+			except IOError:
+				pass
+
+		# depends.txt
+		import re
+		pattern = re.compile("^([a-z0-9_]+)\??$")
+		if not "depends" in result and not "optional_depends" in result:
+			try:
+				with open(self.baseDir + "/depends.txt", "r") as myfile:
+					contents = myfile.read()
+					soft = []
+					hard = []
+					for line in contents.split("\n"):
+						line = line.strip()
+						if pattern.match(line):
+							if line[len(line) - 1] == "?":
+								soft.append( line[:-1])
+							else:
+								hard.append(line)
+
+					result["depends"] = hard
+					result["optional_depends"] = soft
+
+			except IOError:
+				pass
+
+		else:
+			if "depends" in result:
+				result["depends"] = [x.strip() for x in result["depends"].split(",")]
+			if "optional_depends" in result:
+				result["optional_depends"] = [x.strip() for x in result["optional_depends"].split(",")]
+
+
+		# Calculate Title
+		if "name" in result and not "title" in result:
+			result["title"] = result["name"].replace("_", " ").title()
+
+		# Calculate short description
+		if "description" in result:
+			desc = result["description"]
+			idx = desc.find(".") + 1
+			cutIdx = min(len(desc), 200 if idx < 5 else idx)
+			result["short_description"] = desc[:cutIdx]
+
+		if "name" in result:
+			self.name = result["name"]
+			del result["name"]
+
+		self.meta = result
+
+	def add_children_from_mod_dir(self, dir):
+		for entry in next(os.walk(dir))[1]:
+			path = os.path.join(dir, entry)
+			if not entry.startswith('.') and os.path.isdir(path):
+				child = PackageTreeNode(path, self.relative + entry + "/", name=entry)
+				if not child.type.isModLike():
+					raise MinetestCheckError(("Expecting mod or modpack, found {} at {} inside {}") \
+							.format(child.type.value, child.relative, self.type.value))
+
+				self.children.append(child)
+
+
+	def fold(self, attr, key=None, acc=None):
+		if acc is None:
+			acc = set()
+
+		if self.meta is None:
+			return acc
+
+		at = getattr(self, attr)
+		value = at if key is None else at.get(key)
+
+		if isinstance(value, list):
+			acc |= set(value)
+		elif value is not None:
+			acc.add(value)
+
+		for child in self.children:
+			child.fold(attr, key, acc)
+
+		return acc
+
+	def get(self, key):
+		return self.meta.get(key)
+
+	def validate(self):
+		for child in self.children:
+			child.validate()

app/tasks/phpbbparser.py

@@ -121,7 +121,7 @@ def parseForumListPage(id, page, out, extra=None):
 
 		if id in out:
 			print("   - got {} again, title: {}".format(id, title))
-			assert(title == out[id]['title'])
+			assert title == out[id]['title']
 			return False
 
 		row = {

app/tasks/pkgtasks.py

@@ -15,9 +15,9 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 
-from app.models import Package, PackageRelease
+from app.models import Package
 from app.tasks import celery
 
 @celery.task()
 def updatePackageScores():
-	Package.query.update({ "score": PackageRelease.score * 0.8 })
+	Package.query.update({ "score": Package.score * 0.8 })

app/templates/admin/list.html

@@ -20,6 +20,7 @@
 			<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
 			<select name="action">
 				<option value="delstuckreleases" selected>Delete stuck releases</option>
+				<option value="checkreleases">Validate all Zip releases</option>
 				<option value="importmodlist">Import forum topics</option>
 				<option value="recalcscores">Recalculate package scores</option>
 				<option value="checkusers">Check forum users</option>

app/templates/api/create_edit_token.html

@@ -0,0 +1,53 @@
+{% extends "base.html" %}
+
+{% block title %}
+	{% if token %}
+		{{ _("Edit - %(name)s", name=token.name) }}
+	{% else %}
+		{{ _("Create API Token") }}
+	{% endif %}
+{% endblock %}
+
+{% from "macros/forms.html" import render_field, render_submit_field, render_radio_field %}
+
+{% block content %}
+	{% if token %}
+		<form class="float-right" method="POST" action="{{ url_for('api.delete_token', username=token.owner.username, id=token.id) }}">
+			<input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
+			<input class="btn btn-danger" type="submit" value="Delete">
+		</form>
+	{% endif %}
+
+	<h1 class="mt-0">{{ self.title() }}</h1>
+
+	<div class="alert alert-warning">
+		{{ _("Use carefully, as you may be held responsible for any damage caused by rogue scripts") }}
+	</div>
+
+	{% if token %}
+		<div class="card mb-3">
+			<div class="card-header">{{ _("Access Token") }}</div>
+			<div class="card-body">
+				<p>
+					For security reasons, access tokens will only be shown once.
+					Reset the token if it is lost.
+				</p>
+				{% if access_token %}
+					<input class="form-control my-3" type="text" readonly value="{{ access_token }}" class="form-control">
+				{% endif %}
+				<form method="POST" action="{{ url_for('api.reset_token', username=token.owner.username, id=token.id) }}">
+					<input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
+					<input class="btn btn-primary" type="submit" value="Reset">
+				</form>
+			</div>
+		</div>
+	{% endif %}
+
+	<form method="POST" action="" enctype="multipart/form-data">
+		{{ form.hidden_tag() }}
+
+		{{ render_field(form.name, placeholder="Human readable") }}
+
+		{{ render_submit_field(form.submit) }}
+	</form>
+{% endblock %}

app/templates/api/list_tokens.html

@@ -0,0 +1,23 @@
+{% extends "base.html" %}
+
+{% block title %}
+	{{ _("List tokens for %(username)s", username=user.username) }}
+{% endblock %}
+
+
+{% block content %}
+	<a class="btn btn-primary float-right" href="{{ url_for('api.create_edit_token', username=user.username) }}">Create</a>
+	<h1 class="mt-0">{{ self.title() }}</h1>
+
+	<ul>
+		{% for token in user.tokens %}
+			<li>
+				<a href="{{ url_for('api.create_edit_token', username=user.username, id=token.id) }}">{{ token.name }}</a>
+			</li>
+		{% else %}
+			<li>
+				<i>No tokens created</i>
+			</li>
+		{% endfor %}
+	</ul>
+{% endblock %}

app/templates/base.html

@@ -7,7 +7,7 @@
 	<meta name="viewport" content="width=device-width, initial-scale=1">
 	<title>{% block title %}title{% endblock %} - {{ config.USER_APP_NAME }}</title>
 	<link rel="stylesheet" type="text/css" href="/static/bootstrap.css">
-	<link rel="stylesheet" type="text/css" href="/static/custom.css?v=7">
+	<link rel="stylesheet" type="text/css" href="/static/custom.css?v=8">
 	<link rel="search" type="application/opensearchdescription+xml" href="/static/opensearch.xml" title="ContentDB" />
 	<link rel="shortcut icon" href="/favicon-16.png" sizes="16x16">
 	<link rel="icon" href="/favicon-128.png" sizes="128x128">

app/templates/flask_user/login.html

@@ -15,7 +15,7 @@ Sign in
 				{{ form.hidden_tag() }}
 
 				{# Username or Email field #}
-				{% set field = form.username if user_manager.enable_username else form.email %}
+				{% set field = form.username if user_manager.USER_ENABLE_USERNAME else form.email %}
 				<div class="form-group {% if field.errors %}has-error{% endif %}">
 					{# Label on left, "New here? Register." on right #}
 					<label for="{{ field.id }}" class="control-label">{{ field.label.text }}</label>
@@ -31,7 +31,7 @@ Sign in
 				{% set field = form.password %}
 				<div class="form-group {% if field.errors %}has-error{% endif %}">
 					<label for="{{ field.id }}" class="control-label">{{ field.label.text }}
-					{% if user_manager.enable_forgot_password %}
+					{% if user_manager.USER_ENABLE_FORGOT_PASSWORD %}
 						<a href="{{ url_for('user.forgot_password') }}" tabindex='195'>
 							[{%trans%}Forgot My Password{%endtrans%}]</a>
 					{% endif %}
@@ -45,7 +45,7 @@ Sign in
 				</div>
 
 				{# Remember me #}
-				{% if user_manager.enable_remember_me %}
+				{% if user_manager.USER_ENABLE_REMEMBER_ME %}
 				{{ render_checkbox_field(login_form.remember_me, tabindex=130) }}
 				{% endif %}
 

app/templates/macros/packagegridtile.html

@@ -40,8 +40,10 @@
 		{% else %}
 			<li><i>No packages available</i></ul>
 		{% endfor %}
-		{% for i in range(4) %}
-			<li class="packagetile flex-fill"></li>
-		{% endfor %}
+		{% if packages %}
+			{% for i in range(4) %}
+				<li class="packagetile flex-fill"></li>
+			{% endfor %}
+		{% endif %}
 	</ul>
 {% endmacro %}

app/templates/packages/release_edit.html

@@ -5,6 +5,7 @@
 {% endblock %}
 
 {% block content %}
+	<h2>{{ _("Edit Release") }}</h2>
 	{% from "macros/forms.html" import render_field, render_submit_field, render_checkbox_field %}
 	<form method="POST" action="">
 		{{ form.hidden_tag() }}
@@ -59,6 +60,22 @@
 
 		{{ render_submit_field(form.submit) }}
 	</form>
+
+	<h2 class="mt-5">{{ _("Delete Release") }}</h2>
+
+	{% if release.checkPerm(current_user, "DELETE_RELEASE") %}
+		<form method="POST" action="{{ release.getDeleteURL() }}" class="alert alert-secondary mb-5">
+			<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
+			<input class="btn btn-sm btn-danger float-right" type="submit" value="{{ _('Delete') }}">
+			<b>{{ _("This is permanent.") }}</b>
+			{{ _("Any associated uploads will not be deleted immediately, but the release will no longer be listed.") }}
+			<div style="clear:both;"></div>
+		</form>
+	{% else %}
+		<div class="alert alert-secondary mb-5">
+			{{ _("You cannot delete the latest release; please create a newer one first.") }}
+		</div>
+	{% endif %}
 {% endblock %}
 
 {% block scriptextra %}

app/templates/packages/view.html

@@ -23,14 +23,14 @@
 			</p>
 
 			<div class="row" style="margin-top: 2rem;">
-				<div class="col">
+				<div class="col text-secondary">
 					{{ package.getDownloadCount() }} downloads
 				</div>
 				<div class="btn-group-horizontal col-md-auto">
-					{% if package.repo %}<a class="btn btn-secondary" href="{{ package.repo }}">View Source</a>{% endif %}
-					{% if package.forums %}<a class="btn btn-secondary" href="https://forum.minetest.net/viewtopic.php?t={{ package.forums }}">Forums</a>{% endif %}
-					{% if package.issueTracker %}<a class="btn btn-secondary" href="{{ package.issueTracker }}">Issue Tracker</a>{% endif %}
-					{% if package.website %}<a class="btn btn-secondary" href="{{ package.website }}">Website</a>{% endif %}
+					{% if package.repo %}<a class="btn btn-outline-secondary" href="{{ package.repo }}">View Source</a>{% endif %}
+					{% if package.forums %}<a class="btn btn-outline-secondary" href="https://forum.minetest.net/viewtopic.php?t={{ package.forums }}">Forums</a>{% endif %}
+					{% if package.issueTracker %}<a class="btn btn-outline-secondary" href="{{ package.issueTracker }}">Issue Tracker</a>{% endif %}
+					{% if package.website %}<a class="btn btn-outline-secondary" href="{{ package.website }}">Website</a>{% endif %}
 				</div>
 			</div>
 		</div>

app/templates/users/profile.html

@@ -127,6 +127,15 @@
 								</td>
 							</tr>
 						{% endif %}
+						{% if user.checkPerm(current_user, "CREATE_TOKEN") %}
+						<tr>
+							<td>API Tokens:</td>
+							<td>
+								<a href="{{ url_for('api.list_tokens', username=user.username) }}">Manage</a>
+								<span class="badge badge-primary">{{ user.tokens.count() }}</span>
+							</td>
+						</tr>
+						{% endif %}
 					</table>
 				</div>
 			</div>

app/tests/test_api.py

@@ -0,0 +1,105 @@
+import pytest
+from app import app
+from app.default_data import populate_test_data
+from app.models import db, License, Tag, User, UserRank, Package
+from utils import client, recreate_db, parse_json
+from utils import is_str, is_int, is_optional
+
+def validate_package_list(packages, strict=False):
+	valid_keys = {
+		"author", "name", "release",
+		"short_description", "thumbnail",
+		"title", "type"
+	}
+
+	for package in packages:
+		assert set(package.keys()).issubset(valid_keys)
+
+		assert is_str(package.get("author"))
+		assert is_str(package.get("name"))
+		if strict:
+			assert is_int(package.get("release"))
+		else:
+			assert is_optional(int, package.get("release"))
+		assert is_str(package.get("short_description"))
+		assert is_optional(str, package.get("thumbnail"))
+		assert is_str(package.get("title"))
+		assert is_str(package.get("type"))
+
+
+def test_packages_empty(client):
+	"""Start with a blank database."""
+
+	rv = client.get("/api/packages/")
+	assert parse_json(rv.data) == []
+
+
+def test_packages_with_contents(client):
+	"""Start with a test database."""
+
+
+	populate_test_data(db.session)
+	db.session.commit()
+
+	rv = client.get("/api/packages/")
+
+	packages = parse_json(rv.data)
+
+	assert len(packages) > 0
+	assert len(packages) == Package.query.filter_by(approved=True).count()
+
+	validate_package_list(packages)
+
+
+def test_packages_with_query(client):
+	"""Start with a test database."""
+
+	populate_test_data(db.session)
+	db.session.commit()
+
+	rv = client.get("/api/packages/?q=food")
+
+	packages = parse_json(rv.data)
+
+	assert len(packages) == 2
+
+	validate_package_list(packages)
+
+	assert (packages[0]["name"] == "food" and packages[1]["name"] == "food_sweet") or \
+		(packages[1]["name"] == "food" and packages[0]["name"] == "food_sweet")
+
+
+def test_packages_with_protocol_high(client):
+	"""Start with a test database."""
+
+	populate_test_data(db.session)
+	db.session.commit()
+
+	rv = client.get("/api/packages/?protocol_version=40")
+
+	packages = parse_json(rv.data)
+
+	assert len(packages) == 4
+
+	for package in packages:
+		assert package["name"] != "mesecons"
+
+	validate_package_list(packages, True)
+
+
+def test_packages_with_protocol_low(client):
+	"""Start with a test database."""
+
+	populate_test_data(db.session)
+	db.session.commit()
+
+	rv = client.get("/api/packages/?protocol_version=20")
+
+	packages = parse_json(rv.data)
+
+	assert len(packages) == 4
+
+	for package in packages:
+		assert package["name"] != "awards"
+
+	validate_package_list(packages, True)

app/tests/test_homepage.py

@@ -0,0 +1,22 @@
+import pytest
+from app import app
+from app.default_data import populate_test_data
+from app.models import db, License, Tag, User, UserRank
+from utils import client, recreate_db
+
+def test_homepage_empty(client):
+	"""Start with a blank database."""
+
+	rv = client.get("/")
+	assert b"No packages available" in rv.data and b"packagetile" not in rv.data
+
+
+def test_homepage_with_contents(client):
+	"""Start with a test database."""
+
+	populate_test_data(db.session)
+	db.session.commit()
+
+	rv = client.get("/")
+
+	assert b"No packages available" not in rv.data and b"packagetile" in rv.data

app/tests/utils.py

@@ -0,0 +1,45 @@
+import pytest, json
+from app import app
+from app.models import db, User
+from app.default_data import populate
+
+def clear_data(session):
+	meta = db.metadata
+	for table in reversed(meta.sorted_tables):
+		session.execute(f'ALTER TABLE "{table.name}" DISABLE TRIGGER ALL;')
+		session.execute(table.delete())
+		session.execute(f'ALTER TABLE "{table.name}" ENABLE TRIGGER ALL;')
+		#session.execute(table.delete())
+
+def recreate_db():
+	clear_data(db.session)
+	populate(db.session)
+	db.session.commit()
+
+def parse_json(b):
+	return json.loads(b.decode("utf8"))
+
+def is_type(t, v):
+	return v and isinstance(v, t)
+
+def is_optional(t, v):
+	return not v or isinstance(v, t)
+
+def is_str(v):
+	return is_type(str, v)
+
+def is_int(v):
+	return is_type(int, v)
+
+
+@pytest.fixture
+def client():
+	app.config["TESTING"] = True
+
+	recreate_db()
+	assert User.query.count() == 1
+
+	with app.test_client() as client:
+		yield client
+
+	app.config["TESTING"] = False

app/utils.py

@@ -49,7 +49,9 @@ def randomString(n):
 def doFileUpload(file, fileType, fileTypeDesc):
 	if not file or file is None or file.filename == "":
 		flash("No selected file", "error")
-		return None
+		return None, None
+
+	assert os.path.isdir(app.config["UPLOAD_DIR"]), "UPLOAD_DIR must exist"
 
 	allowedExtensions = []
 	isImage = False
@@ -64,17 +66,18 @@ def doFileUpload(file, fileType, fileTypeDesc):
 	ext = getExtension(file.filename)
 	if ext is None or not ext in allowedExtensions:
 		flash("Please upload load " + fileTypeDesc, "danger")
-		return None
+		return None, None
 
 	if isImage and not isAllowedImage(file.stream.read()):
 		flash("Uploaded image isn't actually an image", "danger")
-		return None
+		return None, None
 
 	file.stream.seek(0)
 
 	filename = randomString(10) + "." + ext
-	file.save(os.path.join("app/public/uploads", filename))
-	return "/uploads/" + filename
+	filepath = os.path.join(app.config["UPLOAD_DIR"], filename)
+	file.save(filepath)
+	return "/uploads/" + filename, filepath
 
 def make_flask_user_password(plaintext_str):
 	# http://passlib.readthedocs.io/en/stable/modular_crypt_format.html
@@ -99,7 +102,7 @@ def make_flask_user_password(plaintext_str):
 	else:
 		return password.decode("UTF-8")
 
-def _do_login_user(user, remember_me=False):
+def loginUser(user):
 	def _call_or_get(v):
 		if callable(v):
 			return v()
@@ -125,29 +128,14 @@ def _do_login_user(user, remember_me=False):
 		flash("Your account has not been enabled.", "error")
 		return False
 
-	# Check if user has a confirmed email address
-	user_manager = current_app.user_manager
-	if user_manager.enable_email and user_manager.enable_confirm_email \
-			and not current_app.user_manager.enable_login_without_confirm_email \
-			and not user.has_confirmed_email():
-		url = url_for("user.resend_confirm_email")
-		flash("Your email address has not yet been confirmed", "error")
-		return False
-
 	# Use Flask-Login to sign in user
-	login_user(user, remember=remember_me)
+	login_user(user, remember=True)
 	signals.user_logged_in.send(current_app._get_current_object(), user=user)
 
 	flash("You have signed in successfully.", "success")
 
 	return True
 
-def loginUser(user):
-	user_mixin = None
-	if user_manager.enable_username:
-		user_mixin = user_manager.find_user_by_username(user.username)
-
-	return _do_login_user(user_mixin, True)
 
 def rank_required(rank):
 	def decorator(f):

config.example.cfg

@@ -1,29 +1,35 @@
-USER_APP_NAME="Content DB"
-SERVER_NAME="content.minetest.net"
-BASE_URL="http://" + SERVER_NAME
+USER_APP_NAME = "Content DB"
+SERVER_NAME = "content.minetest.net"
+BASE_URL = "http://" + SERVER_NAME
 
-SECRET_KEY=""
-WTF_CSRF_SECRET_KEY=""
+SECRET_KEY = ""
+WTF_CSRF_SECRET_KEY = ""
 
 SQLALCHEMY_DATABASE_URI = "sqlite:///../db.sqlite"
 
 GITHUB_CLIENT_ID = ""
 GITHUB_CLIENT_SECRET = ""
 
-REDIS_URL='redis://redis:6379'
-CELERY_BROKER_URL='redis://redis:6379'
-CELERY_RESULT_BACKEND='redis://redis:6379'
+REDIS_URL = 'redis://redis:6379'
+CELERY_BROKER_URL = 'redis://redis:6379'
+CELERY_RESULT_BACKEND = 'redis://redis:6379'
 
+USER_ENABLE_USERNAME = True
 USER_ENABLE_REGISTER = False
 USER_ENABLE_CHANGE_USERNAME = False
 
-MAIL_USERNAME=""
-MAIL_PASSWORD=""
-MAIL_DEFAULT_SENDER=""
-MAIL_SERVER=""
-MAIL_PORT=587
-MAIL_USE_TLS=True
-MAIL_UTILS_ERROR_SEND_TO=[""]
+MAIL_USERNAME = ""
+MAIL_PASSWORD = ""
+USER_EMAIL_SENDER_NAME = ""
+USER_EMAIL_SENDER_EMAIL = ""
+MAIL_DEFAULT_SENDER = ""
+MAIL_SERVER = ""
+MAIL_PORT = 587
+MAIL_USE_TLS = True
+MAIL_UTILS_ERROR_SEND_TO = [""]
+
+UPLOAD_DIR = "/var/cdb/uploads/"
+THUMBNAIL_DIR = "/var/cdb/thumbnails/"
 
 LANGUAGES = {
     'en': 'English',

docker-compose.yml

@@ -15,15 +15,15 @@ services:
 
   app:
     build: .
-    command: ./utils/run.sh
+    command: ./utils/entrypoint.sh
     env_file:
       - config.env
     ports:
       - 5123:5123
     volumes:
-      - "./data/uploads:/home/cdb/app/public/uploads"
-      - "./app:/home/cdb/appsrc"
-      - "./migrations:/home/cdb/migrations"
+      - "./data/uploads:/var/cdb/uploads"
+      - "./app:/source/app"
+      - "./migrations:/source/migrations"
     depends_on:
       - db
       - redis
@@ -36,7 +36,8 @@ services:
     environment:
       - FLASK_CONFIG=../config.cfg
     volumes:
-      - "./data/uploads:/home/cdb/app/public/uploads"
+      - "./data/uploads:/var/cdb/uploads"
+      - "./app:/home/cdb/app"
     depends_on:
       - redis
 

migrations/versions/306ce331a2a7_.py

@@ -0,0 +1,26 @@
+"""empty message
+
+Revision ID: 306ce331a2a7
+Revises: 6dca6eceb04d
+Create Date: 2020-01-18 23:00:40.487425
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '306ce331a2a7'
+down_revision = '6dca6eceb04d'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+	conn = op.get_bind()
+	op.create_check_constraint("CK_approval_valid", "package_release", "not approved OR (task_id IS NULL AND NOT url = '')")
+
+
+def downgrade():
+	conn = op.get_bind()
+	op.drop_constraint("CK_approval_valid", "package_release", type_="check")

migrations/versions/64fee8e5ab34_.py

@@ -0,0 +1,24 @@
+"""empty message
+
+Revision ID: 64fee8e5ab34
+Revises: 306ce331a2a7
+Create Date: 2020-01-19 02:28:05.432244
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '64fee8e5ab34'
+down_revision = '306ce331a2a7'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+	op.alter_column('user', 'confirmed_at', nullable=False, new_column_name='email_confirmed_at')
+
+
+def downgrade():
+	op.alter_column('user', 'email_confirmed_at', nullable=False, new_column_name='confirmed_at')

migrations/versions/6dca6eceb04d_.py

@@ -0,0 +1,30 @@
+"""empty message
+
+Revision ID: 6dca6eceb04d
+Revises: fd25bf3e57c3
+Create Date: 2020-01-18 17:32:21.885068
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+from sqlalchemy_searchable import sync_trigger
+
+
+# revision identifiers, used by Alembic.
+revision = '6dca6eceb04d'
+down_revision = 'fd25bf3e57c3'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+	conn = op.get_bind()
+	sync_trigger(conn, 'package', 'search_vector', ["name", "title", "short_desc", "desc"])
+	op.create_check_constraint("name_valid", "package", "name ~* '^[a-z0-9_]+$'")
+
+
+def downgrade():
+	conn = op.get_bind()
+	sync_trigger(conn, 'package', 'search_vector', ["title", "short_desc", "desc"])
+	op.drop_constraint("name_valid", "package", type_="check")

migrations/versions/a0f6c8743362_.py

@@ -0,0 +1,31 @@
+"""empty message
+
+Revision ID: a0f6c8743362
+Revises: 64fee8e5ab34
+Create Date: 2020-01-19 19:12:39.402679
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = 'a0f6c8743362'
+down_revision = '64fee8e5ab34'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+	op.alter_column('user', 'password',
+				existing_type=sa.VARCHAR(length=255),
+				nullable=False,
+				existing_server_default=sa.text("''::character varying"),
+				server_default='')
+
+
+def downgrade():
+	op.alter_column('user', 'password',
+				existing_type=sa.VARCHAR(length=255),
+				nullable=True,
+				existing_server_default=sa.text("''::character varying"))

migrations/versions/fd25bf3e57c3_.py

@@ -0,0 +1,37 @@
+"""empty message
+
+Revision ID: fd25bf3e57c3
+Revises: d6ae9682c45f
+Create Date: 2019-11-26 23:43:47.476346
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = 'fd25bf3e57c3'
+down_revision = 'd6ae9682c45f'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('api_token',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('access_token', sa.String(length=34), nullable=True),
+    sa.Column('name', sa.String(length=100), nullable=False),
+    sa.Column('owner_id', sa.Integer(), nullable=False),
+    sa.Column('created_at', sa.DateTime(), nullable=False),
+    sa.ForeignKeyConstraint(['owner_id'], ['user.id'], ),
+    sa.PrimaryKeyConstraint('id'),
+    sa.UniqueConstraint('access_token')
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('api_token')
+    # ### end Alembic commands ###

requirements.txt

@@ -1,4 +1,4 @@
-Flask~=1.0
+Flask~=1.1
 Flask-FlatPages~=0.7
 Flask-Gravatar~=0.5
 Flask-Login~=0.4.1
@@ -6,17 +6,21 @@ Flask-Markdown~=0.3
 Flask-Menu~=0.7
 Flask-Migrate~=2.3
 Flask-SQLAlchemy~=2.3
-Flask-User~=0.6
+Flask-User~=1.0
 Flask-Babel
 GitHub-Flask~=3.2
-SQLAlchemy-Searchable==1.0.3
+SQLAlchemy-Searchable~=1.1
 
 beautifulsoup4~=4.6
-celery==4.1.1
-kombu==4.2.0
-GitPython~=2.1
+celery~=4.4
+kombu~=4.6
+GitPython~=3.0
+git-archive-all~=1.20
 lxml~=4.2
-pillow~=6.2
+pillow~=7.0
 pyScss~=1.3
-redis==2.10.6
+redis~=3.3
 psycopg2~=2.7
+
+pytest ~= 5.3
+pytest-cov ~= 2.8

utils/bash.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+# Open SSH to app instance
+
+docker exec -it contentdb_app_1 bash

utils/create_migration.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+# Create a database migration, and copy it back to the host.
+
+docker exec contentdb_app_1 sh -c "FLASK_CONFIG=../config.cfg FLASK_APP=app/__init__.py flask db migrate"
+docker exec -u root contentdb_app_1 sh -c "cp /home/cdb/migrations/versions/* /source/migrations/versions/"
+
+USER=$(whoami)
+sudo chown -R $USER:$USER migrations/versions

utils/db.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+# Open SQL console for the database
+
+docker exec -it contentdb_db_1 sh -c "psql contentdb contentdb"

utils/entrypoint.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+#
+# The entrypoint for the docker containers
+#
+
+# Debug
+# FLASK_APP=app/__init__.py FLASK_CONFIG=../config.cfg FLASK_DEBUG=1 python3 -m flask run -h 0.0.0.0 -p 5123
+
+if [ -z "$FLASK_DEBUG" ]; then
+	echo "FLASK_DEBUG is required in config.env"
+	exit 1
+fi
+
+if [ "$FLASK_DEBUG" -eq "1" ]; then
+	FLASK_APP=app/__init__.py FLASK_CONFIG=../config.cfg FLASK_RUN_PORT=5123 flask run --host=0.0.0.0
+else
+	ENV="-e FLASK_APP=app/__init__.py -e FLASK_CONFIG=../config.cfg -e FLASK_DEBUG=$FLASK_DEBUG"
+	gunicorn -w 4 -b :5123 $ENV app:app
+fi

utils/gitlabci/config.cfg

@@ -0,0 +1,31 @@
+USER_APP_NAME="Content DB"
+SERVER_NAME="localhost:5123"
+BASE_URL="http://" + SERVER_NAME
+
+SECRET_KEY="changeme"
+WTF_CSRF_SECRET_KEY="changeme"
+
+SQLALCHEMY_DATABASE_URI = "postgres://contentdb:password@db:5432/contentdb"
+
+GITHUB_CLIENT_ID = ""
+GITHUB_CLIENT_SECRET = ""
+
+REDIS_URL='redis://redis:6379'
+CELERY_BROKER_URL='redis://redis:6379'
+CELERY_RESULT_BACKEND='redis://redis:6379'
+
+USER_ENABLE_USERNAME = True
+USER_ENABLE_REGISTER = False
+USER_ENABLE_CHANGE_USERNAME = False
+USER_ENABLE_EMAIL = False
+
+MAIL_UTILS_ERROR_SEND_TO = [""]
+
+UPLOAD_DIR="/var/cdb/uploads/"
+THUMBNAIL_DIR="/var/cdb/thumbnails/"
+
+TEMPLATES_AUTO_RELOAD = True
+
+LANGUAGES = {
+    'en': 'English',
+}

utils/gitlabci/config.env

@@ -0,0 +1,4 @@
+POSTGRES_USER=contentdb
+POSTGRES_PASSWORD=password
+POSTGRES_DB=contentdb
+FLASK_DEBUG=1

utils/reload.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+# Hot/live reload - only works in debug mode
+
+docker exec contentdb_app_1 sh -c "cp -r /source/* ."

utils/run.sh

@@ -1,21 +0,0 @@
-#!/bin/bash
-
-# Debug
-# FLASK_APP=app/__init__.py FLASK_CONFIG=../config.cfg FLASK_DEBUG=1 python3 -m flask run -h 0.0.0.0 -p 5123
-
-if [ -z "$FLASK_DEBUG" ]; then
-	echo "FLASK_DEBUG is required in config.env"
-	exit 1
-fi
-
-ENV="-e FLASK_APP=app/__init__.py -e FLASK_CONFIG=../config.cfg -e FLASK_DEBUG=$FLASK_DEBUG"
-
-if [ "$FLASK_DEBUG" -eq "1" ]; then
-	EXTRA="--reload"
-fi
-
-echo "Running gunicorn with:"
-echo " - env: $ENV"
-echo " - extra: $EXTRA"
-
-gunicorn -w 4 -b :5123 $ENV $EXTRA app:app

utils/run_migrations.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+# Run all pending migrations
+
+./utils/reload.sh
+docker exec contentdb_app_1 sh -c "FLASK_CONFIG=../config.cfg FLASK_APP=app/__init__.py flask db upgrade"

utils/setup.py

@@ -27,360 +27,22 @@ test_data = len(sys.argv) >= 2 and sys.argv[1].strip() == "-t" or not create_db
 # Allow finding the `app` module
 currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
 parentdir = os.path.dirname(currentdir)
-sys.path.insert(0,parentdir) 
-
-from app.models import *
-from app.utils import make_flask_user_password
-
-def defineDummyData(licenses, tags, ruben):
-	ez = User("Shara")
-	ez.github_username = "Ezhh"
-	ez.forums_username = "Shara"
-	ez.rank = UserRank.EDITOR
-	db.session.add(ez)
-
-	not1 = Notification(ruben, ez, "Awards approved", "/packages/rubenwardy/awards/")
-	db.session.add(not1)
-
-	jeija = User("Jeija")
-	jeija.github_username = "Jeija"
-	jeija.forums_username = "Jeija"
-	db.session.add(jeija)
-
-
-	mod = Package()
-	mod.approved = True
-	mod.name = "alpha"
-	mod.title = "Alpha Test"
-	mod.license = licenses["MIT"]
-	mod.type = PackageType.MOD
-	mod.author = ruben
-	mod.tags.append(tags["mapgen"])
-	mod.tags.append(tags["environment"])
-	mod.repo = "https://github.com/ezhh/other_worlds"
-	mod.issueTracker = "https://github.com/ezhh/other_worlds/issues"
-	mod.forums = 16015
-	mod.short_desc = "The content library should not be used yet as it is still in alpha"
-	mod.desc = "This is the long desc"
-	db.session.add(mod)
-
-	rel = PackageRelease()
-	rel.package = mod
-	rel.title = "v1.0.0"
-	rel.url = "https://github.com/ezhh/handholds/archive/master.zip"
-	rel.approved = True
-	db.session.add(rel)
-
-	mod1 = Package()
-	mod1.approved = True
-	mod1.name = "awards"
-	mod1.title = "Awards"
-	mod1.license = licenses["LGPLv2.1"]
-	mod1.type = PackageType.MOD
-	mod1.author = ruben
-	mod1.tags.append(tags["player_effects"])
-	mod1.repo = "https://github.com/rubenwardy/awards"
-	mod1.issueTracker = "https://github.com/rubenwardy/awards/issues"
-	mod1.forums = 4870
-	mod1.short_desc = "Adds achievements and an API to register new ones."
-	mod1.desc = """
-Majority of awards are back ported from Calinou's old fork in Carbone, under same license.
-
-```
-awards.register_achievement("award_mesefind",{
-    title = "First Mese Find",
-    description = "Found some Mese!",
-    trigger = {
-        type   = "dig",          -- award is given when
-        node   = "default:mese", -- this type of node has been dug
-        target = 1,              -- this number of times
-    },
-})
-```
-"""
-
-	rel = PackageRelease()
-	rel.package = mod1
-	rel.title = "v1.0.0"
-	rel.url = "https://github.com/rubenwardy/awards/archive/master.zip"
-	rel.approved = True
-	db.session.add(rel)
-
-	mod2 = Package()
-	mod2.approved = True
-	mod2.name = "mesecons"
-	mod2.title = "Mesecons"
-	mod2.tags.append(tags["tools"])
-	mod2.type = PackageType.MOD
-	mod2.license = licenses["LGPLv3"]
-	mod2.author = jeija
-	mod2.repo = "https://github.com/minetest-mods/mesecons/"
-	mod2.issueTracker = "https://github.com/minetest-mods/mesecons/issues"
-	mod2.forums = 628
-	mod2.short_desc = "Mesecons adds everything digital, from all kinds of sensors, switches, solar panels, detectors, pistons, lamps, sound blocks to advanced digital circuitry like logic gates and programmable blocks."
-	mod2.desc = """
-    ########################################################################
-    ##  __    __   _____   _____   _____   _____   _____   _   _   _____  ##
-    ## |  \  /  | |  ___| |  ___| |  ___| |  ___| |  _  | | \ | | |  ___| ##
-    ## |   \/   | | |___  | |___  | |___  | |     | | | | |  \| | | |___  ##
-    ## | |\__/| | |  ___| |___  | |  ___| | |     | | | | |     | |___  | ##
-    ## | |    | | | |___   ___| | | |___  | |___  | |_| | | |\  |  ___| | ##
-    ## |_|    |_| |_____| |_____| |_____| |_____| |_____| |_| \_| |_____| ##
-    ##                                                                    ##
-    ########################################################################
-
-MESECONS by Jeija and contributors
-
-Mezzee-what?
-------------
-[Mesecons](http://mesecons.net/)! They're yellow, they're conductive, and they'll add a whole new dimension to Minetest's gameplay.
-
-Mesecons is a mod for [Minetest](http://minetest.net/) that implements a ton of items related to digital circuitry, such as wires, buttons, lights, and even programmable controllers. Among other things, there are also pistons, solar panels, pressure plates, and note blocks.
-
-Mesecons has a similar goal to Redstone in Minecraft, but works in its own way, with different rules and mechanics.
-
-OK, I want in.
---------------
-Go get it!
-
-[DOWNLOAD IT NOW](https://github.com/minetest-mods/mesecons/archive/master.zip)
-
-Now go ahead and install it like any other Minetest mod. Don't know how? Check out [the wonderful page about it](http://wiki.minetest.com/wiki/Mods) over at the Minetest Wiki. For your convenience, here's a quick summary:
-
-1. If Mesecons is still in a ZIP file, extract the folder inside to somewhere on the computer.
-2. Make sure that when you open the folder, you can directly find `README.md` in the listing. If you just see another folder, move that folder up one level and delete the old one.
-3. Open up the Minetest mods folder - usually `/mods/`. If you see the `minetest` or folder inside of that, that is your mod folder instead.
-4. Copy the Mesecons folder into the mods folder.
-
-Don't like some parts of Mesecons? Open up the Mesecons folder and delete the subfolder containing the mod you don't want. If you didn't want movestones, for example, all you have to do is delete the `mesecons_movestones` folder and they will no longer be available.
-
-There are no dependencies - it will work right after installing!
-
-How do I use this thing?
-------------------------
-How about a [quick overview video](https://www.youtube.com/watch?v=6kmeQj6iW5k)?
-
-Or maybe a [comprehensive reference](http://mesecons.net/items.html) is your style?
-
-An overview for the very newest of new beginners? How does [this one](http://uberi.mesecons.net/projects/MeseconsBasics/index.html) look?
-
-Want to get more into building? Why not check out the [Mesecons Laboratory](http://uberi.mesecons.net/), a website dedicated to advanced Mesecons builders?
-
-Want to contribute to Mesecons itself? Check out the [source code](https://github.com/minetest-mods/mesecons)!
-
-Who wrote it anyways?
----------------------
-These awesome people made Mesecons possible!
-
-| Contributor     | Contribution                     |
-| --------------- | -------------------------------- |
-| Hawk777         | Code for VoxelManip caching      |
-| Jat15           | Various tweaks.                  |
-| Jeija           | **Main developer! Everything.**  |
-| Jordach         | Noteblock sounds.                |
-| khonkhortistan  | Code, recipes, textures.         |
-| Kotolegokot     | Nodeboxes for items.             |
-| minerd247       | Textures.                        |
-| Nore/Novatux    | Code.                            |
-| RealBadAngel    | Fixes, improvements.             |
-| sfan5           | Code, recipes, textures.         |
-| suzenako        | Piston sounds.                   |
-| Uberi/Temperest | Code, textures, documentation.   |
-| VanessaE        | Code, recipes, textures, design. |
-| Whiskers75      | Logic gates implementation.      |
-
-There are also a whole bunch of other people helping with everything from code to testing and feedback. Mesecons would also not be possible without their help!
-
-Alright, how can I use it?
---------------------------
-All textures in this project are licensed under the CC-BY-SA 3.0 (Creative Commons Attribution-ShareAlike 3.0 Generic). That means you can distribute and remix them as much as you want to, under the condition that you give credit to the authors and the project, and that if you remix and release them, they must be under the same or similar license to this one.
-
-All code in this project is licensed under the LGPL version 3 or later. That means you have unlimited freedom to distribute and modify the work however you see fit, provided that if you decide to distribute it or any modified versions of it, you must also use the same license. The LGPL also grants the additional freedom to write extensions for the software and distribute them without the extensions being subject to the terms of the LGPL, although the software itself retains its license.
-
-No warranty is provided, express or implied, for any part of the project.
-
-"""
-
-	db.session.add(mod1)
-	db.session.add(mod2)
-
-	mod = Package()
-	mod.approved = True
-	mod.name = "handholds"
-	mod.title = "Handholds"
-	mod.license = licenses["MIT"]
-	mod.type = PackageType.MOD
-	mod.author = ez
-	mod.tags.append(tags["player_effects"])
-	mod.repo = "https://github.com/ezhh/handholds"
-	mod.issueTracker = "https://github.com/ezhh/handholds/issues"
-	mod.forums = 17069
-	mod.short_desc = "Adds hand holds and climbing thingies"
-	mod.desc = "This is the long desc"
-	db.session.add(mod)
-
-	rel = PackageRelease()
-	rel.package = mod
-	rel.title = "v1.0.0"
-	rel.url = "https://github.com/ezhh/handholds/archive/master.zip"
-	rel.approved = True
-	db.session.add(rel)
-
-	mod = Package()
-	mod.approved = True
-	mod.name = "other_worlds"
-	mod.title = "Other Worlds"
-	mod.license = licenses["MIT"]
-	mod.type = PackageType.MOD
-	mod.author = ez
-	mod.tags.append(tags["mapgen"])
-	mod.tags.append(tags["environment"])
-	mod.repo = "https://github.com/ezhh/other_worlds"
-	mod.issueTracker = "https://github.com/ezhh/other_worlds/issues"
-	mod.forums = 16015
-	mod.short_desc = "Adds space with asteroids and comets"
-	mod.desc = "This is the long desc"
-	db.session.add(mod)
-
-	mod = Package()
-	mod.approved = True
-	mod.name = "food"
-	mod.title = "Food"
-	mod.license = licenses["LGPLv2.1"]
-	mod.type = PackageType.MOD
-	mod.author = ruben
-	mod.tags.append(tags["player_effects"])
-	mod.repo = "https://github.com/rubenwardy/food/"
-	mod.issueTracker = "https://github.com/rubenwardy/food/issues/"
-	mod.forums = 2960
-	mod.short_desc = "Adds lots of food and an API to manage ingredients"
-	mod.desc = "This is the long desc"
-	food = mod
-	db.session.add(mod)
-
-	mod = Package()
-	mod.approved = True
-	mod.name = "food_sweet"
-	mod.title = "Sweet Foods"
-	mod.license = licenses["CC0"]
-	mod.type = PackageType.MOD
-	mod.author = ruben
-	mod.tags.append(tags["player_effects"])
-	mod.repo = "https://github.com/rubenwardy/food_sweet/"
-	mod.issueTracker = "https://github.com/rubenwardy/food_sweet/issues/"
-	mod.forums = 9039
-	mod.short_desc = "Adds sweet food"
-	mod.desc = "This is the long desc"
-	food_sweet = mod
-	db.session.add(mod)
-
-	game1 = Package()
-	game1.approved = True
-	game1.name = "capturetheflag"
-	game1.title = "Capture The Flag"
-	game1.type = PackageType.GAME
-	game1.license = licenses["LGPLv2.1"]
-	game1.author = ruben
-	game1.tags.append(tags["pvp"])
-	game1.tags.append(tags["survival"])
-	game1.tags.append(tags["multiplayer"])
-	game1.repo = "https://github.com/rubenwardy/capturetheflag"
-	game1.issueTracker = "https://github.com/rubenwardy/capturetheflag/issues"
-	game1.forums = 12835
-	game1.short_desc = "Two teams battle to snatch and return the enemy's flag, before the enemy takes their own!"
-	game1.desc = """
-As seen on the Capture the Flag server (minetest.rubenwardy.com:30000)
-
-Uses the CTF PvP Engine.
-"""
-
-	db.session.add(game1)
-
-	rel = PackageRelease()
-	rel.package = game1
-	rel.title = "v1.0.0"
-	rel.url = "https://github.com/rubenwardy/capturetheflag/archive/master.zip"
-	rel.approved = True
-	db.session.add(rel)
-
-
-	mod = Package()
-	mod.approved = True
-	mod.name = "pixelbox"
-	mod.title = "PixelBOX Reloaded"
-	mod.license = licenses["CC0"]
-	mod.type = PackageType.TXP
-	mod.author = ruben
-	mod.forums = 14132
-	mod.short_desc = "This is an update of the original PixelBOX texture pack by the brillant artist Gambit"
-	mod.desc = "This is the long desc"
-	db.session.add(mod)
-
-	rel = PackageRelease()
-	rel.package = mod
-	rel.title = "v1.0.0"
-	rel.url = "http://mamadou3.free.fr/Minetest/PixelBOX.zip"
-	rel.approved = True
-	db.session.add(rel)
-
-	db.session.commit()
-
-	metas = {}
-	for package in Package.query.filter_by(type=PackageType.MOD).all():
-		meta = None
-		try:
-			meta = metas[package.name]
-		except KeyError:
-			meta = MetaPackage(package.name)
-			db.session.add(meta)
-			metas[package.name] = meta
-		package.provides.append(meta)
-
-	dep = Dependency(food_sweet, meta=metas["food"])
-	db.session.add(dep)
+sys.path.insert(0,parentdir)
 
+from app.models import db, User, UserRank
+from app.default_data import populate, populate_test_data
 
 if delete_db and os.path.isfile("db.sqlite"):
 	os.remove("db.sqlite")
 
-
 if create_db:
 	print("Creating database tables...")
 	db.create_all()
 
 print("Filling database...")
 
-ruben = User("rubenwardy")
-ruben.active = True
-ruben.password = make_flask_user_password("tuckfrump")
-ruben.github_username = "rubenwardy"
-ruben.forums_username = "rubenwardy"
-ruben.rank = UserRank.ADMIN
-db.session.add(ruben)
-
-tags = {}
-for tag in ["Inventory", "Mapgen", "Building", \
-		"Mobs and NPCs", "Tools", "Player effects", \
-		"Environment", "Transport", "Maintenance", "Plants and farming", \
-		"PvP", "PvE", "Survival", "Creative", "Puzzle", "Multiplayer", "Singleplayer"]:
-	row = Tag(tag)
-	tags[row.name] = row
-	db.session.add(row)
-
-licenses = {}
-for license in ["GPLv2.1", "GPLv3", "LGPLv2.1", "LGPLv3", "AGPLv2.1", "AGPLv3",
-				"Apache", "BSD 3-Clause", "BSD 2-Clause", "CC0", "CC-BY-SA",
-				"CC-BY", "MIT", "ZLib", "Other (Free)"]:
-	row = License(license)
-	licenses[row.name] = row
-	db.session.add(row)
-
-for license in ["CC-BY-NC-SA", "Other (Non-free)"]:
-	row = License(license, False)
-	licenses[row.name] = row
-	db.session.add(row)
-
+populate(db.session)
 if test_data:
-	defineDummyData(licenses, tags, ruben)
+	populate_test_data(db.session)
 
 db.session.commit()

utils/start.sh

@@ -2,6 +2,8 @@
 
 #
 # Call from a docker host to build and start CDB.
+#   This is really only for production mode, for debugging it's better to use
+#   docker-compose directly:  docker-compose up --build
 #
 
 sudo docker-compose up --build -d --scale worker=2

utils/tests.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+docker exec contentdb_app_1 sh -c "FLASK_CONFIG=../config.cfg FLASK_APP=app/__init__.py python -m pytest app/tests/ --disable-warnings"

utils/tests_cov.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+docker exec contentdb_app_1 sh -c "FLASK_CONFIG=../config.cfg FLASK_APP=app/__init__.py python -m pytest app/tests/ --cov=app --disable-warnings"

utils/update.sh

@@ -2,6 +2,7 @@
 
 #
 # Call from a docker host to rebuild and update running instances of CDB.
+#    This is for production use. See reload.sh for debug mode hot/live reloading.
 #
 
 sudo docker-compose build app