Add prometheus support

app/blueprints/api/endpoints.py

@@ -28,6 +28,9 @@ from app.querybuilder import QueryBuilder
 
 @bp.route("/api/packages/")
 def packages():
+	import sys
+	print("\n\n############", file=sys.stderr)
+
 	qb    = QueryBuilder(request.args)
 	query = qb.buildPackageQuery()
 	ver   = qb.getMinetestVersion()
@@ -140,6 +143,9 @@ def markdown():
 @is_package_page
 @is_api_authd
 def create_release(token, package):
+	if not package.checkPerm(token.owner, Permission.APPROVE_RELEASE):
+		return error(403, "You do not have the permission to approve releases")
+
 	json = request.json
 	if json is None:
 		return error(400, "JSON post data is required")

app/blueprints/github/__init__.py

@@ -124,7 +124,7 @@ def webhook():
 		return error(403, "Invalid authentication, couldn't validate API token")
 
 	if not package.checkPerm(actual_token.owner, Permission.APPROVE_RELEASE):
-		return error(403, "Only trusted members can use webhooks")
+		return error(403, "You do not have the permission to approve releases")
 
 	#
 	# Check event

app/blueprints/gitlab/__init__.py

@@ -44,7 +44,7 @@ def webhook():
 		return error(403, "Invalid authentication")
 
 	if not package.checkPerm(token.owner, Permission.APPROVE_RELEASE):
-		return error(403, "Only trusted members can use webhooks")
+		return error(403, "You do not have the permission to approve releases")
 
 	#
 	# Check event

app/blueprints/homepage/__init__.py

@@ -4,17 +4,23 @@ bp = Blueprint("homepage", __name__)
 
 from app.models import *
 import flask_menu as menu
+from sqlalchemy.orm import joinedload
 from sqlalchemy.sql.expression import func
 
 @bp.route("/")
 @menu.register_menu(bp, ".", "Home")
 def home():
+	def join(query):
+		return query.options( \
+			joinedload(Package.license), \
+			joinedload(Package.media_license))
+
 	query   = Package.query.filter_by(approved=True, soft_deleted=False)
 	count   = query.count()
-	new     = query.order_by(db.desc(Package.created_at)).limit(8).all()
-	pop_mod = query.filter_by(type=PackageType.MOD).order_by(db.desc(Package.score)).limit(8).all()
-	pop_gam = query.filter_by(type=PackageType.GAME).order_by(db.desc(Package.score)).limit(4).all()
-	pop_txp = query.filter_by(type=PackageType.TXP).order_by(db.desc(Package.score)).limit(4).all()
+	new     = join(query.order_by(db.desc(Package.created_at))).limit(8).all()
+	pop_mod = join(query.filter_by(type=PackageType.MOD).order_by(db.desc(Package.score))).limit(8).all()
+	pop_gam = join(query.filter_by(type=PackageType.GAME).order_by(db.desc(Package.score))).limit(4).all()
+	pop_txp = join(query.filter_by(type=PackageType.TXP).order_by(db.desc(Package.score))).limit(4).all()
 	downloads_result = db.session.query(func.sum(PackageRelease.downloads)).one_or_none()
 	downloads = 0 if not downloads_result or not downloads_result[0] else downloads_result[0]
 	return render_template("index.html", count=count, downloads=downloads, \

app/blueprints/metrics/__init__.py

@@ -0,0 +1,74 @@
+# Content DB
+# Copyright (C) 2020  rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+from flask import Blueprint, make_response
+from app.models import Package, PackageRelease, db, User, UserRank
+from sqlalchemy.sql.expression import func
+
+bp = Blueprint("metrics", __name__)
+
+def generate_metrics(full=False):
+	def write_single_stat(name, help, type, value):
+		fmt = "# HELP {name} {help}\n# TYPE {name} {type}\n{name} {value}\n\n"
+
+		return fmt.format(name=name, help=help, type=type, value=value)
+
+	def gen_labels(labels):
+		pieces = [key + "=" + str(val) for key, val in labels.items()]
+		return (",").join(pieces)
+
+
+	def write_array_stat(name, help, type, data):
+		ret = ("# HELP {name} {help}\n# TYPE {name} {type}\n") \
+				.format(name=name, help=help, type=type)
+
+		for entry in data:
+			assert(len(entry) == 2)
+			ret += ("{name}{{{labels}}} {value}\n") \
+					.format(name=name, labels=gen_labels(entry[0]), value=entry[1])
+
+		return ret + "\n"
+
+
+	downloads_result = db.session.query(func.sum(PackageRelease.downloads)).one_or_none()
+	downloads = 0 if not downloads_result or not downloads_result[0] else downloads_result[0]
+
+	packages = Package.query.filter_by(approved=True, soft_deleted=False).count()
+	users = User.query.filter(User.rank != UserRank.NOT_JOINED).count()
+
+	ret = ""
+	ret += write_single_stat("contentdb_packages", "Total packages", "counter", packages)
+	ret += write_single_stat("contentdb_users", "Number of registered users", "counter", users)
+	ret += write_single_stat("contentdb_downloads", "Total downloads", "counter", downloads)
+
+	if full:
+		scores = Package.query.join(User).with_entities(User.username, Package.name, Package.score) \
+			.filter(Package.approved==True, Package.soft_deleted==False).all()
+
+		ret += write_array_stat("contentdb_package_score", "Package score", "gauge", \
+			[({ "author": score[0], "name": score[1] }, score[2])  for score in scores])
+	else:
+		score_result = db.session.query(func.sum(Package.score)).one_or_none()
+		score = 0 if not score_result or not score_result[0] else score_result[0]
+		ret += write_single_stat("contentdb_score", "Total package score", "gauge", score)
+
+	return ret
+
+@bp.route("/metrics")
+def metrics():
+    response = make_response(generate_metrics(), 200)
+    response.mimetype = "text/plain"
+    return response

app/blueprints/packages/packages.py

@@ -31,6 +31,7 @@ from wtforms import *
 from wtforms.validators import *
 from wtforms.ext.sqlalchemy.fields import QuerySelectField, QuerySelectMultipleField
 from sqlalchemy import or_, func
+from sqlalchemy.orm import joinedload, subqueryload
 
 
 @menu.register_menu(bp, ".mods", "Mods", order=11, endpoint_arguments_constructor=lambda: { 'type': 'mod' })
@@ -43,6 +44,11 @@ def list_all():
 	query = qb.buildPackageQuery()
 	title = qb.title
 
+	query = query.options( \
+			joinedload(Package.license), \
+			joinedload(Package.media_license), \
+			subqueryload(Package.tags))
+
 	if qb.lucky:
 		package = query.first()
 		if package:

app/flatpages/help.md

@@ -1,9 +1,15 @@
 title: Help
 
+## Content
+
 * [Package Tags](package_tags)
 * [Ranks and Permissions](ranks_permissions)
 * [Content Ratings and Flags](content_flags)
 * [Reporting Content](reporting)
 * [Top Packages Algorithm](top_packages)
+
+## Developers
+
 * [API](api)
+* [Metrics](metrics)
 * [Creating Releases using Webhooks](release_webhooks)

app/flatpages/help/metrics.md

@@ -0,0 +1,16 @@
+title: Prometheus Metrics
+
+## What is Prometheus?
+
+[Prometheus](https://prometheus.io) is an "open-source monitoring system with a
+dimensional data model, flexible query language, efficient time series database
+and modern alerting approach".
+
+Prometheus Metrics can be accessed at [/metrics](/metrics).
+
+## Metrics
+
+* `contentdb_packages` - Total packages (counter).
+* `contentdb_users` - Number of registered users (counter).
+* `contentdb_downloads` - Total downloads (counter).
+* `contentdb_score` - Total package score (gauge).

app/flatpages/help/release_webhooks.md

@@ -17,10 +17,6 @@ The process is as follows:
 3. The git host posts a webhook notification to ContentDB, using the API token assigned to it.
 4. ContentDB checks the API token and issues a new release.
 
-<p class="alert alert-info">
-	This feature is in beta, and is only available for Trusted Members.
-</p>
-
 ## Setting up
 
 ### GitHub (automatic)

app/flatpages/help/top_packages.md

@@ -10,7 +10,7 @@ whereas a non-free package will only gain 0.1 score.
 A package currently only gains score through downloads.
 In the future, a package will also gain score through reviews.
 
-## Seed using a legacy heuristic
+## Seeded using a legacy heuristic
 
 The scoring system was seeded (ie: the scores were initially set to) 20% of an
 arbitrary legacy heuristic that was previously used to rank packages.

app/flatpages/policy_and_guidance.md

@@ -76,7 +76,7 @@ to change the name of the package, or your package won't be accepted.
 
 We reserve the right to issue exceptions for this where we feel necessary.
 
-### 3.2 Mod Forks and Reimplementations
+### 3.2. Mod Forks and Reimplementations
 
 An exception to the above is that mods are allowed to have the same name as a
 mod if its a fork of that mod (or a close reimplementation). In real terms, it
@@ -88,7 +88,7 @@ reimplementation of the mod that owns the name.
 
 ## 4. Licenses
 
-### 4.1 Allowed Licenses
+### 4.1. Allowed Licenses
 
 Please ensure that you correctly credit any resources (code, assets, or otherwise)
 that you have used in your package.
@@ -106,7 +106,7 @@ get around to adding it.
 Please note that the definitions of "free" and "non-free" is the same as that
 of the [Free Software Foundation](https://www.gnu.org/philosophy/free-sw.en.html).
 
-### 4.2 Recommended Licenses
+### 4.2. Recommended Licenses
 
 It is highly recommended that you use a free and open source software license.
 FOSS licenses result in a sharing community and will increase the number of potential users your package has.

app/models.py

@@ -147,7 +147,7 @@ class User(db.Model, UserMixin):
 	notifications = db.relationship("Notification", primaryjoin="User.id==Notification.user_id")
 
 	# causednotifs  = db.relationship("Notification", backref="causer", lazy="dynamic")
-	packages      = db.relationship("Package", backref="author", lazy="dynamic")
+	packages      = db.relationship("Package", backref=db.backref("author", lazy="joined"), lazy="dynamic")
 	requests      = db.relationship("EditRequest", backref="author", lazy="dynamic")
 	threads       = db.relationship("Thread", backref="author", lazy="dynamic")
 	tokens        = db.relationship("APIToken", backref="owner", lazy="dynamic")
@@ -437,12 +437,12 @@ class Package(db.Model):
 	forums       = db.Column(db.Integer,     nullable=True)
 
 	provides = db.relationship("MetaPackage", \
-			secondary=provides, lazy="subquery", order_by=db.asc("name"), \
+			secondary=provides, lazy="select", order_by=db.asc("name"), \
 			backref=db.backref("packages", lazy="dynamic", order_by=db.desc("score")))
 
 	dependencies = db.relationship("Dependency", backref="depender", lazy="dynamic", foreign_keys=[Dependency.depender_id])
 
-	tags = db.relationship("Tag", secondary=tags, lazy="subquery",
+	tags = db.relationship("Tag", secondary=tags, lazy="select",
 			backref=db.backref("packages", lazy=True))
 
 	releases = db.relationship("PackageRelease", backref="package",
@@ -1135,3 +1135,8 @@ class ForumTopic(db.Model):
 
 # Setup Flask-User
 user_manager = UserManager(app, db, User)
+
+if app.config.get("LOG_SQL"):
+	import logging
+	logging.basicConfig()
+	logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)

app/scss/custom.scss

@@ -3,12 +3,54 @@
 @import "packagegrid.scss";
 @import "comments.scss";
 
+h1 {
+	font-size: 2em;
+	font-weight: bold;
+	margin: 0 0 0.5em;
+	letter-spacing: .05em
+}
+
+h2 {
+	font-size: 1.8em;
+	font-weight: bold;
+	color: white;
+	margin: 1.5em 0 1em;
+	letter-spacing: .05em;
+	padding: 0 0 0.5em 0;
+	border-bottom: 1px solid #444;
+}
+
+h3 {
+	font-size: 1.3em;
+	font-weight: bold;
+	color: white;
+	margin: 1.5em 0 1em;
+	letter-spacing: .05em
+}
+
+p, .content li {
+	-moz-osx-font-smoothing: grayscale;
+	-webkit-font-smoothing: antialiased !important;
+	-moz-font-smoothing: antialiased !important;
+	text-rendering: optimizelegibility !important;
+	letter-spacing: .03em;
+	line-height: 1.6em;
+}
+
+pre code {
+	display: block;
+	border: 1px solid rgba(255, 255, 255, 0.1);
+	background: rgba(255, 255, 255, 0.03);
+	padding: 0.75rem 1.25rem;
+	border-radius: 0.25rem;
+}
+
 .dropdown-menu {
 	margin-top: 0;
 }
 
 .dropdown:hover .dropdown-menu {
-   display: block;
+	display: block;
 }
 
 .nav-link > img {
@@ -57,8 +99,15 @@
 	text-decoration: none;
 }
 
-.card .table {
-	margin-bottom: 0;
+.card {
+	.card-header {
+		margin: 0;
+		font-size: 100%;
+		font-weight: normal;
+	}
+	.table {
+		margin-bottom: 0;
+	}
 }
 
 .btn-download {

app/scss/packagegrid.scss

@@ -38,35 +38,43 @@ li.d-flex {
 	bottom: 0;
 	left: 0;
 	padding: 1em;
+
+	h3 {
+		color: white;
+		font-size: 120%;
+		font-weight: bold;
+		margin: 0;
+		padding: 0;
+	}
+
+	small {
+		color: #ddd;
+		font-size: 75%;
+		font-weight: bold;
+	}
+
+	p {
+		display: none;
+		color: #ddd;
+		font-weight: normal;
+	}
 }
 
-.packagegridinfo h3 {
-	color: white;
-	font-size: 120%;
-	font-weight: bold;
-}
+.packagetile a:hover  {
+	.packagegridinfo {
+		top: 0;
+	}
 
-.packagegridinfo small {
-	color: #ddd;
-	font-size: 75%;
-	font-weight: bold;
-}
+	h3 {
+		margin-bottom: 0.5em;
+	}
 
-.packagegridinfo p {
-	display: none;
-	color: #ddd;
-	font-weight: normal;
-}
+	p {
+		display: block;
+	}
 
-.packagetile a:hover .packagegridinfo {
-	top: 0;
-}
-
-.packagetile a:hover p {
-	display: block;
-}
-
-.packagetile a:hover .packagegridscrub {
-	top: 0;
-	background: rgba(0,0,0,0.8);
+	.packagegridscrub {
+		top: 0;
+		background: rgba(0,0,0,0.8);
+	}
 }

app/templates/base.html

@@ -7,7 +7,7 @@
 	<meta name="viewport" content="width=device-width, initial-scale=1">
 	<title>{% block title %}title{% endblock %} - {{ config.USER_APP_NAME }}</title>
 	<link rel="stylesheet" type="text/css" href="/static/bootstrap.css">
-	<link rel="stylesheet" type="text/css" href="/static/custom.css?v=10">
+	<link rel="stylesheet" type="text/css" href="/static/custom.css?v=11">
 	<link rel="search" type="application/opensearchdescription+xml" href="/static/opensearch.xml" title="ContentDB" />
 	<link rel="shortcut icon" href="/favicon-16.png" sizes="16x16">
 	<link rel="icon" href="/favicon-128.png" sizes="128x128">

app/templates/flatpage.html

@@ -4,8 +4,10 @@
 {{ page['title'] }}
 {% endblock %}
 
-{% block content %}
-	{% if not page["no_h1"] %}<h1>{{ page['title'] }}</h1>{% endif %}
+{% block container %}
+	<main class="container mt-4 content">
+		{% if not page["no_h1"] %}<h1>{{ page['title'] }}</h1>{% endif %}
 
-	{{ page.html | safe }}
+		{{ page.html | safe }}
+	</main>
 {% endblock %}

config.example.cfg

@@ -32,6 +32,9 @@ MAIL_UTILS_ERROR_SEND_TO = [""]
 UPLOAD_DIR = "/var/cdb/uploads/"
 THUMBNAIL_DIR = "/var/cdb/thumbnails/"
 
+TEMPLATES_AUTO_RELOAD = False
+LOG_SQL = False
+
 LANGUAGES = {
     'en': 'English',
 }