Add skip button to importer

This reverts commit dd6257a0a0.

.gitignore

@@ -6,6 +6,8 @@ tmp
 log.txt
 *.rdb
 uploads
+thumbnails
+celerybeat-schedule
 
 # Created by https://www.gitignore.io/api/linux,macos,python,windows
 

README.md

@@ -33,7 +33,7 @@ the current session:
 
 If you need to, reset the db like so:
 
-	python3 setup.py -d
+	python3 setup.py -t
 
 Then run the server:
 
@@ -43,6 +43,12 @@ Then view in your web browser: http://localhost:5000/
 
 ## How-tos
 
+### Start celery worker
+
+```sh
+FLASK_CONFIG=../config.cfg celery -A app.tasks.celery worker
+```
+
 ### Create migration
 
 ```sh
@@ -54,5 +60,5 @@ rm db.sqlite && python setup.py -t && FLASK_CONFIG=../config.cfg FLASK_APP=app/_
 FLASK_CONFIG=../config.cfg FLASK_APP=app/__init__.py flask db migrate
 
 # Run migration
-FLASK_CONFIG=../config.cfg FLASK_APP=app/__init__.py flask db migrate
+FLASK_CONFIG=../config.cfg FLASK_APP=app/__init__.py flask db upgrade
 ```

app/__init__.py

@@ -37,5 +37,9 @@ csrf = CsrfProtect(app)
 mail = Mail(app)
 pages = FlatPages(app)
 
+if not app.debug:
+	from .maillogger import register_mail_error_handler
+	register_mail_error_handler(app, mail)
+
 from . import models, tasks
 from .views import *

app/flatpages/help.md

@@ -1,4 +1,5 @@
 title: Help
 
-* [Ranks and Permissions](ranks_permissions)
 * [Package Tags](package_tags)
+* [Ranks and Permissions](ranks_permissions)
+* [Reporting Content](reporting)

app/flatpages/help/package_tags.md

@@ -2,26 +2,32 @@ title: Package Tags
 
 ## Overview
 
-Tags should be added to packages to enable easy identification of different types of mods and games. 
+Tags should be added to packages to enable easy identification of different types of mods, games and texture packs. 
 
 They are only beneficial when applied correctly, so please use the following guidelines.
 
 ## Tag Usage
 
-* **Inventory** - For mods that add new inventory systems or new inventory pages.
-* **Mapgen** - For mods that add new biomes, new mapgen decorations, or any other mapgen elements.
 * **Building** - For mods that focus on adding new materials or nodes to build with.
-* **Mobs and NPCs** - For mods that add mobs or NPCs, or provide tools that assist with mob and NPC creation or manipulation.
-* **Tools** - For mods that add new tools or new features for existing tools.
-* **Player effects** - For mods that change player effects, for example speed, jump height or gravity.
+* **Education** - For mods or games created for educational purposes.
 * **Environment** - For mods that add environmental effects, including ambient sound and weather effects.
-* **Transport** - For mods that add transportation methods. This includes teleportation, vehicles and ridable mobs.
+* **Inventory** - For mods that add new inventory systems or new inventory pages.
+* **Machines and Electronics** - For mods that include placeable machinery or electronic components which interact to complete tasks.
 * **Maintenance** - For mods that assist with world or player maintenance. This includes large-scale map manipulation, area protection and other administrative tools.
-* **Plants and farming** - For mods that add new plants or other farmable resources. 
+* **Mapgen** - For mods that add new biomes, new mapgen decorations, or any other mapgen elements.
+* **Mobs and NPCs** - For mods that add mobs or NPCs, or provide tools that assist with mob and NPC creation or manipulation.
+* **Plants and Farming** - For mods that add new plants or other farmable resources. 
+* **Player effects/Food** - For mods that change player effects, for example speed, jump height or gravity, and food.
+* **Tools** - For mods that add new tools or new features for existing tools.
+* **Transport** - For mods that add transportation methods. This includes teleportation, vehicles and ridable mobs.
 * **Survival** - For mods written specifically for survival games. For example, these mods might focus on game-balance or increase the difficulty level. This tag should also be used for games with a heavy survival focus.
 * **Creative** - For mods written specifically (and often exclusively) for use in creative mode. For example, these mods may add a large amount of decorative content, or content without crafting recipes. This tag should also be used for games with a heavy creative focus.
+* **Multiplayer-focused** - For games that can be played with other players.
+* **Singleplayer-focused** - For games that can be played alone.
 * **PvP** - For games designed to be played competitively against other players.
 * **PvE** - For games designed for one or multiple players which focus on combat against mobs or NPCs.
-* **Puzzle** - For games with a focus on puzzle solving instead of combat.
-* **Multiplayer** - For games that can be played with other players.
-* **Singleplayer** - For games that can be played alone.
+* **Puzzle** - For mods and games with a focus on puzzle solving instead of combat.
+* **16px** - For 16px texture packs.
+* **32px** - For 32px texture packs.
+* **64px** - For 64px texture packs.
+* **128px+** - For 128px or higher texture packs.

app/flatpages/help/ranks_permissions.md

@@ -4,6 +4,7 @@ title: Ranks and Permissions
 
 * **New Members** - mostly untrusted, cannot change package meta data or publish releases without approval.
 * **Members** - Trusted to change the meta data of their own packages', but cannot publish releases.
+* **Trusted Members** - Same as above, but can approve their own releases and packages.
 * **Editors** - Trusted to change the meta data of any package, and also make and publish releases.
 * **Moderators** - Same as above, but can manage users.
 * **Admins** - Full access.
@@ -16,6 +17,7 @@ title: Ranks and Permissions
 			<th>Rank</th>
 			<th colspan=2>New Member</th>
 			<th colspan=2>Member</th>
+			<th colspan=2>Trusted Member</th>
 			<th colspan=2>Editor</th>
 			<th colspan=2>Moderator</th>
 			<th colspan=2>Admin</th>
@@ -32,6 +34,8 @@ title: Ranks and Permissions
 			<th>N</th>
 			<th>Y</th>
 			<th>N</th>
+			<th>Y</th>
+			<th>N</th>
 		</tr>
 	</thead>
 	<tbody>
@@ -41,6 +45,8 @@ title: Ranks and Permissions
 			<th></th>
 			<th>✓</th> <!-- member -->
 			<th></th>
+			<th>✓</th> <!-- trusted member -->
+			<th></th>
 			<th>✓</th> <!-- editor -->
 			<th>✓</th>
 			<th>✓</th> <!-- moderator -->
@@ -54,6 +60,8 @@ title: Ranks and Permissions
 			<th></th>
 			<th></th> <!-- member -->
 			<th></th>
+			<th></th> <!-- trusted member -->
+			<th></th>
 			<th>✓</th> <!-- editor -->
 			<th>✓</th>
 			<th>✓</th> <!-- moderator -->
@@ -67,6 +75,8 @@ title: Ranks and Permissions
 			<th></th>
 			<th>✓</th> <!-- member -->
 			<th></th>
+			<th>✓</th> <!-- trusted member -->
+			<th></th>
 			<th>✓</th> <!-- editor -->
 			<th>✓</th>
 			<th>✓</th> <!-- moderator -->
@@ -80,6 +90,8 @@ title: Ranks and Permissions
 			<th></th>
 			<th>✓</th> <!-- member -->
 			<th></th>
+			<th>✓</th> <!-- trusted member -->
+			<th></th>
 			<th>✓</th> <!-- editor -->
 			<th>✓</th>
 			<th>✓</th> <!-- moderator -->
@@ -91,7 +103,9 @@ title: Ranks and Permissions
 			<td>Approve Screenshot</td>
 			<th></th> <!-- new -->
 			<th></th>
-			<th>✓</th> <!-- member -->
+			<th></th> <!-- member -->
+			<th></th>
+			<th>✓</th> <!-- trusted member -->
 			<th></th>
 			<th>✓</th> <!-- editor -->
 			<th>✓</th>
@@ -106,6 +120,8 @@ title: Ranks and Permissions
 			<th></th>
 			<th>✓</th> <!-- member -->
 			<th></th>
+			<th>✓</th> <!-- trusted member -->
+			<th></th>
 			<th>✓</th> <!-- editor -->
 			<th>✓</th>
 			<th>✓</th> <!-- moderator -->
@@ -119,6 +135,8 @@ title: Ranks and Permissions
 			<th></th>
 			<th>✓</th> <!-- member -->
 			<th></th>
+			<th>✓</th> <!-- trusted member -->
+			<th></th>
 			<th>✓</th> <!-- editor -->
 			<th>✓</th>
 			<th>✓</th> <!-- moderator -->
@@ -132,6 +150,8 @@ title: Ranks and Permissions
 			<th></th>
 			<th>✓</th> <!-- member -->
 			<th></th>
+			<th>✓</th> <!-- trusted member -->
+			<th></th>
 			<th>✓</th> <!-- editor -->
 			<th>✓</th>
 			<th>✓</th> <!-- moderator -->
@@ -145,6 +165,8 @@ title: Ranks and Permissions
 			<th></th>
 			<th></th> <!-- member -->
 			<th></th>
+			<th>✓</th> <!-- trusted member -->
+			<th></th>
 			<th>✓</th> <!-- editor -->
 			<th>✓</th>
 			<th>✓</th> <!-- moderator -->
@@ -158,6 +180,8 @@ title: Ranks and Permissions
 			<th></th>
 			<th></th> <!-- member -->
 			<th></th>
+			<th></th> <!-- trusted member -->
+			<th></th>
 			<th></th> <!-- editor -->
 			<th></th>
 			<th></th> <!-- moderator -->
@@ -165,12 +189,29 @@ title: Ranks and Permissions
 			<th>✓</th> <!-- admin -->
 			<th>✓</th>
 		</tr>
+		<tr>
+			<td>See Private Thread</td>
+			<th>✓</th> <!-- new -->
+			<th></th>
+			<th>✓</th> <!-- member -->
+			<th></th>
+			<th>✓</th> <!-- trusted member -->
+			<th></th>
+			<th>✓</th> <!-- editor -->
+			<th>✓</th>
+			<th>✓</th> <!-- moderator -->
+			<th>✓</th>
+			<th>✓</th> <!-- admin -->
+			<th>✓</th>
+		</tr>
 		<tr>
 			<td>Set Email</td>
 			<th>✓</th> <!-- new -->
 			<th></th>
 			<th>✓</th> <!-- member -->
 			<th></th>
+			<th>✓</th> <!-- trusted member -->
+			<th></th>
 			<th>✓</th> <!-- editor -->
 			<th></th>
 			<th>✓</th> <!-- moderator -->
@@ -184,6 +225,8 @@ title: Ranks and Permissions
 			<th></th>
 			<th></th> <!-- member -->
 			<th></th>
+			<th></th> <!-- trusted member -->
+			<th></th>
 			<th></th> <!-- editor -->
 			<th></th>
 			<th>✓<sup>3</sup></th> <!-- moderator -->

app/flatpages/help/reporting.md

@@ -0,0 +1,8 @@
+title: Reporting Content
+
+Please let us know if anything on the ContentDB violates our rules or any applicable
+laws.
+
+We take copyright violation and other offenses very seriously.
+
+<a href="https://rubenwardy.com/contact/" class="button btn_green">Contact</a>

app/flatpages/help/wtfpl.md

@@ -0,0 +1,42 @@
+title: WTFPL is a terrible license
+no_h1: true
+
+<div id="warning" class="box box_grey alert alert-warning">
+	<span class="icon_message"></span>
+
+	Please reconsider the choice of WTFPL as a license.
+
+	<script src="/static/jquery.min.js"></script>
+	<script>
+		// @author rubenwardy
+		// @license magnet:?xt=urn:btih:1f739d935676111cfff4b4693e3816e664797050&dn=gpl-3.0.txt GPL-v3-or-Later
+
+		var params = new URLSearchParams(location.search);
+		var r      = params.get("r");
+		if (r)
+			document.write("<a class='alert_right button' href='" + r + "'>Okay</a>");
+		else
+			$("#warning").hide();
+	</script>
+</div>
+
+# WTFPL is a terrible license
+
+The use of WTFPL as a license is discouraged for multiple reasons.
+
+* **No Warranty disclaimer:** This could open you up to being sued.<sup>[1]</sup>
+* **Swearing:** This prevents settings like schools from using your content.
+* **Not OSI Approved:** Same as public domain?
+
+The Open Source Initiative chose not to approve the license as an open-source
+license, saying:<sup>[3]</sup>
+
+> It's no different from dedication to the public domain.
+> Author has submitted license approval request – author is free to make public domain dedication.
+> Although he agrees with the recommendation, Mr. Michlmayr notes that public domain doesn't exist in Europe. Recommend: Reject.
+
+## Sources
+
+1. [WTFPL is harmful to software developers](https://cubicspot.blogspot.com/2017/04/wtfpl-is-harmful-to-software-developers.html)
+2. [FSF](https://www.gnu.org/licenses/license-list.en.html)
+3. [OSI](https://opensource.org/minutes20090304)

app/flatpages/policy_and_guidance.md

@@ -0,0 +1,131 @@
+title: Package Inclusion Policy and Guidance
+
+<div class="box box_grey alert alert-warning">
+	<b>Note:</b> This is a draft
+</div>
+
+## 0. Overview
+
+ContentDB is for the community, and as such listings should be useful to the
+community. To help with this, there are a few rules to improve the quality of
+the listings and to combat abuse.
+
+* No inappropriate content.
+* Content must be playable/useful, but not necessarily finished.
+* Don't use the name of another mod unless your mod is a fork or reimplementation.
+* Licenses must allow derivatives, redistribution, and must not discriminate.
+* Don't put promotions are advertisements in package listings, except for
+  donation and personal website links which are permitted in the long description.
+
+
+## 1. General
+
+It is not permitted to submit abusive, obscene, vulgar, slanderous, hateful,
+threatening, sexually-orientated or any material that may violate any laws be
+it of your country, the country where "Content DB” is hosted or International Law.
+
+The ContentDB admin reserves the right to remove packages for any reason,
+including ones not covered by this document, and to ban users who abuse this service.
+
+Also see the [help page on tags](/help/package_tags/).
+
+
+## 2. Accepted Content and State of Completion
+
+The submission of malware is strictly prohibited. This includes software which
+does not do as it advertises, for example if it posts telemetry without stating
+clearly that it does in the package meta.
+
+ContentDB should only currently contain playable content - content which is
+sufficiently complete to be useful to end users. It's fine to add stuff which
+is still a work in progress (WIP) as long as it adds sufficient value -
+Mineclone 2 is a good example of a WIP package which may break between releases
+but still has value. Note that this doesn't mean that you should add a thing
+you started working on yesterday, it's worth adding all the basic stuff to
+make your package useful.
+
+Adding non-player facing mods, such as libraries and server tools, is perfectly fine
+and encouraged. ContentDB isn't just for player-facing things, and adding
+libraries allows them to be installed when a mod depends on it.
+
+
+## 3. Technical Names
+
+### 3.1 Right to a name
+
+The first package to use a name based on the creation of its forum topic or
+contentdb submission has the right to the technical name. The use of a package
+on a server or in private doesn't reserve its name. No other packages of the same
+type may use the same name, except for the exception given by 3.2.
+
+If it turns out that we made a mistake by approving a package and that the
+name should have been given to another package, then we *may* unapprove the
+package and give the name to the correct one.
+
+If you submit a package where you don't have the right to the name you will be asked
+to change the name of the package, or your package won't be accepted.
+
+We reserve the right to issue exceptions for this where we feel necessary.
+
+### 3.2 Mod Forks and Reimplementations
+
+An exception to the above is that mods are allowed to have the same name as a
+mod if its a fork of that mod (or a close reimplementation). In real terms, it
+should be possible to use the new mod as a drop-in replacement.
+
+We reserve the right to decide whether a mod counts as a fork or
+reimplementation of the mod that owns the name.
+
+
+## 4. Licenses
+
+### 4.1 Allowed Licenses
+
+Please ensure that you correctly credit any resources (code, assets, or otherwise)
+that you have used in your package.
+
+**The use of licenses which do not allow derivatives or redistribution is not
+permitted. This includes CC-ND (No-Derivatives) and lots of closed source licenses.
+The use of licenses which discriminate between groups of people or forbid the use
+of the content on servers or singleplayer is also not permitted.**
+
+However, closed sourced licenses are allowed if they allow the above.
+
+If the license you use is not on the list then please select "Other", and we'll
+get around to adding it.
+
+Please note that the definitions of "free" and "non-free" is the same as that
+of the [Free Software Foundation](https://www.gnu.org/philosophy/free-sw.en.html).
+
+### 4.2 Recommended Licenses
+
+It is highly recommended that you use a free and open source software license.
+FOSS licenses result in a sharing community and will increase the number of potential users your package has.
+Using a closed source license will result in your package being massively penalised in the search results and package lists.
+
+It is recommended that you use a proper license for code with a warranty
+disclaimer, such as the (L)GPL or MIT. You should also use a proper media license
+for media, such as a Creative Commons license.
+
+The use of WTFPL is discouraged as it doesn't contain a [valid warranty disclaimer](https://cubicspot.blogspot.com/2017/04/wtfpl-is-harmful-to-software-developers.html),
+and also includes swearing which prevents settings like schools from using your content.
+[Read more](/help/wtfpl/).
+
+Public domain is not a valid license in many countries, please use CC0 or MIT instead.
+
+
+## 5. Promotions and Advertisements (inc. asking for donations)
+
+Any information other than the long description - including screenshots - must
+not contain any promotions or advertisements. This includes asking for donations,
+promoting online shops, or linking to personal websites and social media.
+
+ContentDB is for the community. We may remove any promotions if we feel that
+they're inappropriate.
+
+Paid promotions are not allowed at all, anywhere.
+
+
+## 6. Reporting Violations
+
+See the [Reporting Content](/help/reporting/) page.

app/maillogger.py

@@ -0,0 +1,109 @@
+import logging
+from enum import Enum
+from app.tasks.emails import sendEmailRaw
+
+def _has_newline(line):
+	"""Used by has_bad_header to check for \\r or \\n"""
+	if line and ("\r" in line or "\n" in line):
+		return True
+	return False
+
+def _is_bad_subject(subject):
+	"""Copied from: flask_mail.py class Message def has_bad_headers"""
+	if _has_newline(subject):
+		for linenum, line in enumerate(subject.split("\r\n")):
+			if not line:
+				return True
+			if linenum > 0 and line[0] not in "\t ":
+				return True
+			if _has_newline(line):
+				return True
+			if len(line.strip()) == 0:
+				return True
+	return False
+
+
+class FlaskMailSubjectFormatter(logging.Formatter):
+	def format(self, record):
+		record.message = record.getMessage()
+		if self.usesTime():
+			record.asctime = self.formatTime(record, self.datefmt)
+		s = self.formatMessage(record)
+		return s
+
+class FlaskMailTextFormatter(logging.Formatter):
+	pass
+
+# TODO: hier nog niet tevreden over (vooral logger.error(..., exc_info, stack_info))
+class FlaskMailHTMLFormatter(logging.Formatter):
+	pre_template = "<h1>%s</h1><pre>%s</pre>"
+	def formatException(self, exc_info):
+		formatted_exception = logging.Handler.formatException(self, exc_info)
+		return FlaskMailHTMLFormatter.pre_template % ("Exception information", formatted_exception)
+	def formatStack(self, stack_info):
+		return FlaskMailHTMLFormatter.pre_template % ("<h1>Stack information</h1><pre>%s</pre>", stack_info)
+
+
+# see: https://github.com/python/cpython/blob/3.6/Lib/logging/__init__.py (class Handler)
+
+class FlaskMailHandler(logging.Handler):
+	def __init__(self, mailer, subject_template, level=logging.NOTSET):
+		logging.Handler.__init__(self, level)
+		self.mailer = mailer
+		self.send_to = mailer.app.config["MAIL_UTILS_ERROR_SEND_TO"]
+		self.subject_template = subject_template
+		self.html_formatter = None
+
+	def setFormatter(self, text_fmt, html_fmt=None):
+		"""
+		Set the formatters for this handler. Provide at least one formatter.
+		When no text_fmt is provided, no text-part is created for the email body.
+		"""
+		assert (text_fmt, html_fmt) != (None, None), "At least one formatter should be provided"
+		if type(text_fmt)==str:
+			text_fmt = FlaskMailTextFormatter(text_fmt)
+		self.formatter = text_fmt
+		if type(html_fmt)==str:
+			html_fmt = FlaskMailHTMLFormatter(html_fmt)
+		self.html_formatter = html_fmt
+
+	def getSubject(self, record):
+		fmt = FlaskMailSubjectFormatter(self.subject_template)
+		subject = fmt.format(record)
+		#Since templates can cause header problems, and we rather have a incomplete email then an error, we fix this
+		if _is_bad_subject(subject):
+			subject="FlaskMailHandler log-entry from %s [original subject is replaced, because it would result in a bad header]" % self.mailer.app.name
+		return subject
+
+	def emit(self, record):
+		text = self.format(record)				if self.formatter	  else None
+		html = self.html_formatter.format(record) if self.html_formatter else None
+		sendEmailRaw.delay(self.send_to, self.getSubject(record), text, html)
+
+
+def register_mail_error_handler(app, mailer):
+	subject_template = "ContentDB crashed (%(module)s > %(funcName)s)"
+	text_template = """
+Message type: %(levelname)s
+Location:	 %(pathname)s:%(lineno)d
+Module:	   %(module)s
+Function:	 %(funcName)s
+Time:		 %(asctime)s
+Message:
+%(message)s"""
+	html_template = """
+<style>th { text-align: right}</style><table>
+<tr><th>Message type:</th><td>%(levelname)s</td></tr>
+<tr>	<th>Location:</th><td>%(pathname)s:%(lineno)d</td></tr>
+<tr>	  <th>Module:</th><td>%(module)s</td></tr>
+<tr>	<th>Function:</th><td>%(funcName)s</td></tr>
+<tr>		<th>Time:</th><td>%(asctime)s</td></tr>
+</table>
+<h2>Message</h2>
+<pre>%(message)s</pre>"""
+
+	import logging
+	mail_handler = FlaskMailHandler(mailer, subject_template)
+	mail_handler.setLevel(logging.ERROR)
+	mail_handler.setFormatter(text_template, html_template)
+	app.logger.addHandler(mail_handler)

app/models.py

@@ -31,13 +31,14 @@ migrate = Migrate(app, db)
 
 
 class UserRank(enum.Enum):
-	BANNED     = 0
-	NOT_JOINED = 1
-	NEW_MEMBER = 2
-	MEMBER     = 3
-	EDITOR     = 4
-	MODERATOR  = 5
-	ADMIN      = 6
+	BANNED         = 0
+	NOT_JOINED     = 1
+	NEW_MEMBER     = 2
+	MEMBER         = 3
+	TRUSTED_MEMBER = 4
+	EDITOR         = 5
+	MODERATOR      = 6
+	ADMIN          = 7
 
 	def atLeast(self, min):
 		return self.value >= min.value
@@ -75,6 +76,8 @@ class Permission(enum.Enum):
 	CHANGE_RANK        = "CHANGE_RANK"
 	CHANGE_EMAIL       = "CHANGE_EMAIL"
 	EDIT_EDITREQUEST   = "EDIT_EDITREQUEST"
+	SEE_THREAD         = "SEE_THREAD"
+	CREATE_THREAD      = "CREATE_THREAD"
 
 	# Only return true if the permission is valid for *all* contexts
 	# See Package.checkPerm for package-specific contexts
@@ -83,27 +86,27 @@ class Permission(enum.Enum):
 			return False
 
 		if self == Permission.APPROVE_NEW or \
-				self == Permission.APPROVE_CHANGES or \
-				self == Permission.APPROVE_RELEASE or \
-				self == Permission.APPROVE_SCREENSHOT:
+				self == Permission.APPROVE_CHANGES    or \
+				self == Permission.APPROVE_RELEASE    or \
+				self == Permission.APPROVE_SCREENSHOT or \
+				self == Permission.SEE_THREAD:
 			return user.rank.atLeast(UserRank.EDITOR)
 		else:
 			raise Exception("Non-global permission checked globally. Use Package.checkPerm or User.checkPerm instead.")
 
-
 class User(db.Model, UserMixin):
 	id = db.Column(db.Integer, primary_key=True)
 
 	# User authentication information
-	username = db.Column(db.String(50), nullable=False, unique=True)
-	password = db.Column(db.String(255), nullable=False, server_default="")
+	username = db.Column(db.String(50, collation="NOCASE"), nullable=False, unique=True, index=True)
+	password = db.Column(db.String(255), nullable=True)
 	reset_password_token = db.Column(db.String(100), nullable=False, server_default="")
 
 	rank = db.Column(db.Enum(UserRank))
 
 	# Account linking
-	github_username = db.Column(db.String(50), nullable=True, unique=True)
-	forums_username = db.Column(db.String(50), nullable=True, unique=True)
+	github_username = db.Column(db.String(50, collation="NOCASE"), nullable=True, unique=True)
+	forums_username = db.Column(db.String(50, collation="NOCASE"), nullable=True, unique=True)
 
 	# User email information
 	email = db.Column(db.String(255), nullable=True, unique=True)
@@ -119,13 +122,18 @@ class User(db.Model, UserMixin):
 	# causednotifs  = db.relationship("Notification", backref="causer", lazy="dynamic")
 	packages      = db.relationship("Package", backref="author", lazy="dynamic")
 	requests      = db.relationship("EditRequest", backref="author", lazy="dynamic")
+	threads       = db.relationship("Thread", backref="author", lazy="dynamic")
+	replies       = db.relationship("ThreadReply", backref="author", lazy="dynamic")
 
-	def __init__(self, username):
+	def __init__(self, username, active=False, email=None, password=None):
 		import datetime
 
 		self.username = username
 		self.confirmed_at = datetime.datetime.now() - datetime.timedelta(days=6000)
 		self.display_name = username
+		self.active = active
+		self.email = email
+		self.password = password
 		self.rank = UserRank.NOT_JOINED
 
 	def canAccessTodoList(self):
@@ -180,12 +188,13 @@ class Notification(db.Model):
 
 
 class License(db.Model):
-	id = db.Column(db.Integer, primary_key=True)
-	name = db.Column(db.String(50), nullable=False, unique=True)
-	packages = db.relationship("Package", backref="license", lazy="dynamic")
+	id      = db.Column(db.Integer, primary_key=True)
+	name    = db.Column(db.String(50), nullable=False, unique=True)
+	is_foss = db.Column(db.Boolean,    nullable=False, default=True)
 
-	def __init__(self, v):
+	def __init__(self, v, is_foss=True):
 		self.name = v
+		self.is_foss = is_foss
 
 	def __str__(self):
 		return self.name
@@ -202,6 +211,13 @@ class PackageType(enum.Enum):
 	def __str__(self):
 		return self.name
 
+	@classmethod
+	def get(cls, name):
+		try:
+			return PackageType[name.upper()]
+		except KeyError:
+			return None
+
 	@classmethod
 	def choices(cls):
 		return [(choice, choice.value) for choice in cls]
@@ -212,43 +228,105 @@ class PackageType(enum.Enum):
 
 
 class PackagePropertyKey(enum.Enum):
-	name         = "Name"
-	title        = "Title"
-	shortDesc    = "Short Description"
-	desc         = "Description"
-	type         = "Type"
-	license      = "License"
-	tags         = "Tags"
-	harddeps     = "Hard Dependencies"
-	softdeps     = "Soft Dependencies"
-	repo         = "Repository"
-	website      = "Website"
-	issueTracker = "Issue Tracker"
-	forums       = "Forum Topic ID"
+	name          = "Name"
+	title         = "Title"
+	shortDesc     = "Short Description"
+	desc          = "Description"
+	type          = "Type"
+	license       = "License"
+	media_license = "Media License"
+	tags          = "Tags"
+	provides      = "Provides"
+	repo          = "Repository"
+	website       = "Website"
+	issueTracker  = "Issue Tracker"
+	forums        = "Forum Topic ID"
 
 	def convert(self, value):
 		if self == PackagePropertyKey.tags:
 			return ",".join([t.title for t in value])
-		elif self == PackagePropertyKey.harddeps or self == PackagePropertyKey.softdeps:
-			return ",".join([t.author.username + "/" + t.name for t in value])
-
+		elif self == PackagePropertyKey.provides:
+			return ",".join([t.name for t in value])
 		else:
 			return str(value)
 
+provides = db.Table("provides",
+	db.Column("package_id",    db.Integer, db.ForeignKey("package.id"), primary_key=True),
+    db.Column("metapackage_id", db.Integer, db.ForeignKey("meta_package.id"), primary_key=True)
+)
+
 tags = db.Table("tags",
     db.Column("tag_id", db.Integer, db.ForeignKey("tag.id"), primary_key=True),
     db.Column("package_id", db.Integer, db.ForeignKey("package.id"), primary_key=True)
 )
 
-harddeps = db.Table("harddeps",
-	db.Column("package_id",    db.Integer, db.ForeignKey("package.id"), primary_key=True),
-    db.Column("dependency_id", db.Integer, db.ForeignKey("package.id"), primary_key=True)
-)
+class Dependency(db.Model):
+	id              = db.Column(db.Integer, primary_key=True)
+	depender_id     = db.Column(db.Integer, db.ForeignKey("package.id"),     nullable=True)
+	package_id      = db.Column(db.Integer, db.ForeignKey("package.id"),     nullable=True)
+	package         = db.relationship("Package", foreign_keys=[package_id])
+	meta_package_id = db.Column(db.Integer, db.ForeignKey("meta_package.id"), nullable=True)
+	optional        = db.Column(db.Boolean, nullable=False, default=False)
+	__table_args__  = (db.UniqueConstraint('depender_id', 'package_id', 'meta_package_id', name='_dependency_uc'), )
+
+	def __init__(self, depender=None, package=None, meta=None):
+		if depender is None:
+			return
+
+		self.depender = depender
+
+		packageProvided = package is not None
+		metaProvided = meta is not None
+
+		if packageProvided and not metaProvided:
+			self.package = package
+		elif metaProvided and not packageProvided:
+			self.meta_package = meta
+		else:
+			raise Exception("Either meta or package must be given, but not both!")
+
+	def __str__(self):
+		if self.package is not None:
+			return self.package.author.username + "/" + self.package.name
+		elif self.meta_package is not None:
+			return self.meta_package.name
+		else:
+			raise Exception("Meta and package are both none!")
+
+	@staticmethod
+	def SpecToList(depender, spec, cache={}):
+		retval = []
+		arr = spec.split(",")
+
+		import re
+		pattern1 = re.compile("^([a-z0-9_]+)$")
+		pattern2 = re.compile("^([A-Za-z0-9_]+)/([a-z0-9_]+)$")
+
+		for x in arr:
+			x = x.strip()
+			if x == "":
+				continue
+
+			if pattern1.match(x):
+				meta = MetaPackage.GetOrCreate(x, cache)
+				retval.append(Dependency(depender, meta=meta))
+			else:
+				m = pattern2.match(x)
+				username = m.group(1)
+				name     = m.group(2)
+				user = User.query.filter_by(username=username).first()
+				if user is None:
+					raise Exception("Unable to find user " + username)
+
+				package = Package.query.filter_by(author=user, name=name).first()
+				if package is None:
+					raise Exception("Unable to find package " + name + " by " + username)
+
+				retval.append(Dependency(depender, package=package))
+
+		return retval
+
 
-softdeps = db.Table("softdeps",
-	db.Column("package_id",    db.Integer, db.ForeignKey("package.id"), primary_key=True),
-    db.Column("dependency_id", db.Integer, db.ForeignKey("package.id"), primary_key=True)
-)
 
 class Package(db.Model):
 	id           = db.Column(db.Integer, primary_key=True)
@@ -262,32 +340,33 @@ class Package(db.Model):
 	type         = db.Column(db.Enum(PackageType))
 	created_at   = db.Column(db.DateTime, nullable=False, default=datetime.utcnow)
 
-	license_id   = db.Column(db.Integer, db.ForeignKey("license.id"))
+	license_id   = db.Column(db.Integer, db.ForeignKey("license.id"), nullable=False, default=1)
+	license      = db.relationship("License", foreign_keys=[license_id])
+	media_license_id = db.Column(db.Integer, db.ForeignKey("license.id"), nullable=False, default=1)
+	media_license    = db.relationship("License", foreign_keys=[media_license_id])
 
 	approved     = db.Column(db.Boolean, nullable=False, default=False)
 	soft_deleted = db.Column(db.Boolean, nullable=False, default=False)
 
+	score        = db.Column(db.Float, nullable=False, default=0)
+
+	review_thread_id = db.Column(db.Integer, db.ForeignKey("thread.id"), nullable=True, default=None)
+	review_thread    = db.relationship("Thread", foreign_keys=[review_thread_id])
+
 	# Downloads
 	repo         = db.Column(db.String(200), nullable=True)
 	website      = db.Column(db.String(200), nullable=True)
 	issueTracker = db.Column(db.String(200), nullable=True)
 	forums       = db.Column(db.Integer,     nullable=True)
 
+	provides = db.relationship("MetaPackage", secondary=provides, lazy="subquery",
+			backref=db.backref("packages", lazy="dynamic"))
+
+	dependencies = db.relationship("Dependency", backref="depender", lazy="dynamic", foreign_keys=[Dependency.depender_id])
+
 	tags = db.relationship("Tag", secondary=tags, lazy="subquery",
 			backref=db.backref("packages", lazy=True))
 
-	harddeps = db.relationship("Package",
-				secondary=harddeps,
-				primaryjoin=id==harddeps.c.package_id,
-				secondaryjoin=id==harddeps.c.dependency_id,
-				backref="dependents")
-
-	softdeps = db.relationship("Package",
-				secondary=softdeps,
-				primaryjoin=id==softdeps.c.package_id,
-				secondaryjoin=id==softdeps.c.dependency_id,
-				backref="softdependents")
-
 	releases = db.relationship("PackageRelease", backref="package",
 			lazy="dynamic", order_by=db.desc("package_release_releaseDate"))
 
@@ -308,20 +387,56 @@ class Package(db.Model):
 		for e in PackagePropertyKey:
 			setattr(self, e.name, getattr(package, e.name))
 
-	def getAsDictionary(self, base_url):
+	def getAsDictionaryShort(self, base_url):
+		tnurl = self.getThumbnailURL()
 		return {
 			"name": self.name,
 			"title": self.title,
 			"author": self.author.display_name,
-			"shortDesc": self.shortDesc,
+			"short_description": self.shortDesc,
 			"type": self.type.toName(),
+			"release": self.getDownloadRelease().id if self.getDownloadRelease() is not None else None,
+			"thumbnail": (base_url + tnurl) if tnurl is not None else None,
+			"score": round(self.score * 10) / 10
+		}
+
+	def getAsDictionary(self, base_url):
+		tnurl = self.getThumbnailURL()
+		return {
+			"author": self.author.display_name,
+			"name": self.name,
+			"title": self.title,
+			"short_description": self.shortDesc,
+			"desc": self.desc,
+			"type": self.type.toName(),
+			"created_at": self.created_at,
+
 			"license": self.license.name,
+			"media_license": self.media_license.name,
+
 			"repo": self.repo,
+			"website": self.website,
+			"issue_tracker": self.issueTracker,
+			"forums": self.forums,
+
+			"provides": [x.name for x in self.provides],
+			"thumbnail": (base_url + tnurl) if tnurl is not None else None,
+			"screenshots": [base_url + ss.url for ss in self.screenshots],
+
 			"url": base_url + self.getDownloadURL(),
 			"release": self.getDownloadRelease().id if self.getDownloadRelease() is not None else None,
-			"screenshots": [base_url + ss.url for ss in self.screenshots]
+
+			"score": round(self.score * 10) / 10
 		}
 
+	def getThumbnailURL(self):
+		screenshot = self.screenshots.filter_by(approved=True).first()
+		return screenshot.getThumbnailURL() if screenshot is not None else None
+
+	def getMainScreenshotURL(self):
+		screenshot = self.screenshots.filter_by(approved=True).first()
+		return screenshot.url if screenshot is not None else None
+
 	def getDetailsURL(self):
 		return url_for("package_page",
 				author=self.author.username, name=self.name)
@@ -354,10 +469,6 @@ class Package(db.Model):
 		return url_for("package_download_page",
 				author=self.author.username, name=self.name)
 
-	def getMainScreenshotURL(self):
-		screenshot = self.screenshots.filter_by(approved=True).first()
-		return screenshot.url if screenshot is not None else None
-
 	def getDownloadRelease(self):
 		for rel in self.releases:
 			if rel.approved:
@@ -365,26 +476,6 @@ class Package(db.Model):
 
 		return None
 
-	def canImportScreenshot(self):
-		if self.repo is None:
-			return False
-
-		url = urlparse(self.repo)
-		if url.netloc == "github.com":
-			return True
-
-		return False
-
-	def canMakeReleaseFromVCS(self):
-		if self.repo is None:
-			return False
-
-		url = urlparse(self.repo)
-		if url.netloc == "github.com":
-			return True
-
-		return False
-
 	def checkPerm(self, user, perm):
 		if not user.is_authenticated:
 			return False
@@ -397,17 +488,22 @@ class Package(db.Model):
 		isOwner = user == self.author
 
 		# Members can edit their own packages, and editors can edit any packages
-		if perm == Permission.MAKE_RELEASE or perm == Permission.ADD_SCREENSHOTS:
+		if perm == Permission.MAKE_RELEASE or perm == Permission.ADD_SCREENSHOTS or perm == Permission.CREATE_THREAD:
 			return isOwner or user.rank.atLeast(UserRank.EDITOR)
 
 		if perm == Permission.EDIT_PACKAGE or perm == Permission.APPROVE_CHANGES:
-			return user.rank.atLeast(UserRank.MEMBER if isOwner else UserRank.EDITOR)
+			if isOwner:
+				return user.rank.atLeast(UserRank.MEMBER if self.approved else UserRank.NEW_MEMBER)
+			else:
+				return user.rank.atLeast(UserRank.EDITOR)
 
-		# Editors can change authors, approve new packages, and approve releases
-		elif perm == Permission.CHANGE_AUTHOR or perm == Permission.APPROVE_NEW \
-				or perm == Permission.APPROVE_RELEASE or perm == Permission.APPROVE_SCREENSHOT:
+		# Editors can change authors and approve new packages
+		elif perm == Permission.APPROVE_NEW or perm == Permission.CHANGE_AUTHOR:
 			return user.rank.atLeast(UserRank.EDITOR)
 
+		elif perm == Permission.APPROVE_RELEASE or perm == Permission.APPROVE_SCREENSHOT:
+			return user.rank.atLeast(UserRank.TRUSTED_MEMBER if isOwner else UserRank.EDITOR)
+
 		# Moderators can delete packages
 		elif perm == Permission.DELETE_PACKAGE or perm == Permission.CHANGE_RELEASE_URL:
 			return user.rank.atLeast(UserRank.MODERATOR)
@@ -415,6 +511,73 @@ class Package(db.Model):
 		else:
 			raise Exception("Permission {} is not related to packages".format(perm.name))
 
+	def recalcScore(self):
+		import datetime
+
+		self.score = 10
+
+		if self.forums is not None:
+			topic = ForumTopic.query.get(self.forums)
+			if topic:
+				days   = (datetime.datetime.now() - topic.created_at).days
+				months = days / 30
+				years  = days / 365
+				self.score = topic.views / max(years, 0.0416) + 80*min(max(months, 0.5), 6)
+
+		if self.getMainScreenshotURL() is None:
+			self.score *= 0.8
+
+		if not self.license.is_foss or not self.media_license.is_foss:
+			self.score *= 0.1
+
+class MetaPackage(db.Model):
+	id           = db.Column(db.Integer, primary_key=True)
+	name         = db.Column(db.String(100), unique=True, nullable=False)
+	dependencies = db.relationship("Dependency", backref="meta_package", lazy="dynamic")
+
+	def __init__(self, name=None):
+		self.name = name
+
+	def __str__(self):
+		return self.name
+
+	@staticmethod
+	def ListToSpec(list):
+		return ",".join([str(x) for x in list])
+
+	@staticmethod
+	def GetOrCreate(name, cache={}):
+		mp = cache.get(name)
+		if mp is None:
+			mp = MetaPackage.query.filter_by(name=name).first()
+
+		if mp is None:
+			mp = MetaPackage(name)
+			db.session.add(mp)
+
+		cache[name] = mp
+		return mp
+
+	@staticmethod
+	def SpecToList(spec, cache={}):
+		retval = []
+		arr = spec.split(",")
+
+		import re
+		pattern = re.compile("^([a-z0-9_]+)$")
+
+		for x in arr:
+			x = x.strip()
+			if x == "":
+				continue
+
+			if not pattern.match(x):
+				continue
+
+			retval.append(MetaPackage.GetOrCreate(x, cache))
+
+		return retval
+
 class Tag(db.Model):
 	id              = db.Column(db.Integer,    primary_key=True)
 	name            = db.Column(db.String(100), unique=True, nullable=False)
@@ -440,6 +603,7 @@ class PackageRelease(db.Model):
 	url          = db.Column(db.String(200), nullable=False)
 	approved     = db.Column(db.Boolean, nullable=False, default=False)
 	task_id      = db.Column(db.String(37), nullable=True)
+	commit_hash  = db.Column(db.String(41), nullable=True, default=None)
 
 
 	def getEditURL(self):
@@ -448,9 +612,24 @@ class PackageRelease(db.Model):
 				name=self.package.name,
 				id=self.id)
 
+	def getDownloadURL(self):
+		return url_for("download_release_page",
+				author=self.package.author.username,
+				name=self.package.name,
+				id=self.id)
+
+
 	def __init__(self):
 		self.releaseDate = datetime.now()
 
+
+class PackageReview(db.Model):
+	id         = db.Column(db.Integer, primary_key=True)
+	package_id = db.Column(db.Integer, db.ForeignKey("package.id"))
+	thread_id  = db.Column(db.Integer, db.ForeignKey("thread.id"), nullable=False)
+	recommend  = db.Column(db.Boolean, nullable=False, default=True)
+
+
 class PackageScreenshot(db.Model):
 	id         = db.Column(db.Integer, primary_key=True)
 	package_id = db.Column(db.Integer, db.ForeignKey("package.id"))
@@ -466,7 +645,7 @@ class PackageScreenshot(db.Model):
 				id=self.id)
 
 	def getThumbnailURL(self):
-		return self.url  # TODO
+		return self.url.replace("/uploads/", "/thumbnails/350x233/")
 
 class EditRequest(db.Model):
 	id           = db.Column(db.Integer, primary_key=True)
@@ -552,45 +731,113 @@ class EditRequestChange(db.Model):
 				tag = Tag.query.filter_by(title=tagTitle.strip()).first()
 				package.tags.append(tag)
 
-		elif self.key == PackagePropertyKey.harddeps:
-			package.harddeps.clear()
-			for pair in self.newValue.split(","):
-				key, value = pair.split("/")
-				if key is None or value is None:
-					continue
-
-				user = User.query.filter_by(username=key).first()
-				if user is None:
-					continue
-
-				dep = Package.query.filter_by(author=user, name=value, soft_deleted=False).first()
-				if dep is None:
-					continue
-
-				package.harddeps.append(dep)
-
-		elif self.key == PackagePropertyKey.softdeps:
-			package.softdeps.clear()
-			for pair in self.newValue.split(","):
-				key, value = pair.split("/")
-				if key is None or value is None:
-					continue
-
-				user = User.query.filter_by(username=key).first()
-				if user is None:
-					raise Exception("No such user!")
-					continue
-
-				dep = Package.query.filter_by(author=user, name=value).first()
-				if dep is None:
-					raise Exception("No such package!")
-					continue
-
-				package.softdeps.append(dep)
-
 		else:
 			setattr(package, self.key.name, self.newValue)
 
+
+watchers = db.Table("watchers",
+    db.Column("user_id", db.Integer, db.ForeignKey("user.id"), primary_key=True),
+    db.Column("thread_id", db.Integer, db.ForeignKey("thread.id"), primary_key=True)
+)
+
+class Thread(db.Model):
+	id         = db.Column(db.Integer, primary_key=True)
+
+	package_id = db.Column(db.Integer, db.ForeignKey("package.id"), nullable=True)
+	package    = db.relationship("Package", foreign_keys=[package_id])
+
+	author_id  = db.Column(db.Integer, db.ForeignKey("user.id"), nullable=False)
+	title      = db.Column(db.String(100), nullable=False)
+	private    = db.Column(db.Boolean, server_default="0")
+
+	created_at = db.Column(db.DateTime, nullable=False, default=datetime.utcnow)
+
+	replies    = db.relationship("ThreadReply", backref="thread", lazy="dynamic")
+
+	watchers   = db.relationship("User", secondary=watchers, lazy="subquery", \
+						backref=db.backref("watching", lazy=True))
+
+
+	def getSubscribeURL(self):
+		return url_for("thread_subscribe_page",
+				id=self.id)
+
+	def getUnsubscribeURL(self):
+		return url_for("thread_unsubscribe_page",
+				id=self.id)
+
+	def checkPerm(self, user, perm):
+		if not user.is_authenticated:
+			return not self.private
+
+		if type(perm) == str:
+			perm = Permission[perm]
+		elif type(perm) != Permission:
+			raise Exception("Unknown permission given to Thread.checkPerm()")
+
+		isOwner = user == self.author or (self.package is not None and self.package.author == user)
+
+		if perm == Permission.SEE_THREAD:
+			return not self.private or isOwner or user.rank.atLeast(UserRank.EDITOR)
+
+		else:
+			raise Exception("Permission {} is not related to threads".format(perm.name))
+
+class ThreadReply(db.Model):
+	id         = db.Column(db.Integer, primary_key=True)
+	thread_id  = db.Column(db.Integer, db.ForeignKey("thread.id"), nullable=False)
+	comment    = db.Column(db.String(500), nullable=False)
+	author_id  = db.Column(db.Integer, db.ForeignKey("user.id"), nullable=False)
+	created_at = db.Column(db.DateTime, nullable=False, default=datetime.utcnow)
+
+
+REPO_BLACKLIST = [".zip", "mediafire.com", "dropbox.com", "weebly.com", \
+		"minetest.net", "dropboxusercontent.com", "4shared.com", \
+		"digitalaudioconcepts.com", "hg.intevation.org", "www.wtfpl.net", \
+		"imageshack.com", "imgur.com"]
+
+class ForumTopic(db.Model):
+	topic_id  = db.Column(db.Integer, primary_key=True, autoincrement=False)
+	author_id = db.Column(db.Integer, db.ForeignKey("user.id"), nullable=False)
+	author    = db.relationship("User")
+
+	wip       = db.Column(db.Boolean, server_default="0")
+
+	type      = db.Column(db.Enum(PackageType), nullable=False)
+	title     = db.Column(db.String(200), nullable=False)
+	name      = db.Column(db.String(30), nullable=True)
+	link      = db.Column(db.String(200), nullable=True)
+
+	posts     = db.Column(db.Integer, nullable=False)
+	views     = db.Column(db.Integer, nullable=False)
+
+	created_at = db.Column(db.DateTime, nullable=False, default=datetime.utcnow)
+
+	def getRepoURL(self):
+		if self.link is None:
+			return None
+
+		for item in REPO_BLACKLIST:
+			if item in self.link:
+				return None
+
+		return self.link.replace("repo.or.cz/w/", "repo.or.cz/")
+
+	def getAsDictionary(self):
+		return {
+			"author": self.author.username,
+			"name":   self.name,
+			"type":   self.type.toName(),
+			"title":  self.title,
+			"id":     self.topic_id,
+			"link":   self.link,
+			"posts":  self.posts,
+			"views":  self.views,
+			"is_wip": self.wip,
+			"created_at": self.created_at.isoformat(),
+		}
+
+
 # Setup Flask-User
 db_adapter = SQLAlchemyAdapter(db, User)        # Register the User model
 user_manager = UserManager(db_adapter, app)     # Initialize Flask-User

app/public/static/package_create.js

@@ -9,33 +9,44 @@ $(function() {
 		$(".pkg_meta").show()
 	}
 
-	function repoIsSupported(url) {
-		try {
-			return URI(url).hostname() == "github.com"
-		} catch(e) {
-			return false
-		}
-	}
-
 	$(".pkg_meta").hide()
 	$(".pkg_wiz_1").show()
+
+	$("#pkg_wiz_1_skip").click(finish)
 	$("#pkg_wiz_1_next").click(function() {
 		const repoURL = $("#repo").val();
-		if (repoIsSupported(repoURL)) {
+		if (repoURL.trim() != "") {
 			$(".pkg_wiz_1").hide()
 			$(".pkg_wiz_2").show()
 			$(".pkg_repo").hide()
 
+			function setSpecial(id, value) {
+				if (value != "") {
+					var ele = $(id);
+					ele.val(value);
+					ele.trigger("change")
+				}
+			}
+
 			performTask("/tasks/getmeta/new/?url=" + encodeURI(repoURL)).then(function(result) {
-				$("#name").val(result.name || "")
-				$("#title").val(result.title || "")
+				$("#name").val(result.name)
+				setSpecial("#provides_str", result.provides)
+				$("#title").val(result.title)
 				$("#repo").val(result.repo || repoURL)
-				$("#issueTracker").val(result.issueTracker || "")
-				$("#desc").val(result.description || "")
-				$("#shortDesc").val(result.short_description || "")
+				$("#issueTracker").val(result.issueTracker)
+				$("#desc").val(result.description)
+				$("#shortDesc").val(result.short_description)
+				setSpecial("#harddep_str", result.depends)
+				setSpecial("#softdep_str", result.optional_depends)
+				$("#shortDesc").val(result.short_description)
 				if (result.forumId) {
 					$("#forums").val(result.forumId)
 				}
+
+				if (result.type && result.type.length > 2) {
+					$("#type").val(result.type)
+				}
+
 				finish()
 			}).catch(function(e) {
 				alert(e)

app/public/static/package_edit.js

@@ -0,0 +1,11 @@
+// @author rubenwardy
+// @license magnet:?xt=urn:btih:1f739d935676111cfff4b4693e3816e664797050&dn=gpl-3.0.txt GPL-v3-or-Later
+
+$(function() {
+	$("#type").change(function() {
+		$(".not_mod, .not_game, .not_txp").show()
+		$(".not_" + this.value.toLowerCase()).hide()
+	})
+	$(".not_mod, .not_game, .not_txp").show()
+	$(".not_" + $("#type").val().toLowerCase()).hide()
+})

app/public/static/polltask.js

@@ -22,7 +22,7 @@ function pollTask(poll_url, disableTimeout) {
 		var tries = 0;
 		function retry() {
 			tries++;
-			if (!disableTimeout && tries > 10) {
+			if (!disableTimeout && tries > 30) {
 				reject("timeout")
 			} else {
 				const interval = Math.min(tries*100, 1000)

app/public/static/tagselector.js

@@ -5,7 +5,7 @@
  * https://petprojects.googlecode.com/svn/trunk/GPL-LICENSE.txt
  */
 (function($) {
-	$.fn.tagSelector = function(source, name, select) {
+	$.fn.selectSelector = function(source, name, select) {
 		return this.each(function() {
 				var selector = $(this),
 					input = $('input[type=text]', this);
@@ -80,15 +80,136 @@
 			});
 	}
 
+	$.fn.csvSelector = function(source, name, result, allowSlash) {
+		return this.each(function() {
+				var selector = $(this),
+					input = $('input[type=text]', this);
+
+				var selected = [];
+				var lookup = {};
+				for (var i = 0; i < source.length; i++) {
+					lookup[source[i].id] = source[i];
+				}
+
+				selector.click(function() { input.focus(); })
+					.delegate('.tag a', 'click', function() {
+						var id = $(this).parent().data("id");
+						for (var i = 0; i < selected.length; i++) {
+							if (selected[i] == id) {
+								selected.splice(i, 1);
+							}
+						}
+						recreate();
+					});
+
+				function selectItem(id) {
+					for (var i = 0; i < selected.length; i++) {
+						if (selected[i] == id) {
+							return false;
+						}
+					}
+					selected.push(id);
+					return true;
+				}
+
+				function addTag(id, value) {
+					var tag = $('<span class="tag"/>')
+						.text(value)
+						.data("id", id)
+						.append(' <a>x</a>')
+						.insertBefore(input);
+
+					input.attr("placeholder", null);
+				}
+
+				function recreate() {
+					selector.find("span").remove();
+					for (var i = 0; i < selected.length; i++) {
+						var value = lookup[selected[i]] || { value: selected[i] };
+						addTag(selected[i], value.value);
+					}
+					result.val(selected.join(","))
+				}
+
+				function readFromResult() {
+					selected = [];
+					var selected_raw = result.val().split(",");
+					for (var i = 0; i < selected_raw.length; i++) {
+						var raw = selected_raw[i].trim();
+						if (lookup[raw] || raw.match(/^([a-z0-9_]+)$/)) {
+							selected.push(raw);
+						}
+					}
+
+					recreate();
+				}
+				readFromResult();
+
+				result.change(readFromResult);
+
+				input.keydown(function(e) {
+						if (e.keyCode === $.ui.keyCode.TAB && $(this).data('ui-autocomplete').menu.active)
+							e.preventDefault();
+						else if (e.keyCode === $.ui.keyCode.COMMA) {
+							var item = input.val();
+							if (item.length == 0) {
+								input.data("ui-autocomplete").search("");
+							} else if (item.match(/^([a-z0-9_]+)$/)) {
+								selectItem(item);
+								recreate();
+								input.val("");
+							} else {
+								alert("Only lowercase alphanumeric and number names allowed.");
+							}
+							e.preventDefault();
+							return true;
+						} else if (e.keyCode === $.ui.keyCode.BACKSPACE) {
+							if (input.val() == "") {
+								var item = selected[selected.length - 1];
+								selected.splice(selected.length - 1, 1);
+								recreate();
+								if (!(item.indexOf("/") > 0))
+									input.val(item);
+								e.preventDefault();
+								return true;
+							}
+						}
+					})
+					.autocomplete({
+						minLength: 0,
+						source: source,
+						select: function(event, ui) {
+							selectItem(ui.item.id);
+							recreate();
+							input.val("");
+							return false;
+						}
+					});
+
+				input.data('ui-autocomplete')._renderItem = function(ul, item) {
+						return $('<li/>')
+							.data('item.autocomplete', item)
+							.append($('<a/>').text(item.toString()))
+							.appendTo(ul);
+					};
+
+				input.data('ui-autocomplete')._resizeMenu = function(ul, item) {
+						var ul = this.menu.element;
+						ul.outerWidth(Math.max(
+							ul.width('').outerWidth(),
+							selector.outerWidth()
+						));
+					};
+			});
+	}
+
 	$(function() {
 		$(".multichoice_selector").each(function() {
 			var ele = $(this);
 			var sel = ele.parent().find("select");
-			console.log(sel.attr("name"));
-			sel.css("display", "none");
+			sel.hide();
 
 			var options = [];
-
 			sel.find("option").each(function() {
 				var text = $(this).text();
 				options.push({
@@ -100,7 +221,19 @@
 			});
 
 			console.log(options);
-			ele.tagSelector(options, sel.attr("name"), sel);
-		})
+			ele.selectSelector(options, sel.attr("name"), sel);
+		});
+
+		$(".metapackage_selector").each(function() {
+			var input = $(this).parent().children("input[type='text']");
+			input.hide();
+			$(this).csvSelector(meta_packages, input.attr("name"), input);
+		});
+
+		$(".deps_selector").each(function() {
+			var input = $(this).parent().children("input[type='text']");
+			input.hide();
+			$(this).csvSelector(all_packages, input.attr("name"), input);
+		});
 	});
 })(jQuery);

app/scss/comments.scss

@@ -0,0 +1,49 @@
+.comments, .comments li {
+	list-style: none;
+	padding: 0;
+	margin: 0;
+	border: 1px solid #444;
+}
+
+.comments {
+	border-radius: 5px;
+	margin: 15px 0;
+	background: #333;
+
+	.info_strip, .msg {
+		display: block;
+		margin: 0;
+	}
+
+	.info_strip {
+		padding: 0.2em 1em;
+		border-bottom: 1px solid #444;
+	}
+
+	.msg {
+		padding: 1em;
+		background: #222;
+	}
+
+	.author {
+		font-weight: bold;
+		float: left;
+		display: inline-block;
+	}
+
+	.info_strip span {
+		float: right;
+		display: inline-block;
+		color: #bbb;
+	}
+}
+
+.comment_form {
+	margin: 1em 0;
+}
+
+.comment_form textarea {
+	min-width: 60%;
+	max-width: 100%;
+	margin: 0 0 1em 0;
+}

app/scss/components.scss

@@ -25,7 +25,7 @@ a:hover {
 	padding: 0;
 }
 
-.box h2, .box h3 {
+.box > h2, .box > h3 {
 	margin: 0;
 	padding: 0.5em 0.5em 0.5em 15px;
 	border-bottom: 1px solid #444;
@@ -87,7 +87,7 @@ a:hover {
 }
 
 .button, .buttonset li a, input[type=submit], input[type=text],
-		input[type=password], textarea, select, .multichoice_selector {
+		input[type=password], textarea, select, .bulletselector {
 	text-align: center;
 	display: inline-block;
 	padding: 0.4em 1em;
@@ -99,7 +99,11 @@ a:hover {
 	font-size: 100%;
 }
 
-input[type=text], input[type=password], textarea, select, .multichoice_selector {
+select > * {
+	color: #222;
+}
+
+input[type=text], input[type=password], textarea, select, .bulletselector {
 	text-align: left;
 }
 
@@ -147,13 +151,13 @@ select:not([multiple]) {
 	padding: 0 8px 8px 0;
 }
 
-.form-group input, .form-group textarea, .form-group .multichoice_selector {
+.form-group input, .form-group textarea, .form-group .bulletselector {
 	display: block;
 	min-width: 100%;
 	max-width: 100%;
 }
 
-.box .form-group input, .box .form-group textarea, .form-group .multichoice_selector {
+.box .form-group input, .box .form-group textarea, .form-group .bulletselector {
 	min-width: 95%;
 	max-width: 95%;
 }
@@ -197,7 +201,7 @@ select:not([multiple]) {
 }
 
 
-.multichoice_selector input {
+.bulletselector input {
 	border: none;
 	border-radius: 0;
     -moz-border-radius: 0;
@@ -211,7 +215,7 @@ select:not([multiple]) {
 	white-space: nowrap;
 	background: transparent;
 }
-.multichoice_selector .tag {
+.bulletselector .tag {
 	background: #375D81;
 	border-radius: 3px;
     -moz-border-radius: 3px;
@@ -223,11 +227,11 @@ select:not([multiple]) {
 	margin-bottom: 0.3em;
     vertical-align: baseline;
 }
-.multichoice_selector .tag a {
+.bulletselector .tag a {
 	color: #FFF;
 	cursor: pointer;
 }
-.multichoice_selector .tag a:hover {
+.bulletselector .tag a:hover {
 	color: #0099CC;
 	text-decoration: none;
 }
@@ -235,43 +239,51 @@ select:not([multiple]) {
 
 /* Alerts */
 
-.alert {
-	padding: 10px;
-	position: relative;
-}
 
 .alert .alert_right, .alert > form {
 	display: inline-block;
 	margin: 0;
-	padding: 0;
 	position: absolute;
 	top: 0;
 	right: 0;
 	bottom: 0;
 }
 
-.alert .alert_right form {
-	height: 100%;
-}
+.alert {
+	padding: 10px;
+	position: relative;
 
-.alert form {
-	display: inline-block;
-	margin: 0;
-	padding: 0;
-}
+	.alert_right:not(.button) {
+		padding: 0;
+	}
 
-.alert input {
-	margin: 0;
-	height: 100%;
-	background: 0;
-	border: 0;
-	border-left: 1px solid rgba(255,255,255,0.12);
-	border-radius: 0;
-}
+	.alert_right form {
+		height: 100%;
+	}
 
-.alert input:hover {
-	border: 0;
-	border-left: 1px solid rgba(255,255,255,0.2);
+	form {
+		display: inline-block;
+		margin: 0;
+		padding: 0;
+	}
+
+	input {
+		height: 100%;
+	}
+
+	input, .button {
+		margin: 0;
+		background: 0;
+		border: 0;
+		border-left: 1px solid rgba(255,255,255,0.12);
+		border-radius: 0;
+		vertical-align: middle;
+	}
+
+	input:hover, .button:hover {
+		border: 0;
+		border-left: 1px solid rgba(255,255,255,0.2);
+	}
 }
 
 #alerts {
@@ -280,6 +292,7 @@ select:not([multiple]) {
 	bottom: 15px;
 	left: 0;
 	right: 0;
+	z-index: 1000;
 }
 
 #alerts .alert {
@@ -306,6 +319,11 @@ select:not([multiple]) {
 	border: 1px solid #c96;
 }
 
+.alert-primary {
+	background: #339;
+	border: 1px solid #66a;
+}
+
 .alert-success {
 	background: #161;
 	border: 1px solid #393;
@@ -383,6 +401,10 @@ table.fancyTable tfoot td {
 	color: #b6f;
 }
 
+.TRUSTED_MEMBER a, a.TRUSTED_MEMBER {
+	color: #2c2;
+}
+
 /*
 	Aside
 */
@@ -433,3 +455,19 @@ table.fancyTable tfoot td {
 .table-topalign td {
 	vertical-align: top;
 }
+
+.wiptopic a {
+	color: #7ac;
+}
+
+.editor-toolbar {
+	background-color: #333 !important;
+}
+
+.CodeMirror {
+	background-color: #222 !important;
+}
+
+.editor-preview-side {
+	background-color: #222 !important;
+}

app/scss/main.scss

@@ -3,3 +3,4 @@
 @import "nav.scss";
 @import "packages.scss";
 @import "packagegrid.scss";
+@import "comments.scss";

app/scss/packagegrid.scss

@@ -13,6 +13,7 @@
 	display: block;
 	min-width: 300px;
 	min-height: 200px;
+	max-width: 332px;
 	padding: 0;
 	margin: 7px;
 }

app/tasks/__init__.py

@@ -18,6 +18,7 @@
 import flask
 from flask.ext.sqlalchemy import SQLAlchemy
 from celery import Celery
+from celery.schedules import crontab
 from app import app
 from app.models import *
 
@@ -64,4 +65,12 @@ def make_celery(app):
 
 celery = make_celery(app)
 
+CELERYBEAT_SCHEDULE = {
+	'topic_list_import': {
+		'task': 'app.tasks.forumtasks.importTopicList',
+		'schedule': crontab(minute=1, hour=1),
+	}
+}
+celery.conf.beat_schedule = CELERYBEAT_SCHEDULE
+
 from . import importtasks, forumtasks, emails

app/tasks/emails.py

@@ -22,7 +22,17 @@ from app.tasks import celery
 
 @celery.task()
 def sendVerifyEmail(newEmail, token):
-    msg = Message("Verify email address", recipients=[newEmail])
-    msg.body = "This is a verification email!"
-    msg.html = render_template("emails/verify.html", token=token)
-    mail.send(msg)
+	msg = Message("Verify email address", recipients=[newEmail])
+	msg.body = "This is a verification email!"
+	msg.html = render_template("emails/verify.html", token=token)
+	mail.send(msg)
+
+@celery.task()
+def sendEmailRaw(to, subject, text, html):
+	from flask_mail import Message
+	msg = Message(subject, recipients=to)
+	if text:
+		msg.body = text
+	if html:
+		msg.html = html
+	mail.send(msg)

app/tasks/forumtasks.py

@@ -15,12 +15,12 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 
-import flask, json
+import flask, json, re
 from flask.ext.sqlalchemy import SQLAlchemy
 from app import app
 from app.models import *
 from app.tasks import celery
-from .phpbbparser import getProfile
+from .phpbbparser import getProfile, getTopicsFromForum
 import urllib.request
 from urllib.parse import urlparse, quote_plus
 
@@ -51,27 +51,94 @@ def checkForumAccount(username, token=None):
 	if needsSaving:
 		db.session.commit()
 
+
+regex_tag    = re.compile(r"\[([a-z0-9_]+)\]")
+BANNED_NAMES = ["mod", "game", "old", "outdated", "wip", "api", "beta", "alpha", "git"]
+def getNameFromTaglist(taglist):
+	for tag in reversed(regex_tag.findall(taglist)):
+		if len(tag) < 30 and not tag in BANNED_NAMES and \
+				not re.match(r"^[a-z]?[0-9]+$", tag):
+			return tag
+
+	return None
+
+regex_title = re.compile(r"^((?:\[[^\]]+\] *)*)([^\[]+) *((?:\[[^\]]+\] *)*)[^\[]*$")
+def parseTitle(title):
+	m = regex_title.match(title)
+	if m is None:
+		print("Invalid title format: " + title)
+		return title, getNameFromTaglist(title)
+	else:
+		return m.group(2).strip(), getNameFromTaglist(m.group(3))
+
+def getLinksFromModSearch():
+	links = {}
+
+	contents = urllib.request.urlopen("https://krock-works.uk.to/minetest/modList.php").read().decode("utf-8")
+	for x in json.loads(contents):
+		link = x.get("link")
+		if link is not None:
+			links[int(x["topicId"])] = link
+
+	return links
+
 @celery.task()
-def importUsersFromModList():
-	contents = urllib.request.urlopen("http://krock-works.16mb.com/MTstuff/modList.php").read().decode("utf-8")
-	list = json.loads(contents)
-	found = {}
-	imported = []
+def importTopicList():
+	links_by_id = getLinksFromModSearch()
 
-	for user in User.query.all():
-		found[user.username] = True
-		if user.forums_username is not None:
-			found[user.forums_username] = True
+	info_by_id = {}
+	getTopicsFromForum(11, out=info_by_id, extra={ 'type': PackageType.MOD,  'wip': False })
+	getTopicsFromForum(9,  out=info_by_id, extra={ 'type': PackageType.MOD,  'wip': True  })
+	getTopicsFromForum(15, out=info_by_id, extra={ 'type': PackageType.GAME, 'wip': False })
+	getTopicsFromForum(50, out=info_by_id, extra={ 'type': PackageType.GAME, 'wip': True  })
 
-	for x in list:
-		author = x.get("author")
-		if author is not None and not author in found:
-			user = User(author)
-			user.forums_username = author
-			imported.append(author)
-			found[author] = True
-			db.session.add(user)
+	# Caches
+	username_to_user = {}
+	topics_by_id     = {}
+	for topic in ForumTopic.query.all():
+		topics_by_id[topic.topic_id] = topic
+
+	# Create or update
+	for info in info_by_id.values():
+		id = int(info["id"])
+
+		# Get author
+		username = info["author"]
+		user = username_to_user.get(username)
+		if user is None:
+			user = User.query.filter_by(forums_username=username).first()
+			if user is None:
+				print(username + " not found!")
+				user = User(username)
+				user.forums_username = username
+				db.session.add(user)
+			username_to_user[username] = user
+
+		# Get / add row
+		topic = topics_by_id.get(id)
+		if topic is None:
+			topic = ForumTopic()
+			db.session.add(topic)
+
+		# Parse title
+		title, name = parseTitle(info["title"])
+
+		# Get link
+		link = links_by_id.get(id)
+
+		# Fill row
+		topic.topic_id   = int(id)
+		topic.author     = user
+		topic.type       = info["type"]
+		topic.title      = title
+		topic.name       = name
+		topic.link       = link
+		topic.wip        = info["wip"]
+		topic.posts      = int(info["posts"])
+		topic.views      = int(info["views"])
+		topic.created_at = info["date"]
+
+	for p in Package.query.all():
+		p.recalcScore()
 
 	db.session.commit()
-	for author in found:
-		checkForumAccount.delay(author, None)

app/tasks/importtasks.py

@@ -15,16 +15,18 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 
-import flask, json, os
+import flask, json, os, git, tempfile, shutil
+from git import GitCommandError
 from flask.ext.sqlalchemy import SQLAlchemy
 from urllib.error import HTTPError
 import urllib.request
-from urllib.parse import urlparse, quote_plus
+from urllib.parse import urlparse, quote_plus, urlsplit
 from app import app
 from app.models import *
 from app.tasks import celery, TaskError
 from app.utils import randomString
 
+
 class GithubURLMaker:
 	def __init__(self, url):
 		# Rewrite path
@@ -46,15 +48,6 @@ class GithubURLMaker:
 	def getRepoURL(self):
 		return "https://github.com/{}/{}".format(self.user, self.repo)
 
-	def getIssueTrackerURL(self):
-		return "https://github.com/{}/{}/issues/".format(self.user, self.repo)
-
-	def getModConfURL(self):
-		return self.baseUrl + "/mod.conf"
-
-	def getDescURL(self):
-		return self.baseUrl + "/description.txt"
-
 	def getScreenshotURL(self):
 		return self.baseUrl + "/screenshot.png"
 
@@ -66,7 +59,6 @@ class GithubURLMaker:
 		return "https://github.com/{}/{}/archive/{}.zip" \
 				.format(self.user, self.repo, commit)
 
-
 krock_list_cache = None
 krock_list_cache_by_name = None
 def getKrockList():
@@ -74,7 +66,7 @@ def getKrockList():
 	global krock_list_cache_by_name
 
 	if krock_list_cache is None:
-		contents = urllib.request.urlopen("http://krock-works.16mb.com/MTstuff/modList.php").read().decode("utf-8")
+		contents = urllib.request.urlopen("https://krock-works.uk.to/minetest/modList.php").read().decode("utf-8")
 		list = json.loads(contents)
 
 		def h(x):
@@ -94,9 +86,9 @@ def getKrockList():
 			return {
 				"title":   x["title"],
 				"author":  x["author"],
-				"name":    x["name"],
+				"name":	x["name"],
 				"topicId": x["topicId"],
-				"link":    x["link"],
+				"link":	x["link"],
 			}
 
 		krock_list_cache = [g(x) for x in list if h(x)]
@@ -140,76 +132,209 @@ def parseConf(string):
 	return retval
 
 
-@celery.task()
-def getMeta(urlstr, author):
-	url = urlparse(urlstr)
+class PackageTreeNode:
+	def __init__(self, baseDir, author=None, repo=None, name=None):
+		print("Scanning " + baseDir)
+		self.baseDir  = baseDir
+		self.author   = author
+		self.name	 = name
+		self.repo	 = repo
+		self.meta	 = None
+		self.children = []
 
-	urlmaker = None
-	if url.netloc == "github.com":
-		urlmaker = GithubURLMaker(url)
-	else:
-		raise TaskError("Unsupported repo")
+		# Detect type
+		type = None
+		is_modpack = False
+		if os.path.isfile(baseDir + "/game.conf"):
+			type = PackageType.GAME
+		elif os.path.isfile(baseDir + "/init.lua"):
+			type = PackageType.MOD
+		elif os.path.isfile(baseDir + "/modpack.txt") or \
+				os.path.isfile(baseDir + "/modpack.conf"):
+			type = PackageType.MOD
+			is_modpack = True
+		elif os.path.isdir(baseDir + "/mods"):
+			type = PackageType.GAME
+		elif os.listdir(baseDir) == []:
+			# probably a submodule
+			return
+		else:
+			raise TaskError("Unable to detect package type!")
 
-	if not urlmaker.isValid():
-		raise TaskError("Error! Url maker not valid")
+		self.type = type
+		self.readMetaFiles()
 
-	result = {}
+		if self.type == PackageType.GAME:
+			self.addChildrenFromModDir(baseDir + "/mods")
+		elif is_modpack:
+			self.addChildrenFromModDir(baseDir)
 
-	result["repo"] = urlmaker.getRepoURL()
-	result["issueTracker"] = urlmaker.getIssueTrackerURL()
 
-	try:
-		contents = urllib.request.urlopen(urlmaker.getModConfURL()).read().decode("utf-8")
-		conf = parseConf(contents)
-		for key in ["name", "description", "title"]:
-			try:
-				result[key] = conf[key]
-			except KeyError:
-				pass
-	except HTTPError:
-		print("mod.conf does not exist")
+	def readMetaFiles(self):
+		result = {}
 
-	if "name" in result:
-		result["title"] = result["name"].replace("_", " ").title()
-
-	if not "description" in result:
+		# .conf file
 		try:
-			contents = urllib.request.urlopen(urlmaker.getDescURL()).read().decode("utf-8")
-			result["description"] = contents.strip()
-		except HTTPError:
+			with open(self.baseDir + "/mod.conf", "r") as myfile:
+				conf = parseConf(myfile.read())
+				for key in ["name", "description", "title", "depends", "optional_depends"]:
+					try:
+						result[key] = conf[key]
+					except KeyError:
+						pass
+		except IOError:
 			print("description.txt does not exist!")
 
-	if "description" in result:
-		desc = result["description"]
-		idx = desc.find(".") + 1
-		cutIdx = min(len(desc), 200 if idx < 5 else idx)
-		result["short_description"] = desc[:cutIdx]
+		# description.txt
+		if not "description" in result:
+			try:
+				with open(self.baseDir + "/description.txt", "r") as myfile:
+					result["description"] = myfile.read()
+			except IOError:
+				print("description.txt does not exist!")
 
-	info = findModInfo(author, result.get("name"), result["repo"])
-	if info is not None:
-		result["forumId"] = info.get("topicId")
+		# depends.txt
+		import re
+		pattern = re.compile("^([a-z0-9_]+)\??$")
+		if not "depends" in result and not "optional_depends" in result:
+			try:
+				with open(self.baseDir + "/depends.txt", "r") as myfile:
+					contents = myfile.read()
+					soft = []
+					hard = []
+					for line in contents.split("\n"):
+						line = line.strip()
+						if pattern.match(line):
+							if line[len(line) - 1] == "?":
+								soft.append( line[:-1])
+							else:
+								hard.append(line)
+
+					result["depends"] = hard
+					result["optional_depends"] = soft
+
+			except IOError:
+				print("depends.txt does not exist!")
+
+		else:
+			if "depends" in result:
+				result["depends"] = [x.strip() for x in result["depends"].split(",")]
+			if "optional_depends" in result:
+				result["optional_depends"] = [x.strip() for x in result["optional_depends"].split(",")]
+
+
+		# Calculate Title
+		if "name" in result and not "title" in result:
+			result["title"] = result["name"].replace("_", " ").title()
+
+		# Calculate short description
+		if "description" in result:
+			desc = result["description"]
+			idx = desc.find(".") + 1
+			cutIdx = min(len(desc), 200 if idx < 5 else idx)
+			result["short_description"] = desc[:cutIdx]
+
+		# Get forum ID
+		info = findModInfo(self.author, result.get("name"), self.repo)
+		if info is not None:
+			result["forumId"] = info.get("topicId")
+
+		if "name" in result:
+			self.name = result["name"]
+			del result["name"]
+
+		self.meta = result
+
+	def addChildrenFromModDir(self, dir):
+		for entry in next(os.walk(dir))[1]:
+			path = dir + "/" + entry
+			if not entry.startswith('.') and os.path.isdir(path):
+				self.children.append(PackageTreeNode(path, name=entry))
+
+
+	def fold(self, attr, key=None, acc=None):
+		if acc is None:
+			acc = set()
+
+		if self.meta is None:
+			return acc
+
+		at = getattr(self, attr)
+		value = at if key is None else at.get(key)
+
+		if isinstance(value, list):
+			acc |= set(value)
+		elif value is not None:
+			acc.add(value)
+
+		for child in self.children:
+			child.fold(attr, key, acc)
+
+		return acc
+
+	def get(self, key):
+		return self.meta.get(key)
+
+def generateGitURL(urlstr):
+	scheme, netloc, path, query, frag = urlsplit(urlstr)
+
+	return "http://:@" + netloc + path + query
+
+# Clones a repo from an unvalidated URL.
+# Returns a tuple of path and repo on sucess.
+# Throws `TaskError` on failure.
+# Caller is responsible for deleting returned directory.
+def cloneRepo(urlstr, ref=None, recursive=False):
+	gitDir = tempfile.gettempdir() + "/" + randomString(10)
+
+	err = None
+	try:
+		gitUrl = generateGitURL(urlstr)
+		print("Cloning from " + gitUrl)
+		repo = git.Repo.clone_from(gitUrl, gitDir, \
+				progress=None, env=None, depth=1, recursive=recursive, kill_after_timeout=15)
+
+		if ref is not None:
+			repo.create_head("myhead", ref).checkout()
+		return gitDir, repo
+	except GitCommandError as e:
+		# This is needed to stop the backtrace being weird
+		err = e.stderr
+
+	raise TaskError(err.replace("stderr: ", "") \
+			.replace("Cloning into '" + gitDir + "'...", "") \
+			.strip())
+
+@celery.task()
+def getMeta(urlstr, author):
+	gitDir, _ = cloneRepo(urlstr, recursive=True)
+	tree = PackageTreeNode(gitDir, author=author, repo=urlstr)
+	shutil.rmtree(gitDir)
+
+	result = {}
+	result["name"] = tree.name
+	result["provides"] = tree.fold("name")
+	result["type"] = tree.type.name
+
+	for key in ["depends", "optional_depends"]:
+		result[key] = tree.fold("meta", key)
+
+	for key in ["title", "repo", "issueTracker", "forumId", "description", "short_description"]:
+		result[key] = tree.get(key)
+
+	for mod in result["provides"]:
+		result["depends"].discard(mod)
+		result["optional_depends"].discard(mod)
+
+	for key, value in result.items():
+		if isinstance(value, set):
+			result[key] = list(value)
 
 	return result
 
 
-@celery.task()
-def makeVCSRelease(id, branch):
-	release = PackageRelease.query.get(id)
-
-	if release is None:
-		raise TaskError("No such release!")
-
-	if release.package is None:
-		raise TaskError("No package attached to release")
-
-	url = urlparse(release.package.repo)
-
-	urlmaker = None
-	if url.netloc == "github.com":
-		urlmaker = GithubURLMaker(url)
-	else:
-		raise TaskError("Unsupported repo")
-
+def makeVCSReleaseFromGithub(id, branch, release, url):
+	urlmaker = GithubURLMaker(url)
 	if not urlmaker.isValid():
 		raise TaskError("Invalid github repo URL")
 
@@ -220,14 +345,47 @@ def makeVCSRelease(id, branch):
 	if len(commits) == 0 or not "sha" in commits[0]:
 		raise TaskError("No commits found")
 
-	release.url = urlmaker.getCommitDownload(commits[0]["sha"])
+	release.url          = urlmaker.getCommitDownload(commits[0]["sha"])
+	release.task_id     = None
+	release.commit_hash = commits[0]["sha"]
 	print(release.url)
-	release.task_id = None
 	db.session.commit()
 
 	return release.url
 
 
+
+@celery.task()
+def makeVCSRelease(id, branch):
+	release = PackageRelease.query.get(id)
+	if release is None:
+		raise TaskError("No such release!")
+	elif release.package is None:
+		raise TaskError("No package attached to release")
+
+	urlmaker = None
+	url = urlparse(release.package.repo)
+	if url.netloc == "github.com":
+		return makeVCSReleaseFromGithub(id, branch, release, url)
+	else:
+		gitDir, repo = cloneRepo(release.package.repo, ref=branch, recursive=True)
+
+		try:
+			filename = randomString(10) + ".zip"
+			destPath = os.path.join("app/public/uploads", filename)
+			with open(destPath, "wb") as fp:
+				repo.archive(fp, format="zip")
+
+			release.url         = "/uploads/" + filename
+			release.task_id     = None
+			release.commit_hash = repo.head.object.hexsha
+			print(release.url)
+			db.session.commit()
+
+			return release.url
+		finally:
+			shutil.rmtree(gitDir)
+
 @celery.task()
 def importRepoScreenshot(id):
 	package = Package.query.get(id)
@@ -235,32 +393,122 @@ def importRepoScreenshot(id):
 		raise Exception("Unexpected none package")
 
 	# Get URL Maker
+	try:
+		gitDir, _ = cloneRepo(package.repo)
+	except TaskError as e:
+		# ignore download errors
+		print(e)
+		return None
+
+	# Find and import screenshot
+	try:
+		for ext in ["png", "jpg", "jpeg"]:
+			sourcePath = gitDir + "/screenshot." + ext
+			if os.path.isfile(sourcePath):
+				filename = randomString(10) + "." + ext
+				destPath = os.path.join("app/public/uploads", filename)
+				shutil.copyfile(sourcePath, destPath)
+
+				ss = PackageScreenshot()
+				ss.approved = True
+				ss.package = package
+				ss.title   = "screenshot.png"
+				ss.url	 = "/uploads/" + filename
+				db.session.add(ss)
+				db.session.commit()
+
+				return "/uploads/" + filename
+	finally:
+		shutil.rmtree(gitDir)
+
+	print("screenshot.png does not exist")
+	return None
+
+
+
+def getDepends(package):
 	url = urlparse(package.repo)
 	urlmaker = None
 	if url.netloc == "github.com":
 		urlmaker = GithubURLMaker(url)
 	else:
-		raise TaskError("Unsupported repo")
+		return {}
 
+	result = {}
 	if not urlmaker.isValid():
-		raise TaskError("Error! Url maker not valid")
+		return {}
 
+	#
+	# Try getting depends on mod.conf
+	#
 	try:
-		filename = randomString(10) + ".png"
-		imagePath = os.path.join("app/public/uploads", filename)
-		print(imagePath)
-		urllib.request.urlretrieve(urlmaker.getScreenshotURL(), imagePath)
+		contents = urllib.request.urlopen(urlmaker.getModConfURL()).read().decode("utf-8")
+		conf = parseConf(contents)
+		for key in ["depends", "optional_depends"]:
+			try:
+				result[key] = conf[key]
+			except KeyError:
+				pass
 
-		ss = PackageScreenshot()
-		ss.approved = True
-		ss.package = package
-		ss.title   = "screenshot.png"
-		ss.url     = "/uploads/" + filename
-		db.session.add(ss)
-		db.session.commit()
-
-		return "/uploads/" + filename
 	except HTTPError:
-		print("screenshot.png does not exist")
+		print("mod.conf does not exist")
 
-	return None
+	if "depends" in result or "optional_depends" in result:
+		return result
+
+
+	#
+	# Try depends.txt
+	#
+	import re
+	pattern = re.compile("^([a-z0-9_]+)\??$")
+	try:
+		contents = urllib.request.urlopen(urlmaker.getDependsURL()).read().decode("utf-8")
+		soft = []
+		hard = []
+		for line in contents.split("\n"):
+			line = line.strip()
+			if pattern.match(line):
+				if line[len(line) - 1] == "?":
+					soft.append( line[:-1])
+				else:
+					hard.append(line)
+
+		result["depends"] = ",".join(hard)
+		result["optional_depends"] = ",".join(soft)
+	except HTTPError:
+		print("depends.txt does not exist")
+
+	return result
+
+
+def importDependencies(package, mpackage_cache):
+	if Dependency.query.filter_by(depender=package).count() != 0:
+		return
+
+	result = getDepends(package)
+
+	if "depends" in result:
+		deps = Dependency.SpecToList(package, result["depends"], mpackage_cache)
+		print("{} hard: {}".format(len(deps), result["depends"]))
+		for dep in deps:
+			dep.optional = False
+			db.session.add(dep)
+
+	if "optional_depends" in result:
+		deps = Dependency.SpecToList(package, result["optional_depends"], mpackage_cache)
+		print("{} soft: {}".format(len(deps), result["optional_depends"]))
+		for dep in deps:
+			dep.optional = True
+			db.session.add(dep)
+
+@celery.task()
+def importAllDependencies():
+	Dependency.query.delete()
+	mpackage_cache = {}
+	packages = Package.query.filter_by(type=PackageType.MOD).all()
+	for i, p in enumerate(packages):
+		print("============= {} ({}/{}) =============".format(p.name, i, len(packages)))
+		importDependencies(p, mpackage_cache)
+
+	db.session.commit()

app/tasks/phpbbparser.py

@@ -5,6 +5,7 @@
 import urllib, socket
 from bs4 import *
 from urllib.parse import urljoin
+from datetime import datetime
 import urllib.request
 import os.path
 import time, re
@@ -77,3 +78,72 @@ def getProfile(url, username):
 		__extract_properties(profile, soup)
 
 		return profile
+
+
+regex_id = re.compile(r"^.*t=([0-9]+).*$")
+
+def parseForumListPage(id, page, out, extra=None):
+	num_per_page = 30
+	start = page*num_per_page+1
+	print(" - Fetching page {} (topics {}-{})".format(page, start, start+num_per_page))
+
+	url = "https://forum.minetest.net/viewforum.php?f=" + str(id) + "&start=" + str(start)
+	r = urllib.request.urlopen(url).read().decode("utf-8")
+	soup = BeautifulSoup(r, "html.parser")
+
+	for row in soup.find_all("li", class_="row"):
+		classes = row.get("class")
+		if "sticky" in classes or "announce" in classes or "global-announce" in classes:
+			continue
+
+		topic = row.find("dl")
+
+		# Link info
+		link   = topic.find(class_="topictitle")
+		id	   = regex_id.match(link.get("href")).group(1)
+		title  = link.find(text=True)
+
+		# Date
+		left   = topic.find("dt")
+		date   = left.get_text().split("»")[1].strip()
+		date   = datetime.strptime(date, "%a %b %d, %Y %H:%M")
+		author = left.find_all("a")[-1].get_text().strip()
+
+		# Get counts
+		posts  = topic.find(class_="posts").find(text=True)
+		views  = topic.find(class_="views").find(text=True)
+
+		if id in out:
+			print("   - got {} again, title: {}".format(id, title))
+			assert(title == out[id]['title'])
+			return False
+
+		row = {
+			"id"    : id,
+			"title" : title,
+			"author": author,
+			"posts" : posts,
+			"views" : views,
+			"date"  : date
+		}
+
+		if extra is not None:
+			for key, value in extra.items():
+				row[key] = value
+
+		out[id] = row
+
+	return True
+
+def getTopicsFromForum(id, out={}, extra=None):
+	print("Fetching all topics from forum {}".format(id))
+	page = 0
+	while parseForumListPage(id, page, out, extra):
+		page = page + 1
+
+	return out
+
+def dumpTitlesToFile(topics, path):
+	with open(path, "w") as out_file:
+		for topic in topics.values():
+			out_file.write(topic["title"] + "\n")

app/templates/admin/licenses/edit.html

@@ -0,0 +1,25 @@
+{% extends "base.html" %}
+
+{% block title %}
+	{% if license %}
+		Edit {{ license.name }}
+	{% else %}
+		New license
+	{% endif %}
+{% endblock %}
+
+{% block content %}
+	<p>
+		<a href="{{ url_for('license_list_page') }}">Back to list</a> |
+		<a href="{{ url_for('createedit_license_page') }}">New License</a>
+	</p>
+
+	{% from "macros/forms.html" import render_field, render_submit_field %}
+	<form method="POST" action="" enctype="multipart/form-data">
+		{{ form.hidden_tag() }}
+
+		{{ render_field(form.name) }}
+		{{ render_field(form.is_foss) }}
+		{{ render_submit_field(form.submit) }}
+	</form>
+{% endblock %}

app/templates/admin/licenses/list.html

@@ -0,0 +1,16 @@
+{% extends "base.html" %}
+
+{% block title %}
+Licenses
+{% endblock %}
+
+{% block content %}
+	<p>
+		<a href="{{ url_for('createedit_license_page') }}">New Tag</a>
+	</p>
+	<ul>
+		{% for l in licenses %}
+			<li><a href="{{ url_for('createedit_license_page', name=l.name) }}">{{ l.name }}</a> [{{ l.is_foss and "Free" or "Non-free"}}]</li>
+		{% endfor %}
+	</ul>
+{% endblock %}

app/templates/admin/list.html

@@ -7,6 +7,8 @@
 {% block content %}
 	<ul>
 		<li><a href="{{ url_for('user_list_page') }}">User list</a></li>
+		<li><a href="{{ url_for('tag_list_page') }}">Tag Editor</a></li>
+		<li><a href="{{ url_for('license_list_page') }}">License Editor</a></li>
 		<li><a href="{{ url_for('switch_user_page') }}">Sign in as another user</a></li>
 	</ul>
 
@@ -16,8 +18,12 @@
 		<form method="post" action="" class="box-body">
 			<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
 			<select name="action">
-				<option value="importusers" selected>Create users from mod list</option>
-				<option value="importscreenshots">Import screenshots from VCS</option>
+				<option value="importmodlist" selected>Import forum topics</option>
+				<option value="recalcscores">Recalculate package scores</option>
+				<!-- <option value="importscreenshots">Import screenshots from VCS</option> -->
+				<!-- <option value="importdepends">Import dependencies from downloads</option> -->
+				<!-- <option value="modprovides">Set provides to mod name</option> -->
+				<!-- <option value="vcsrelease">Create VCS releases</option> -->
 			</select>
 			<input type="submit" value="Perform" />
 		</form>

app/templates/admin/tags/edit.html

@@ -0,0 +1,27 @@
+{% extends "base.html" %}
+
+{% block title %}
+	{% if tag %}
+		Edit {{ tag.title }}
+	{% else %}
+		New tag
+	{% endif %}
+{% endblock %}
+
+{% block content %}
+	<p>
+		<a href="{{ url_for('tag_list_page') }}">Back to list</a> |
+		<a href="{{ url_for('createedit_tag_page') }}">New Tag</a>
+	</p>
+
+	{% from "macros/forms.html" import render_field, render_submit_field %}
+	<form method="POST" action="" enctype="multipart/form-data">
+		{{ form.hidden_tag() }}
+
+		{{ render_field(form.title) }}
+		{% if tag %}
+			{{ render_field(form.name) }}
+		{% endif %}
+		{{ render_submit_field(form.submit) }}
+	</form>
+{% endblock %}

app/templates/admin/tags/list.html

@@ -0,0 +1,16 @@
+{% extends "base.html" %}
+
+{% block title %}
+Tags
+{% endblock %}
+
+{% block content %}
+	<p>
+		<a href="{{ url_for('createedit_tag_page') }}">New Tag</a>
+	</p>
+	<ul>
+		{% for t in tags %}
+			<li><a href="{{ url_for('createedit_tag_page', name=t.name) }}">{{ t.title }}</a> [{{ t.packages | count }} packages]</li>
+		{% endfor %}
+	</ul>
+{% endblock %}

app/templates/base.html

@@ -63,6 +63,10 @@
 							{% if current_user.rank == current_user.rank.ADMIN %}
 								<li><a href="{{ url_for('admin_page') }}">Admin</a></li>
 							{% endif %}
+							{% if current_user.rank == current_user.rank.MODERATOR %}
+								<li><a href="{{ url_for('tag_list_page') }}">Tag Editor</a></li>
+								<li><a href="{{ url_for('license_list_page') }}">License Editor</a></li>
+							{% endif %}
 							<li><a href="{{ url_for('user.logout') }}">Sign out</a></li>
 						</ul>
 					</li>
@@ -100,8 +104,9 @@
 	{% endblock %}
 
 	<footer>
-		Copyright &copy; 2018 to <a href="https://rubenwardy.com/">rubenwardy</a> |
+		ContentDB &copy; 2018 to <a href="https://rubenwardy.com/">rubenwardy</a> |
+		<a href="https://github.com/minetest/contentdb">GitHub</a> |
 		<a href="{{ url_for('flatpage', path='help') }}">Help</a> |
-		<a href="https://github.com/minetest/contentdb">GitHub</a>
+		<a href="{{ url_for('flatpage', path='help/reporting') }}">Report / DMCA</a>
 	</footer>
 </html>

app/templates/flask_user/login.html

@@ -11,9 +11,7 @@ Sign in
 		<h2>{%trans%}Sign in{%endtrans%}</h2>
 
 		<form action="" method="POST" class="form box-body" role="form">
-			<a href="{{ url_for('github_signin_page') }}">GitHub</a>
-
-
+			<h3>Sign in with username/password</h3>
 			{{ form.hidden_tag() }}
 
 			{# Username or Email field #}
@@ -36,17 +34,13 @@ Sign in
 			{# Password field #}
 			{% set field = form.password %}
 			<div class="form-group {% if field.errors %}has-error{% endif %}">
-				{# Label on left, "Forgot your Password?" on right #}
 				<div class="row">
-					<div class="col-xs-6">
-						<label for="{{ field.id }}" class="control-label">{{ field.label.text }}</label>
-					</div>
-					<div class="col-xs-6 text-right">
-						{% if user_manager.enable_forgot_password %}
+					<label for="{{ field.id }}" class="control-label">{{ field.label.text }}
+					{% if user_manager.enable_forgot_password %}
 						<a href="{{ url_for('user.forgot_password') }}" tabindex='195'>
-							{%trans%}Forgot your Password?{%endtrans%}</a>
-							{% endif %}
-					</div>
+							[{%trans%}Forgot My Password{%endtrans%}]</a>
+					{% endif %}
+					</label>
 				</div>
 				{{ field(class_='form-control', tabindex=120) }}
 				{% if field.errors %}
@@ -62,7 +56,12 @@ Sign in
 			{% endif %}
 
 			{# Submit button #}
-			{{ render_submit_field(form.submit, tabindex=180) }}
+			<p>
+				{{ render_submit_field(form.submit, tabindex=180) }}
+			</p>
+
+			<h3>Sign in with Github</h3>
+			<p><a class="button" href="{{ url_for('github_signin_page') }}">GitHub</a></p>
 		</form>
 	</div>
 
@@ -71,7 +70,7 @@ Sign in
 			<h2>New here?</h2>
 
 			<div class="box-body">
-				<p>Create an account using your forum account.</p>
+				<p>Create an account using your forum account or email.</p>
 
 				<a href="{{ url_for('user_claim_page') }}" class="button">{%trans%}Claim your account{%endtrans%}</a>
 			</div>

app/templates/flatpage.html

@@ -5,7 +5,7 @@
 {% endblock %}
 
 {% block content %}
-	<h1>{{ page['title'] }}</h1>
+	{% if not page["no_h1"] %}<h1>{{ page['title'] }}</h1>{% endif %}
 
 	{{ page.html | safe }}
 {% endblock %}

app/templates/index.html

@@ -12,8 +12,8 @@ Welcome
 		<p>
 			Minetest's official content repository.
 			Browse {{ count }} packages,
-			all available under a free and open source
-			license.
+			the majority of which are available under a free
+			and open source license.
 		</p>
 
 		<form method="get" action="/packages/">
@@ -25,6 +25,14 @@ Welcome
 
 <main>
 	{% from "macros/packagegridtile.html" import render_pkggrid %}
-	{{ render_pkggrid(packages) }}
+
+	<h2>Popular</h2>
+	{{ render_pkggrid(popular) }}
+
+	<a href="{{ url_for('packages_page') }}" class="button">Show More</a>
+
+	<h2 style="margin-top:2em;">Newly Added</h2>
+	{{ render_pkggrid(new) }}
+
 </main>
 {% endblock %}

app/templates/macros/forms.html

@@ -20,13 +20,47 @@
 	<script src="/static/tagselector.js"></script>
 {% endmacro %}
 
+{% macro package_lists() -%}
+	<script>
+		meta_packages = [
+			{% for m in mpackages %}
+				{# This is safe as name can only contain `[a-z0-9_]` #}
+				{
+					id: "{{ m.name }}",
+					value: "{{ m.name }}",
+					toString: function() { return "{{ m.name }}"; },
+				},
+			{% endfor %}
+		]
+
+		function escape(unsafe) {
+			return unsafe
+				.replace(/&/g, "&amp;")
+				.replace(/</g, "&lt;")
+				.replace(/>/g, "&gt;")
+				.replace(/"/g, "&quot;")
+				.replace(/'/g, "&#039;");
+		}
+
+		all_packages = meta_packages.slice();
+
+		{% for p in packages %}
+			all_packages.push({
+				id: "{{ p.author.username }}/{{ p.name }}",
+				value: escape({{ p.title | tojson }} + " by " + {{ p.author.display_name | tojson }}),
+				toString: function() { return  escape({{ p.title | tojson }} + " by " + {{ p.author.display_name | tojson }} + " only"); },
+			});
+		{% endfor %}
+	</script>
+{% endmacro %}
+
 {% macro render_multiselect_field(field, label=None, label_visible=true, right_url=None, right_label=None) -%}
 	<div class="form-group {% if field.errors %}has-error{% endif %} {{ kwargs.pop('class_', '') }}">
 		{% if field.type != 'HiddenField' and label_visible %}
 			{% if not label %}{% set label=field.label.text %}{% endif %}
 			<label for="{{ field.id }}" class="control-label">{{ label|safe }}</label>
 		{% endif %}
-		<div class="multichoice_selector">
+		<div class="multichoice_selector bulletselector">
 			<input type="text" placeholder="Start typing to see suggestions">
 			<div class="clearboth"></div>
 		</div>
@@ -39,6 +73,44 @@
 	</div>
 {% endmacro %}
 
+{% macro render_mpackage_field(field, label=None, label_visible=true, right_url=None, right_label=None) -%}
+	<div class="form-group {% if field.errors %}has-error{% endif %} {{ kwargs.pop('class_', '') }}">
+		{% if field.type != 'HiddenField' and label_visible %}
+			{% if not label %}{% set label=field.label.text %}{% endif %}
+			<label for="{{ field.id }}" class="control-label">{{ label|safe }}</label>
+		{% endif %}
+		<div class="metapackage_selector bulletselector">
+			<input type="text" placeholder="Comma-seperated values">
+			<div class="clearboth"></div>
+		</div>
+		{{ field(class_='form-control', **kwargs) }}
+		{% if field.errors %}
+			{% for e in field.errors %}
+				<p class="help-block">{{ e }}</p>
+			{% endfor %}
+		{% endif %}
+	</div>
+{% endmacro %}
+
+{% macro render_deps_field(field, label=None, label_visible=true, right_url=None, right_label=None) -%}
+	<div class="form-group {% if field.errors %}has-error{% endif %} {{ kwargs.pop('class_', '') }}">
+		{% if field.type != 'HiddenField' and label_visible %}
+			{% if not label %}{% set label=field.label.text %}{% endif %}
+			<label for="{{ field.id }}" class="control-label">{{ label|safe }}</label>
+		{% endif %}
+		<div class="deps_selector bulletselector">
+			<input type="text" placeholder="Comma-seperated values">
+			<div class="clearboth"></div>
+		</div>
+		{{ field(class_='form-control', **kwargs) }}
+		{% if field.errors %}
+			{% for e in field.errors %}
+				<p class="help-block">{{ e }}</p>
+			{% endfor %}
+		{% endif %}
+	</div>
+{% endmacro %}
+
 {% macro render_checkbox_field(field, label=None) -%}
 	{% if not label %}{% set label=field.label.text %}{% endif %}
 	<div class="checkbox">

app/templates/macros/packagegridtile.html

@@ -1,21 +1,42 @@
-{% macro render_pkgtile(package) -%}
+{% macro render_pkgtile(package, show_author) -%}
 	<li><a href="{{ package.getDetailsURL() }}"
-		style="background-image: url({{ package.getMainScreenshotURL() or '/static/placeholder.png' }});">
+		style="background-image: url({{ package.getThumbnailURL() or '/static/placeholder.png' }});">
 		<div class="packagegridscrub"></div>
 		<div class="packagegridinfo">
-			<h3>{{ package.title }} by {{ package.author.display_name }}</h3>
+			<h3>
+				{{ package.title }}
+
+				{% if show_author %}
+					by {{ package.author.display_name }}
+				{% endif %}
+			</h3>
 
 			<p>
 				{{ package.shortDesc }}
 			</p>
+
+
+			{% if not package.license.is_foss and not package.media_license.is_foss and package.type != package.type.TXP  %}
+				<p style="color:#f33;">
+					<b>Warning:</b> Non-free code and media.
+				</p>
+			{% elif not package.license.is_foss and package.type != package.type.TXP %}
+				<p style="color:#f33;">
+					<b>Warning:</b> Non-free code.
+				</p>
+			{% elif not package.media_license.is_foss %}
+				<p style="color:#f33;">
+					<b>Warning:</b> Non-free media.
+				</p>
+			{% endif %}
 		</div>
 	</a></li>
 {% endmacro %}
 
-{% macro render_pkggrid(packages) -%}
+{% macro render_pkggrid(packages, show_author=True) -%}
 	<ul class="packagegrid">
 		{% for p in packages %}
-			{{ render_pkgtile(p) }}
+			{{ render_pkgtile(p, show_author) }}
 		{% else %}
 			<li><i>No packages available</i></ul>
 		{% endfor %}

app/templates/macros/threads.html

@@ -0,0 +1,36 @@
+{% macro render_thread(thread, current_user) -%}
+	<ul class="comments">
+		{% for r in thread.replies %}
+			<li>
+				<div class="info_strip">
+					<a class="author {{ r.author.rank.name }}"
+							href="{{ url_for('user_profile_page', username=r.author.username) }}">
+						{{ r.author.display_name }}</a>
+					<span>{{ r.created_at | datetime }}</span>
+					<div class="clearboth"></div>
+				</div>
+				<div class="msg">
+					{{ r.comment | markdown }}
+				</div>
+			</li>
+		{% endfor %}
+	</ul>
+
+	{% if current_user.is_authenticated %}
+		<form method="post" action="{{ url_for('thread_page', id=thread.id)}}" class="comment_form">
+			<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
+			<textarea required maxlength=500 name="comment" placeholder="Markdown supported"></textarea><br />
+			<input type="submit" value="Comment" />
+		</form>
+	{% endif %}
+{% endmacro %}
+
+{% macro render_threadlist(threads) -%}
+	<ul>
+		{% for t in threads %}
+			<li>{% if t.private %}&#x1f512; {% endif %}<a href="{{ url_for('thread_page', id=t.id) }}">{{ t.title }}</a> by {{ t.author.display_name }}</li>
+		{% else %}
+			<li><i>No threads found</i></li>
+		{% endfor %}
+	</ul>
+{% endmacro %}

app/templates/macros/topics.html

@@ -0,0 +1,52 @@
+{% macro render_topics_table(topics, show_author=True) -%}
+<table>
+	<tr>
+		<th>Id</th>
+		<th></th>
+		<th>Title</th>
+		{% if show_author %}<th>Author</th>{% endif %}
+		<th>Name</th>
+		<th>Link</th>
+		<th>Actions</th>
+	</tr>
+	{% for topic in topics %}
+		<tr{% if topic.wip %} class="wiptopic"{% endif %}>
+			<td>{{ topic.topic_id }}</td>
+			<td>
+				[{{ topic.type.value }}]
+			</td>
+			<td>
+				<a href="https://forum.minetest.net/viewtopic.php?t={{ topic.topic_id}}">{{ topic.title }}</a>
+				{% if topic.wip %}[WIP]{% endif %}
+			</td>
+			{% if show_author %}
+				<td><a href="{{ url_for('user_profile_page', username=topic.author.username) }}">{{ topic.author.display_name}}</a></td>
+			{% endif %}
+			<td>{{ topic.name or ""}}</td>
+			<td>{% if topic.link %}<a href="{{ topic.link }}">{{ topic.link | domain }}</a>{% endif %}</td>
+			<td>
+				<a href="{{ url_for('create_edit_package_page', author=topic.author.username, repo=topic.getRepoURL(), forums=topic.topic_id, title=topic.title, bname=topic.name) }}">Create</a>
+			</td>
+		</tr>
+	{% endfor %}
+</table>
+{% endmacro %}
+
+
+{% macro render_topics(topics, current_user, show_author=True) -%}
+<ul>
+	{% for topic in topics %}
+		<li{% if topic.wip %} class="wiptopic"{% endif %}>
+			<a href="https://forum.minetest.net/viewtopic.php?t={{ topic.topic_id}}">{{ topic.title }}</a>
+			{% if topic.wip %}[WIP]{% endif %}
+			{% if topic.name %}[{{ topic.name }}]{% endif %}
+			{% if show_author %}
+				by <a href="{{ url_for('user_profile_page', username=topic.author.username) }}">{{ topic.author.display_name }}</a>
+			{% endif %}
+			{% if topic.author == current_user or topic.author.checkPerm(current_user, "CHANGE_AUTHOR") %}
+				| <a href="{{ url_for('create_edit_package_page', author=topic.author.username, repo=topic.getRepoURL(), forums=topic.topic_id, title=topic.title, bname=topic.name) }}">Create</a>
+			{% endif %}
+		</li>
+	{% endfor %}
+</ul>
+{% endmacro %}

app/templates/meta/list.html

@@ -0,0 +1,15 @@
+{% extends "base.html" %}
+
+{% block title %}
+Meta Packages
+{% endblock %}
+
+{% block content %}
+	<ul>
+		{% for meta in mpackages %}
+			<li><a href="{{ url_for('meta_package_page', name=meta.name) }}">{{ meta.name }}</a> ({{ meta.packages.filter_by(soft_deleted=False, approved=True).all() | count }} packages)</li>
+		{% else %}
+			<li><i>No meta packages found.</i></li>
+		{% endfor %}
+	</ul>
+{% endblock %}

app/templates/meta/view.html

@@ -0,0 +1,12 @@
+{% extends "base.html" %}
+
+{% block title %}
+Packages providing '{{ mpackage.name }}''
+{% endblock %}
+
+{% block content %}
+	<h1>Packages providing '{{ mpackage.name }}'</h1>
+
+	{% from "macros/packagegridtile.html" import render_pkggrid %}
+	{{ render_pkggrid(mpackage.packages.filter_by(approved=True, soft_deleted=False).all()) }}
+{% endblock %}

app/templates/notifications/list.html

@@ -5,6 +5,12 @@ Notifications
 {% endblock %}
 
 {% block content %}
+	{% if current_user.notifications %}
+		<form method="post" action="{{ url_for('clear_notifications_page') }}">
+			<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
+			<input type="submit" value="Clear All" />
+		</form>
+	{% endif %}
 	<ul>
 		{% for n in current_user.notifications %}
 			<li><a href="{{ n.url }}">

app/templates/packages/create_edit.html

@@ -8,39 +8,61 @@
 {% endblock %}
 
 {% block content %}
-	<h2>Create Package</h2>
+	<h1>Create Package</h1>
 
-	{% from "macros/forms.html" import render_field, render_submit_field, form_includes, render_multiselect_field %}
+	<div class="box box_grey alert alert-info">
+		Have you read the Package Inclusion Policy and Guidance yet?
+
+		<a class="alert_right button" href="{{ url_for('flatpage', path='policy_and_guidance') }}">View</a>
+	</div>
+
+
+	{% from "macros/forms.html" import render_field, render_submit_field, form_includes, render_multiselect_field, render_mpackage_field, render_deps_field, package_lists %}
 	{{ form_includes() }}
+	{{ package_lists() }}
 
 	<form method="POST" action="" class="tableform">
 		{{ form.hidden_tag() }}
 
+		<h2 class="pkg_meta">Package</h2>
+
+		{{ render_field(form.type, class_="pkg_meta") }}
 		{{ render_field(form.name, class_="pkg_meta") }}
 		{{ render_field(form.title, class_="pkg_meta") }}
 		{{ render_field(form.shortDesc, class_="pkg_meta") }}
 		{{ render_field(form.desc, class_="pkg_meta") }}
-		{{ render_field(form.type, class_="pkg_meta") }}
-		{{ render_field(form.license, class_="pkg_meta") }}
 		{{ render_multiselect_field(form.tags, class_="pkg_meta") }}
-		{{ render_multiselect_field(form.harddeps, class_="pkg_meta") }}
-		{{ render_multiselect_field(form.softdeps, class_="pkg_meta") }}
+		<div class="pkg_meta">
+			{{ render_field(form.license, class_="not_txp") }}
+		</div>
+		{{ render_field(form.media_license, class_="pkg_meta") }}
+
+		<div class="pkg_meta">
+			<h2 class="not_txp">Dependency Info</h2>
+
+			{{ render_mpackage_field(form.provides_str, class_="not_txp", placeholder="Comma separated list") }}
+			{{ render_deps_field(form.harddep_str, class_="not_txp not_game", placeholder="Comma separated list") }}
+			{{ render_deps_field(form.softdep_str, class_="not_txp not_game", placeholder="Comma separated list") }}
+		</div>
+
+		<h2 class="pkg_meta">Repository and Links</h2>
 
 		<div class="pkg_wiz_1">
 			<p>Enter the repo URL for the package.
-			If it's hosted on Github then metadata will automatically be imported.</p>
+			If the repo uses git then the metadata will be automatically imported.</p>
 
-			<p>Leave blank if you don't have a repo.</p>
+			<p>Leave blank if you don't have a repo. Click skip if the import fails.</p>
 		</div>
 
 		{{ render_field(form.repo, class_="pkg_repo") }}
 
 		<div class="pkg_wiz_1">
-			<a id="pkg_wiz_1_next" class="button button-primary">Next</a>
+			<a id="pkg_wiz_1_next" class="button button-primary">Next (Autoimport)</a>
+			<a id="pkg_wiz_1_skip" class="button button-default">Skip Autoimport</a>
 		</div>
 
 		<div class="pkg_wiz_2">
-			Importing...
+			Importing... (This may take a while)
 		</div>
 
 		{{ render_field(form.website, class_="pkg_meta") }}
@@ -49,6 +71,12 @@
 		<div class="pkg_meta">{{ render_submit_field(form.submit) }}</div>
 	</form>
 
+	<script src="/static/simplemde.min.js"></script>
+	<link rel="stylesheet" type="text/css" href="/static/simplemde.min.css">
+	<script>
+		var simplemde = new SimpleMDE({ element: $("#desc")[0] });
+	</script>
+
 	{% if enable_wizard %}
 		<script src="/static/url.min.js"></script>
 		<script src="/static/polltask.js"></script>
@@ -60,4 +88,5 @@
 			</div>
 		</noscript>
 	{% endif %}
+	<script src="/static/package_edit.js"></script>
 {% endblock %}

app/templates/packages/editrequest_create_edit.html

@@ -5,21 +5,30 @@
 {% endblock %}
 
 {% block content %}
-	{% from "macros/forms.html" import render_field, render_submit_field, form_includes, render_multiselect_field %}
+	{% from "macros/forms.html" import render_field, render_submit_field, form_includes, render_multiselect_field, render_mpackage_field, render_deps_field, package_lists %}
 	{{ form_includes() }}
+	{{ package_lists() }}
 
 	<form method="POST" action="">
 		{{ form.hidden_tag() }}
 
+		<h2 class="pkg_meta">Package</h2>
+
+		{{ render_field(form.type) }}
 		{{ render_field(form.name) }}
 		{{ render_field(form.title) }}
 		{{ render_field(form.shortDesc) }}
 		{{ render_field(form.desc) }}
-		{{ render_field(form.type) }}
-		{{ render_field(form.license) }}
 		{{ render_multiselect_field(form.tags) }}
-		{{ render_multiselect_field(form.harddeps) }}
-		{{ render_multiselect_field(form.softdeps) }}
+
+		<h2 class="not_txp">Dependency Info</h2>
+
+		{{ render_mpackage_field(form.provides_str, class_="not_txp", placeholder="Comma separated list") }}
+		{{ render_deps_field(form.harddep_str, class_="not_txp not_game", placeholder="Comma separated list") }}
+		{{ render_deps_field(form.softdep_str, class_="not_txp not_game", placeholder="Comma separated list") }}
+
+		{{ render_field(form.license) }}
+		{{ render_field(form.media_license) }}
 		{{ render_field(form.repo) }}
 		{{ render_field(form.website) }}
 		{{ render_field(form.issueTracker) }}
@@ -31,4 +40,6 @@
 		{{ render_field(form.edit_desc) }}
 		{{ render_submit_field(form.submit) }}
 	</form>
+
+	<script src="/static/package_edit.js"></script>
 {% endblock %}

app/templates/packages/list.html

@@ -9,9 +9,10 @@
 		{% if type %}<input type="hidden" name="type" value="{{ type }}" />{% endif %}
 		<input type="text" name="q" value="{{ query or ''}}" />
 		<input type="submit" value="Search" />
+		<input type="submit" name="lucky" value="I'm feeling lucky" />
 
 		<p>
-			Found {{ packages | count }} packages.
+			Found {{ packages_count }} packages.
 		</p>
 	</form>
 
@@ -31,4 +32,18 @@
 
 	{% from "macros/packagegridtile.html" import render_pkggrid %}
 	{{ render_pkggrid(packages) }}
+
+	<ul class="buttonset linedbuttonset">
+		{% if prev_url %}<li><a href="{{ prev_url }}">Previous</a></li>{% endif %}
+		<li>{{ page }} / {{ page_max }}</li>
+		{% if next_url %}<li><a href="{{ next_url }}">Next</a></li> {% endif %}
+	</ul>
+
+	{% if topics %}
+		<h2 style="margin-top:2em;">More content from the forums</h2>
+
+		{% from "macros/topics.html" import render_topics %}
+		{{ render_topics(topics, current_user) }}
+	{% endif %}
+
 {% endblock %}

app/templates/packages/release_edit.html

@@ -21,8 +21,12 @@
 			Url: <a href="{{ release.url }}">{{ release.url }}</a><br />
 		{% endif %}
 
+		{% if release.commit_hash %}
+			Commit Hash: {{ release.commit_hash }}<br />
+		{% endif %}
+
 		{% if release.task_id %}
-			Importing... <a href="{ url_for('check_task', id=release.task_id, r=release.getEditURL()) }}">view task</a><br />
+			Importing... <a href="{{ url_for('check_task', id=release.task_id, r=release.getEditURL()) }}">view task</a><br />
 			{% if package.checkPerm(current_user, "CHANGE_RELEASE_URL") %}
 				{{ render_field(form.task_id) }}
 			{% endif %}
@@ -38,23 +42,4 @@
 
 		{{ render_submit_field(form.submit) }}
 	</form>
-
-	{% if package.checkPerm(current_user, "APPROVE_RELEASE") %}
-		<div class="box box_grey">
-			<h2>Approval Checklist</h2>
-			<ul>
-				<li>Link leads to a valid download, ie: is a zip file which
-					has either init.lua or modpack.txt if a mod, mods/ if a game, or textures if a texture pack.
-					It's okay if they're inside an immediate folder, like so:
-
-<pre>
-example.zip/
-└── example
-    └── init.lua
-</pre>
-				</li>
-				<li>There's no obfuscated code.</li>
-			</ul>
-		</div>
-	{% endif %}
 {% endblock %}

app/templates/packages/release_new.html

@@ -9,9 +9,9 @@
 	<form method="POST" action="" enctype="multipart/form-data">
 		{{ form.hidden_tag() }}
 
-		{{ render_field(form.title) }}
+		{{ render_field(form.title, placeholder="Human readable. Eg: 1.0.0 or 2018-05-28") }}
 		{{ render_field(form.uploadOpt) }}
-		{% if package.canMakeReleaseFromVCS() %}
+		{% if package.repo %}
 			{{ render_field(form.vcsLabel) }}
 		{% endif %}
 		{{ render_field(form.fileUpload) }}

app/templates/packages/screenshot_edit.html

@@ -5,7 +5,9 @@
 {% endblock %}
 
 {% block content %}
-	<img src="{{ screenshot.getThumbnailURL() }}" alt="{{ screenshot.title }}" />
+	<a href="{{ screenshot.url }}">
+		<img src="{{ screenshot.getThumbnailURL() }}" alt="{{ screenshot.title }}" />
+	</a>
 
 	{% from "macros/forms.html" import render_field, render_submit_field %}
 	<form method="POST" action="" enctype="multipart/form-data">

app/templates/packages/view.html

@@ -20,6 +20,16 @@
 				{% else %}
 					A release is required before this package can be approved.
 				{% endif %}
+
+			{% elif (package.type == package.type.GAME or package.type == package.type.TXP) and package.screenshots.count() == 0 %}
+				You need to add at least one screenshot.
+
+			{% elif topic_error_lvl == "error" %}
+				Please fix the below topic issue(s).
+
+			{% elif "Other" in package.license.name or "Other" in package.media_license.name %}
+				Please wait for the license to be added to CDB.
+
 			{% else %}
 				{% if package.screenshots.count() == 0 %}
 					<b>You should add at least one screenshot, but this isn't required.</b><br />
@@ -39,6 +49,35 @@
 			{% endif %}
 			<div style="clear: both;"></div>
 		</div>
+
+		{% if topic_error %}
+			<div class="box box_grey alert alert-{{ topic_error_lvl }}">
+				<span class="icon_message"></span>
+				{{ topic_error | safe }}
+				<div style="clear: both;"></div>
+			</div>
+		{% endif %}
+
+		{% if package.author == current_user or package.checkPerm(current_user, "APPROVE_NEW") %}
+			{% if review_thread %}
+				<h2>{% if review_thread.private %}&#x1f512;{% endif %} {{ review_thread.title }}</h2>
+				{% if review_thread.private %}
+					<p><i>
+						This thread is only visible to the package owner and users of
+						Editor rank or above.
+					</i></p>
+				{% endif %}
+
+				{% from "macros/threads.html" import render_thread %}
+				{{ render_thread(review_thread, current_user) }}
+			{% else %}
+				<div class="box box_grey alert alert-info">
+					Privately ask a question or give feedback
+
+					<a class="alert_right button" href="{{ url_for('new_thread_page', pid=package.id, title='Package approval comments') }}">Open Thread</a>
+				</div>
+			{% endif %}
+		{% endif %}
 	{% endif %}
 
 	<h1>{{ package.title }} by {{ package.author.display_name }}</h1>
@@ -57,12 +96,37 @@
 
 	<aside class="asideright box box_grey">
 		<h3>Details</h3>
+
 		<div class="box-body">
+			{% if not package.license.is_foss and not package.media_license.is_foss and package.type != package.type.TXP  %}
+				<div class="box box_grey alert alert-error" style="margin-top: 0;">
+					<b>Warning:</b> Non-free code and media.
+				</div>
+			{% elif not package.license.is_foss and package.type != package.type.TXP %}
+				<div class="box box_grey alert alert-error" style="margin-top: 0;">
+					<b>Warning:</b> Non-free code.
+				</div>
+			{% elif not package.media_license.is_foss %}
+				<div class="box box_grey alert alert-error" style="margin-top: 0;">
+					<b>Warning:</b> Non-free media.
+				</div>
+			{% endif %}
 			<table>
 				<tr>
 					<td>Name</td>
 					<td>{{ package.name }}</td>
 				</tr>
+				{% if package.provides %}
+				<tr>
+					<td>Provides</td>
+					<td>{% for meta in package.provides %}
+						<a href="{{ url_for('meta_package_page', name=meta.name) }}">{{ meta.name }}</a>
+						{%- if not loop.last %}
+						,
+						{% endif %}
+					{% endfor %}</td>
+				</tr>
+				{% endif %}
 				<tr>
 					<td>Author</td>
 					<td class="{{ package.author.rank }}">
@@ -77,12 +141,25 @@
 				</tr>
 				<tr>
 					<td>License</td>
-					<td>{{ package.license.name }}</td>
+					<td>
+						{% if package.license == package.media_license %}
+							{{ package.license.name }}
+						{% elif package.type == package.type.TXP %}
+							{{ package.media_license.name }}
+						{% else %}
+							{{ package.license.name }} for code,<br />
+							{{ package.media_license.name }} for media.
+						{% endif %}
+					</td>
+				</tr>
+				<tr>
+					<td>Added</td>
+					<td>{{ package.created_at | datetime }}</td>
 				</tr>
 			</table>
 
 			<ul class="buttonset linedbuttonset">
-				{% if package.getDownloadRelease() %}<li><a href="{{ package.getDownloadURL() }}">Download</a></li>{% endif %}
+				{% if package.getDownloadRelease() %}<li><a href="{{ package.getDownloadURL() }}" class="btn_green">Download</a></li>{% endif %}
 				{% if package.repo %}<li><a href="{{ package.repo }}">View Source</a></li>{% endif %}
 				{% if package.forums %}<li><a href="https://forum.minetest.net/viewtopic.php?t={{ package.forums }}">Forums</a></li>{% endif %}
 				{% if package.issueTracker %}<li><a href="{{ package.issueTracker }}">Issue Tracker</a></li>{% endif %}
@@ -91,12 +168,15 @@
 					<li><a href="{{ package.getEditURL() }}">Edit</a></li>
 					<li><a href="{{ package.getNewScreenshotURL() }}">Add screenshot</a></li>
 				{% endif %}
-				{% if current_user.is_authenticated %}
+				{# {% if current_user.is_authenticated %}
 					<li><a href="{{ package.getCreateEditRequestURL() }}">Suggest Changes</a></li>
-				{% endif %}
+				{% endif %} #}
 				{% if package.checkPerm(current_user, "MAKE_RELEASE") %}
 					<li><a href="{{ package.getCreateReleaseURL() }}">Create Release</a></li>
 				{% endif %}
+				{% if package.approved and package.checkPerm(current_user, "CREATE_THREAD") %}
+					<li><a href="{{ url_for('new_thread_page', pid=package.id) }}">Open Thread</a></li>
+				{% endif %}
 				{% if package.checkPerm(current_user, "DELETE_PACKAGE") %}
 					<li><a href="{{ package.getDeleteURL() }}">Delete</a></li>
 				{% endif %}
@@ -116,8 +196,9 @@
 				<li>
 					{% if not rel.approved %}<i>{% endif %}
 
-					<a href="{{ rel.url }}">{{ rel.title }}</a>,
-					created {{ rel.releaseDate }}.
+					<a href="{{ rel.getDownloadURL() }}">{{ rel.title }}</a>{% if rel.commit_hash %}
+					[{{ rel.commit_hash | truncate(5, end='') }}]{% endif %},
+					created {{ rel.releaseDate | datetime }}.
 					{% if rel.task_id %}
 						<a href="{{ url_for('check_task', id=rel.task_id, r=package.getDetailsURL()) }}">Importing...</a>
 					{% elif not rel.approved %}
@@ -149,23 +230,29 @@
 		{% endfor %}
 	</ul>
 
-	<table class="table-topalign">
+	<!-- <table class="table-topalign">
 		<tr>
-			<td>
+			<td> -->
 				<h3>Dependencies</h3>
 				<ul>
-					{% for p in package.harddeps %}
-						<li><a href="{{ p.getDetailsURL() }}">{{ p.title }}</a> by {{ p.author.display_name }}</li>
+					{% for dep in package.dependencies %}
+						<li>
+							{%- if dep.package %}
+								<a href="{{ dep.package.getDetailsURL() }}">{{ dep.package.title }}</a> by {{ dep.package.author.display_name }}
+							{% elif dep.meta_package %}
+								<a href="{{ url_for('meta_package_page', name=dep.meta_package.name) }}">{{ dep.meta_package.name }}</a>
+							{% else %}
+								{{ "Excepted package or meta_package in dep!" | throw }}
+							{% endif %}
+							{% if dep.optional %}
+								[optional]
+							{% endif %}
+						</li>
 					{% else %}
-						{% if not package.softdeps %}
-						<li>No dependencies.</li>
-						{% endif %}
-					{% endfor %}
-					{% for p in package.softdeps %}
-						<li><a href="{{ p.getDetailsURL() }}">{{ p.title }}</a> by {{ p.author.display_name }} [optional]</li>
+						<li><i>No dependencies</i></li>
 					{% endfor %}
 				</ul>
-			</td>
+			<!-- </td>
 			<td>
 				<h3>Required by</h3>
 				<ul>
@@ -182,21 +269,67 @@
 				</ul>
 			</td>
 		</tr>
-	</table>
+	</table> -->
 
-	{% if current_user.is_authenticated or requests %}
-		<h3>Edit Requests</h3>
+	{#
+		{% if current_user.is_authenticated or requests %}
+			<h3>Edit Requests</h3>
 
+			<ul>
+				{% for r in requests %}
+					<li>
+						<a href="{{ r.getURL() }}">{{ r.title }}</a>
+						by
+						<a href="{{ url_for('user_profile_page', username=r.author.username) }}">{{ r.author.display_name }}</a>
+					</li>
+				{% else %}
+					<li>No edit requests have been made.</li>
+				{% endfor %}
+			</ul>
+		{% endif %}
+	#}
+
+	{% if alternatives %}
+		<h3>Alternatives</h3>
 		<ul>
-			{% for r in requests %}
-				<li>
-					<a href="{{ r.getURL() }}">{{ r.title }}</a>
-					by
-					<a href="{{ url_for('user_profile_page', username=r.author.username) }}">{{ r.author.display_name }}</a>
-				</li>
-			{% else %}
-				<li>No edit requests have been made.</li>
+			{% for p in alternatives %}
+				<li><a href="{{ p.getDetailsURL() }}">{{ p.title }} by {{ p.author.display_name }}</a></li>
 			{% endfor %}
 		</ul>
 	{% endif %}
+
+	{% if similar_topics %}
+		<h3>Similar Forum Topics</h3>
+		{% if not package.approved and package.type == package.type.MOD %}
+			<div class="box box_grey alert alert-warning">
+				Please make sure that this package has the right to
+				the name '{{ package.name }}'.
+				See the
+				<a href="/policy_and_guidance/">Inclusion Policy</a>
+				for more info.
+			</div>
+		{% endif %}
+		<ul>
+			{% for t in similar_topics %}
+				<li>
+					[{{ t.type.value }}]
+					<a href="https://forum.minetest.net/viewtopic.php?t={{ t.topic_id }}">
+						{{ t.title }} by {{ t.author.display_name }}
+					</a>
+					{% if t.wip %}[WIP]{% endif %}
+				</li>
+			{% endfor %}
+		</ul>
+	{% endif %}
+
+	{% if threads %}
+		<h3>Threads</h3>
+
+		{% if package.approved and package.checkPerm(current_user, "CREATE_THREAD") %}
+			<p><a href="{{ url_for('new_thread_page', pid=package.id) }}">Open Thread</a></p>
+		{% endif %}
+
+		{% from "macros/threads.html" import render_threadlist %}
+		{{ render_threadlist(threads) }}
+	{% endif %}
 {% endblock %}

app/templates/threads/list.html

@@ -0,0 +1,12 @@
+{% extends "base.html" %}
+
+{% block title %}
+Threads
+{% endblock %}
+
+{% block content %}
+	<h1>Threads</h1>
+
+	{% from "macros/threads.html" import render_threadlist %}
+	{{ render_threadlist(threads) }}
+{% endblock %}

app/templates/threads/new.html

@@ -0,0 +1,19 @@
+{% extends "base.html" %}
+
+{% block title %}
+	New Thread
+{% endblock %}
+
+{% block content %}
+	{% from "macros/forms.html" import render_field, render_submit_field %}
+	<form method="POST" action="" enctype="multipart/form-data">
+		{{ form.hidden_tag() }}
+
+		{{ render_field(form.title) }}
+		{{ render_field(form.comment) }}
+		{{ render_field(form.private) }}
+		{{ render_submit_field(form.submit) }}
+
+		<p>Only the you, the package author, and users of Editor rank and above can read private threads.</p>
+	</form>
+{% endblock %}

app/templates/threads/view.html

@@ -0,0 +1,38 @@
+{% extends "base.html" %}
+
+{% block title %}
+Threads
+{% endblock %}
+
+{% block content %}
+	<h1>{% if thread.private %}&#x1f512; {% endif %}{{ thread.title }}</h1>
+	{% if thread.package or current_user.is_authenticated %}
+		{% if thread.package %}
+			<p>Package: <a href="{{ thread.package.getDetailsURL() }}">{{ thread.package.title }}</a></p>
+		{% endif %}
+
+		{% if current_user.is_authenticated %}
+			{% if current_user in thread.watchers %}
+				<form method="post" action="{{ thread.getUnsubscribeURL() }}">
+					<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
+					<input type="submit" value="Unsubscribe" />
+				</form>
+			{% else %}
+				<form method="post" action="{{ thread.getSubscribeURL() }}">
+					<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
+					<input type="submit" value="Subscribe" />
+				</form>
+			{% endif %}
+		{% endif %}
+	{% endif %}
+
+	{% if thread.private %}
+		<i>
+			This thread is only visible to its creator, the package owner, and users of
+			Editor rank or above.
+		</i>
+	{% endif %}
+
+	{% from "macros/threads.html" import render_thread %}
+	{{ render_thread(thread, current_user) }}
+{% endblock %}

app/templates/todo/list.html

@@ -5,8 +5,10 @@
 {% endblock %}
 
 {% block content %}
-	{% if canApproveNew %}
-		<h2>Packages Awaiting Approval</h2>
+	<h2>Awaiting Approval</h2>
+
+	{% if canApproveNew and packages %}
+		<h3>Packages</h3>
 		<ul>
 			{% for p in packages %}
 				<li><a href="{{ p.getDetailsURL() }}">
@@ -18,8 +20,8 @@
 		</ul>
 	{% endif %}
 
-	{% if canApproveScn %}
-		<h2>Screenshots Awaiting Approval</h2>
+	{% if canApproveScn and screenshots %}
+		<h3>Screenshots</h3>
 		<ul>
 			{% for s in screenshots %}
 				<li>
@@ -35,8 +37,8 @@
 		</ul>
 	{% endif %}
 
-	{% if canApproveRel %}
-		<h2>Releases Awaiting Approval</h2>
+	{% if canApproveRel and releases %}
+		<h3>Releases</h3>
 		<ul>
 			{% for r in releases %}
 				<li>
@@ -51,4 +53,18 @@
 			{% endfor %}
 		</ul>
 	{% endif %}
+
+	{% if not (packages or screenshots or releases) %}
+		<p>
+			<i>All done!</i>
+		</p>
+	{% endif %}
+
+	<h2>Unadded Topic List</h2>
+
+	<p>
+		There are
+		<a href="{{ url_for('todo_topics_page') }}">{{ topics_to_add }} packages</a>
+		to be added to cdb, based on cdb's forum parser.
+	</p>
 {% endblock %}

app/templates/todo/topics.html

@@ -0,0 +1,17 @@
+{% extends "base.html" %}
+
+{% block title %}
+Topics to be Added
+{% endblock %}
+
+{% block content %}
+	<h1>Topics to be Added</h1>
+
+	<p>
+		{{ total - (topics | count) }} / {{ total }} packages have been added.
+		{{ topics | count }} remaining.
+	</p>
+
+	{% from "macros/topics.html" import render_topics_table %}
+	{{ render_topics_table(topics) }}
+{% endblock %}

app/templates/users/claim.html

@@ -1,44 +1,45 @@
 {% extends "base.html" %}
 
 {% block title %}
-Verify forum account
+Creating an Account
 {% endblock %}
 
 {% block content %}
 	<div class="box box_grey">
 		<h2>{{ self.title() }}</h2>
 
-		<p>
-			Create an account by linking it to your forum account and optionally
-			your github account.
-		</p>
-
-		{% if current_user.is_authenticated %}
+		<div class="box-body">
 			<p>
-				Please log out to continue.
-			</p>
-			<p>
-				<a href="{{ url_for('user.logout', next=url_for('user_claim_page')) }}" class="button">Logout</a>
-			</p>
-		{% else %}
-			<p>
-				<b>Don't have a forum account?</b>
-				Unfortunately, you need a forum account to register.
-				This is because you also need to create forum topics for any packages
-				you may upload.
+				If you have a forum account, you'll need to prove that you own it
+				to get an account on ContentDB.
 			</p>
 
-			<a href="https://forum.minetest.net/ucp.php?mode=register">
-				Create a Forum Account
-			</a>
-		{% endif %}
+			{% if current_user.is_authenticated %}
+				<p>
+					Please log out to continue.
+				</p>
+				<p>
+					<a href="{{ url_for('user.logout', next=url_for('user_claim_page')) }}" class="button">Logout</a>
+				</p>
+			{% else %}
+				<p>
+					<b>Don't have a forum account?</b>
+					You don't need one, however it's recommended to make the most
+					out of the Minetest community.
+				</p>
+
+				<a href="https://forum.minetest.net/ucp.php?mode=register">
+					Create a Forum Account
+				</a>
+			{% endif %}
+		</div>
 	</div>
 
 	{% if not current_user.is_authenticated %}
 		<div class="box box_grey">
 			<h2>Option 1 - Use GitHub field in forum profile</h2>
 
-			<form method="post" action="{{ url_for('user_claim_page') }}">
+			<form method="post" class="box-body" action="{{ url_for('user_claim_page') }}">
 				<input type="hidden" name="claim_type" value="github">
 				<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
 
@@ -59,10 +60,10 @@ Verify forum account
 			</form>
 		</div>
 
-		<!--<div class="box box_grey">
+		<div class="box box_grey">
 			<h2>Option 2 - Paste verification token into signature</h2>
 
-			<form method="post" action="{{ url_for('user_claim_page') }}">
+			<form method="post" class="box-body" action="{{ url_for('user_claim_page') }}">
 				<input type="hidden" name="claim_type" value="forum">
 				<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
 
@@ -93,6 +94,22 @@ Verify forum account
 
 				<input type="submit" value="Next">
 			</form>
-		</div>-->
+		</div>
+
+		<div class="box box_grey">
+			<h2>Option 3 - Email/password sign up</h2>
+
+			<div class="box-body">
+				<p>
+					<b>Only do this if you don't have a forum account!</b>
+				</p>
+				<p>
+					If you have a forum account, please use one of the other two
+					options.
+				</p>
+
+				<a class="button" href="{{ url_for('user.register') }}">Register</a>
+			</div>
+		</div>
 	{% endif %}
 {% endblock %}

app/templates/users/set_password.html

@@ -0,0 +1,40 @@
+{% extends "base.html" %}
+
+{% block title %}
+	Set Password
+{% endblock %}
+
+{% block content %}
+
+{% if optional %}
+	<div class="box box_grey alert alert-primary">
+		It is recommended that you set a password for your account.
+
+		<a class="alert_right button" href="{{ url_for('home_page') }}">Skip</a>
+	</div>
+{% endif %}
+
+
+<h1>Set Password</h1>
+
+{% from "macros/forms.html" import render_field, render_submit_field %}
+<form action="" method="POST" class="form" role="form">
+	{{ form.hidden_tag() }}
+
+	{% if not current_user.email %}
+		{{ render_field(form.email, tabindex=230) }}
+
+		<p>
+			Your email is needed to recover your account if you forget your
+			password, and to optionally send notifications.
+			Your email will never be shared to a third-party.
+		</p>
+	{% endif %}
+
+	{{ render_field(form.password, tabindex=230) }}
+	{{ render_field(form.password2, tabindex=240) }}
+
+	{{ render_submit_field(form.submit, tabindex=280) }}
+</form>
+
+{% endblock %}

app/templates/users/user_profile_page.html

@@ -6,6 +6,14 @@
 
 {% block content %}
 
+{% if not current_user.is_authenticated and user.rank == user.rank.NOT_JOINED and user.forums_username %}
+<div class="box box_grey alert alert-info">
+	Is this you? Claim your account now!
+
+	<a class="alert_right button" href="{{ url_for('user_claim_page', username=user.forums_username) }}">Claim</a>
+</div>
+{% endif %}
+
 <div class="box box_grey">
 	<h2>{{ user.display_name }}</h2>
 
@@ -24,7 +32,7 @@
 						Minetest Forum
 					</a>
 				{% elif user == current_user %}
-					<a href="">Link Forums Account</a>
+					No forum account
 				{% endif %}
 
 				{% if (user.forums_username and user.github_username) or user == current_user %}
@@ -42,27 +50,19 @@
 				{% endif %}
 			</td>
 		</tr>
-	</table>
-</div>
-
-<div class="box box_grey">
-	<h2>Packages</h2>
-	<div class="box-body">
-		<ul>
-			{% for p in user.packages %}
-				<li><a href="{{ p.getDetailsURL() }}">
-					{{ p.title }} by {{ p.author.display_name }}
-				</a></li>
-			{% else %}
-				<li><i>No packages available</i></ul>
-			{% endfor %}
-		</ul>
-		{% if user == current_user or user.checkPerm(current_user, "CHANGE_AUTHOR") %}
-			<p><a class="button" href="{{ url_for('create_edit_package_page', author=user.username) }}">
-				Create
-			</a></p>
+		{% if user == current_user %}
+			<tr>
+				<td>Password:</td>
+				<td>
+					{% if user.password %}
+						Set | <a href="{{ url_for('user.change_password') }}">Change</a>
+					{% else %}
+						Not set | <a href="{{ url_for('set_password_page') }}">Set</a>
+					{% endif %}
+				</td>
+			</tr>
 		{% endif %}
-	</div>
+	</table>
 </div>
 
 {% if form %}
@@ -94,4 +94,23 @@
 		</form>
 	</div>
 {% endif %}
+
+{% from "macros/packagegridtile.html" import render_pkggrid %}
+{{ render_pkggrid(packages, show_author=False) }}
+
+{% if topics_to_add %}
+	<div class="box box_grey">
+		<h2>Unadded Packages</h2>
+
+		<div class="box-body">
+			<p>
+				List of your forum topics which do not have a matching package.
+			</p>
+
+			{% from "macros/topics.html" import render_topics_table %}
+			{{ render_topics_table(topics_to_add, show_author=False) }}
+		</div>
+	</div>
+{% endif %}
+
 {% endblock %}

app/utils.py

@@ -50,6 +50,25 @@ def doFileUpload(file, allowedExtensions, fileTypeName):
 	file.save(os.path.join("app/public/uploads", filename))
 	return "/uploads/" + filename
 
+def make_flask_user_password(plaintext_str):
+	# http://passlib.readthedocs.io/en/stable/modular_crypt_format.html
+	# http://passlib.readthedocs.io/en/stable/lib/passlib.hash.bcrypt.html#format-algorithm
+	# Flask_User stores passwords in the Modular Crypt Format.
+	# https://github.com/lingthio/Flask-User/blob/master/flask_user/user_manager__settings.py#L166
+	#   Note that Flask_User allows customizing password algorithms.
+	#   USER_PASSLIB_CRYPTCONTEXT_SCHEMES defaults to bcrypt but if
+	#   default changes or is customized, the code below needs adapting.
+	# Individual password values will look like:
+	#   $2b$12$.az4S999Ztvy/wa3UdQvMOpcki1Qn6VYPXmEFMIdWQyYs7ULnH.JW
+	#   $XX$RR$SSSSSSSSSSSSSSSSSSSSSSHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH
+	# $XX : Selects algorithm (2b is bcrypt).
+	# $RR : Selects bcrypt key expansion rounds (12 is 2**12 rounds).
+	# $SSS... : 22 chars of (random, per-password) salt
+	#  HHH... : 31 remaining chars of password hash (note no dollar sign)
+	import bcrypt
+	plaintext = plaintext_str.encode("UTF-8")
+	password = bcrypt.hashpw(plaintext, bcrypt.gensalt())
+	return password.decode("UTF-8")
 
 def _do_login_user(user, remember_me=False):
 	def _call_or_get(v):
@@ -68,7 +87,7 @@ def _do_login_user(user, remember_me=False):
 
 	user.active = True
 	if not user.rank.atLeast(UserRank.NEW_MEMBER):
-		user.rank = UserRank.NEW_MEMBER
+		user.rank = UserRank.MEMBER
 
 	db.session.commit()
 
@@ -99,7 +118,7 @@ def loginUser(user):
 	if user_manager.enable_username:
 		user_mixin = user_manager.find_user_by_username(user.username)
 
-	return _do_login_user(user_mixin, False)
+	return _do_login_user(user_mixin, True)
 
 def rank_required(rank):
 	def decorator(f):
@@ -151,3 +170,13 @@ def clearNotifications(url):
 	if current_user.is_authenticated:
 		Notification.query.filter_by(user=current_user, url=url).delete()
 		db.session.commit()
+
+
+YESES = ["yes", "true", "1", "on"]
+
+def isYes(val):
+	return val and val.lower() in YESES
+
+
+def isNo(val):
+	return val and not isYes(val)

app/views/__init__.py

@@ -27,10 +27,18 @@ from werkzeug.contrib.cache import SimpleCache
 from urllib.parse import urlparse
 cache = SimpleCache()
 
+@app.template_filter()
+def throw(err):
+	raise Exception(err)
+
 @app.template_filter()
 def domain(url):
 	return urlparse(url).netloc
 
+@app.template_filter()
+def datetime(value):
+    return value.strftime("%Y-%m-%d %H:%M") + " UTC"
+
 @app.route("/uploads/<path:path>")
 def send_upload(path):
 	return send_from_directory("public/uploads", path)
@@ -38,12 +46,15 @@ def send_upload(path):
 @app.route("/")
 @menu.register_menu(app, ".", "Home")
 def home_page():
-	query = Package.query.filter_by(approved=True, soft_deleted=False)
-	count = query.count()
-	packages = query.order_by(db.desc(Package.created_at)).limit(15).all()
-	return render_template("index.html", packages=packages, count=count)
+	query   = Package.query.filter_by(approved=True, soft_deleted=False)
+	count   = query.count()
+	new     = query.order_by(db.desc(Package.created_at)).limit(15).all()
+	popular = query.order_by(db.desc(Package.score)).limit(6).all()
+	return render_template("index.html", new=new, popular=popular, count=count)
 
-from . import users, githublogin, packages, sass, tasks, admin, notifications
+from . import users, githublogin, packages, meta, threads, api
+from . import tasks, admin, notifications, tagseditor, licenseseditor
+from . import sass, thumbnails
 
 @menu.register_menu(app, ".help", "Help", order=19, endpoint_arguments_constructor=lambda: { 'path': 'help' })
 @app.route('/<path:path>/')
@@ -54,7 +65,11 @@ def flatpage(path):
 
 @app.before_request
 def do_something_whenever_a_request_comes_in():
-	if current_user.is_authenticated and current_user.rank == UserRank.BANNED:
-		flash("You have been banned.", "error")
-		logout_user()
-		return redirect(url_for('user.login'))
+	if current_user.is_authenticated:
+		if current_user.rank == UserRank.BANNED:
+			flash("You have been banned.", "error")
+			logout_user()
+			return redirect(url_for('user.login'))
+		elif current_user.rank == UserRank.NOT_JOINED:
+			current_user.rank = UserRank.MEMBER
+			db.session.commit()

app/views/admin.py

@@ -20,20 +20,22 @@ from flask_user import *
 from flask.ext import menu
 from app import app
 from app.models import *
-from app.tasks.importtasks import importRepoScreenshot
-from app.tasks.forumtasks  import importUsersFromModList
+from celery import uuid
+from app.tasks.importtasks import importRepoScreenshot, importAllDependencies, makeVCSRelease
+from app.tasks.forumtasks  import importTopicList
 from flask_wtf import FlaskForm
 from wtforms import *
-from app.utils import loginUser, rank_required
+from app.utils import loginUser, rank_required, triggerNotif
+import datetime
 
 @app.route("/admin/", methods=["GET", "POST"])
 @rank_required(UserRank.ADMIN)
 def admin_page():
 	if request.method == "POST":
 		action = request.form["action"]
-		if action == "importusers":
-			task = importUsersFromModList.delay()
-			return redirect(url_for("check_task", id=task.id, r=url_for("user_list_page")))
+		if action == "importmodlist":
+			task = importTopicList.delay()
+			return redirect(url_for("check_task", id=task.id, r=url_for("todo_topics_page")))
 		elif action == "importscreenshots":
 			packages = Package.query \
 				.filter_by(soft_deleted=False) \
@@ -52,6 +54,44 @@ def admin_page():
 				package.soft_deleted = False
 				db.session.commit()
 				return redirect(url_for("admin_page"))
+		elif action == "importdepends":
+			task = importAllDependencies.delay()
+			return redirect(url_for("check_task", id=task.id, r=url_for("admin_page")))
+		elif action == "modprovides":
+			packages = Package.query.filter_by(type=PackageType.MOD).all()
+			mpackage_cache = {}
+			for p in packages:
+				if len(p.provides) == 0:
+					p.provides.append(MetaPackage.GetOrCreate(p.name, mpackage_cache))
+
+			db.session.commit()
+			return redirect(url_for("admin_page"))
+		elif action == "recalcscores":
+			for p in Package.query.all():
+				p.recalcScore()
+
+			db.session.commit()
+			return redirect(url_for("admin_page"))
+		elif action == "vcsrelease":
+			for package in Package.query.filter(Package.repo.isnot(None)).all():
+				if package.releases.count() != 0:
+					continue
+
+				rel = PackageRelease()
+				rel.package  = package
+				rel.title    = datetime.date.today().isoformat()
+				rel.url      = ""
+				rel.task_id  = uuid()
+				rel.approved = True
+				db.session.add(rel)
+				db.session.commit()
+
+				makeVCSRelease.apply_async((rel.id, "master"), task_id=rel.task_id)
+
+				msg = "{}: Release {} created".format(package.title, rel.title)
+				triggerNotif(package.author, current_user, msg, rel.getEditURL())
+				db.session.commit()
+
 		else:
 			flash("Unknown action: " + action, "error")
 

app/views/api.py

@@ -0,0 +1,45 @@
+# Content DB
+# Copyright (C) 2018  rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+
+from flask import *
+from flask_user import *
+from app import app
+from app.models import *
+from app.utils import is_package_page
+from .packages import QueryBuilder
+
+@app.route("/api/packages/")
+def api_packages_page():
+	qb    = QueryBuilder()
+	query = qb.buildPackageQuery()
+
+	pkgs = [package.getAsDictionaryShort(app.config["BASE_URL"]) \
+			for package in query.all() if package.getDownloadRelease() is not None]
+	return jsonify(pkgs)
+
+@app.route("/api/packages/<author>/<name>/")
+@is_package_page
+def api_package_page(package):
+	return jsonify(package.getAsDictionary(app.config["BASE_URL"]))
+
+
+@app.route("/api/topics/")
+def api_topics_page():
+	query = ForumTopic.query \
+			.order_by(db.asc(ForumTopic.wip), db.asc(ForumTopic.name), db.asc(ForumTopic.title))
+	pkgs = [t.getAsDictionary() for t in query.all()]
+	return jsonify(pkgs)

app/views/githublogin.py

@@ -51,9 +51,9 @@ def github_authorized(oauth_token):
 	if current_user and current_user.is_authenticated:
 		if userByGithub is None:
 			current_user.github_username = username
-			db.session.add(auth)
 			db.session.commit()
-			return redirect(url_for("gitAccount", id=auth.id))
+			flash("Linked github to account", "success")
+			return redirect(url_for("home_page"))
 		else:
 			flash("Github account is already associated with another user", "danger")
 			return redirect(url_for("home_page"))
@@ -64,7 +64,10 @@ def github_authorized(oauth_token):
 			flash("Unable to find an account for that Github user", "error")
 			return redirect(url_for("user_claim_page"))
 		elif loginUser(userByGithub):
-			return redirect(next_url or url_for("home_page"))
+			if current_user.password is None:
+				return redirect(next_url or url_for("set_password_page", optional=True))
+			else:
+				return redirect(next_url or url_for("home_page"))
 		else:
 			flash("Authorization failed [err=gh-login-failed]", "danger")
 			return redirect(url_for("user.login"))

app/views/licenseseditor.py

@@ -0,0 +1,62 @@
+# Content DB
+# Copyright (C) 2018  rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+
+from flask import *
+from flask_user import *
+from app import app
+from app.models import *
+from flask_wtf import FlaskForm
+from wtforms import *
+from wtforms.validators import *
+from app.utils import rank_required
+
+@app.route("/licenses/")
+@rank_required(UserRank.MODERATOR)
+def license_list_page():
+	return render_template("admin/licenses/list.html", licenses=License.query.order_by(db.asc(License.name)).all())
+
+class LicenseForm(FlaskForm):
+	name	 = StringField("Name", [InputRequired(), Length(3,100)])
+	is_foss  = BooleanField("Is FOSS")
+	submit   = SubmitField("Save")
+
+@app.route("/licenses/new/", methods=["GET", "POST"])
+@app.route("/licenses/<name>/edit/", methods=["GET", "POST"])
+@rank_required(UserRank.MODERATOR)
+def createedit_license_page(name=None):
+	license = None
+	if name is not None:
+		license = License.query.filter_by(name=name).first()
+		if license is None:
+			abort(404)
+
+	form = LicenseForm(formdata=request.form, obj=license)
+	if request.method == "GET" and license is None:
+		form.is_foss.data = True
+	elif request.method == "POST" and form.validate():
+		if license is None:
+			license = License(form.name.data)
+			db.session.add(license)
+			flash("Created license " + form.name.data, "success")
+		else:
+			flash("Updated license " + form.name.data, "success")
+
+		form.populate_obj(license)
+		db.session.commit()
+		return redirect(url_for("license_list_page"))
+
+	return render_template("admin/licenses/edit.html", license=license, form=form)

app/views/meta.py

@@ -0,0 +1,34 @@
+# Content DB
+# Copyright (C) 2018  rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+
+from flask import *
+from flask_user import *
+from app import app
+from app.models import *
+
+@app.route("/metapackages/")
+def meta_package_list_page():
+	mpackages = MetaPackage.query.order_by(db.asc(MetaPackage.name)).all()
+	return render_template("meta/list.html", mpackages=mpackages)
+
+@app.route("/metapackages/<name>/")
+def meta_package_page(name):
+	mpackage = MetaPackage.query.filter_by(name=name).first()
+	if mpackage is None:
+		abort(404)
+
+	return render_template("meta/view.html", mpackage=mpackage)

app/views/notifications.py

@@ -23,4 +23,11 @@ from app.models import *
 @app.route("/notifications/")
 @login_required
 def notifications_page():
-    return render_template("notifications/list.html")
+	return render_template("notifications/list.html")
+
+@app.route("/notifications/clear/", methods=["POST"])
+@login_required
+def clear_notifications_page():
+	current_user.notifications.clear()
+	db.session.commit()
+	return redirect(url_for("notifications_page"))

app/views/packages/__init__.py

@@ -29,38 +29,111 @@ from flask_wtf import FlaskForm
 from wtforms import *
 from wtforms.validators import *
 from wtforms.ext.sqlalchemy.fields import QuerySelectField, QuerySelectMultipleField
+from sqlalchemy import or_, any_
 
-# TODO: the following could be made into one route, except I"m not sure how
-# to do the menu
+
+class QueryBuilder:
+	title  = None
+	types  = None
+	search = None
+
+	def __init__(self):
+		title = "Packages"
+
+		# Get request types
+		types = request.args.getlist("type")
+		types = [PackageType.get(tname) for tname in types]
+		types = [type for type in types if type is not None]
+		if len(types) > 0:
+			title = ", ".join([type.value + "s" for type in types])
+
+		self.title  = title
+		self.types  = types
+		self.search = request.args.get("q")
+		self.lucky  = "lucky" in request.args
+		self.hide_nonfree = isNo(request.args.get("nonfree"))
+		self.limit  = 1 if self.lucky else None
+
+	def buildPackageQuery(self):
+		query = Package.query.filter_by(soft_deleted=False, approved=True)
+
+		if len(self.types) > 0:
+			query = query.filter(Package.type.in_(self.types))
+
+		if self.search is not None and self.search.strip() != "":
+			query = query.filter(Package.title.ilike('%' + self.search + '%'))
+
+		query = query.order_by(db.desc(Package.score))
+
+		if self.hide_nonfree:
+			query = query.filter(Package.license.has(License.is_foss == True))
+			query = query.filter(Package.media_license.has(License.is_foss == True))
+
+		if self.limit:
+			query = query.limit(self.limit)
+
+		return query
+
+	def buildTopicQuery(self):
+		topics = ForumTopic.query \
+				.filter(~ db.exists().where(Package.forums==ForumTopic.topic_id)) \
+				.order_by(db.asc(ForumTopic.wip), db.asc(ForumTopic.name), db.asc(ForumTopic.title)) \
+				.filter(ForumTopic.title.ilike('%' + self.search + '%'))
+
+		if len(self.types) > 0:
+			topics = topics.filter(ForumTopic.type.in_(self.types))
+
+		if self.hide_nonfree:
+			query = query.filter(Package.license.has(License.is_foss == True))
+			query = query.filter(Package.media_license.has(License.is_foss == True))
+
+		if self.limit:
+			topics = topics.limit(self.limit)
+
+		return topics
 
 @menu.register_menu(app, ".mods", "Mods", order=11, endpoint_arguments_constructor=lambda: { 'type': 'mod' })
 @menu.register_menu(app, ".games", "Games", order=12, endpoint_arguments_constructor=lambda: { 'type': 'game' })
 @menu.register_menu(app, ".txp", "Texture Packs", order=13, endpoint_arguments_constructor=lambda: { 'type': 'txp' })
 @app.route("/packages/")
 def packages_page():
-	type = request.args.get("type")
-	if type is not None:
-		type = PackageType[type.upper()]
+	if shouldReturnJson():
+		return redirect(url_for("api_packages_page"))
 
-	title = "Packages"
-	query = Package.query.filter_by(soft_deleted=False)
+	qb    = QueryBuilder()
+	query = qb.buildPackageQuery()
+	title = qb.title
 
-	if type is not None:
-		title = type.value + "s"
-		query = query.filter_by(type=type, approved=True)
+	if qb.lucky:
+		package = query.first()
+		if package:
+			return redirect(package.getDetailsURL())
+
+		topic = qb.buildTopicQuery().first()
+		if topic:
+			return redirect("https://forum.minetest.net/viewtopic.php?t=" + str(topic.topic_id))
+
+	page  = int(request.args.get("page") or 1)
+	num   = min(42, int(request.args.get("n") or 100))
+	query = query.paginate(page, num, True)
 
 	search = request.args.get("q")
-	if search is not None:
-		query = query.filter(Package.title.contains(search))
+	type_name = request.args.get("type")
 
-	if shouldReturnJson():
-		pkgs = [package.getAsDictionary(app.config["BASE_URL"]) \
-				for package in query.all() if package.getDownloadRelease() is not None]
-		return jsonify(pkgs)
-	else:
-		tags = Tag.query.all()
-		return render_template("packages/list.html", title=title, packages=query.all(), \
-				query=search, tags=tags, type=None if type is None else type.toName())
+	next_url = url_for("packages_page", type=type_name, q=search, page=query.next_num) \
+			if query.has_next else None
+	prev_url = url_for("packages_page", type=type_name, q=search, page=query.prev_num) \
+			if query.has_prev else None
+
+	topics = None
+	if qb.search and not query.has_next:
+		topics = qb.buildTopicQuery().all()
+
+	tags = Tag.query.all()
+	return render_template("packages/list.html", \
+			title=title, packages=query.items, topics=topics, \
+			query=search, tags=tags, type=type_name, \
+			next_url=next_url, prev_url=prev_url, page=page, page_max=query.pages, packages_count=query.total)
 
 
 def getReleases(package):
@@ -73,14 +146,68 @@ def getReleases(package):
 @app.route("/packages/<author>/<name>/")
 @is_package_page
 def package_page(package):
-	if shouldReturnJson():
-		return jsonify(package.getAsDictionary(app.config["BASE_URL"]))
-	else:
-		clearNotifications(package.getDetailsURL())
+	clearNotifications(package.getDetailsURL())
 
-		releases = getReleases(package)
-		requests = [r for r in package.requests if r.status == 0]
-		return render_template("packages/view.html", package=package, releases=releases, requests=requests)
+	alternatives = None
+	if package.type == PackageType.MOD:
+		alternatives = Package.query \
+				.filter_by(name=package.name, type=PackageType.MOD, soft_deleted=False) \
+				.filter(Package.id != package.id) \
+				.order_by(db.asc(Package.title)) \
+				.all()
+
+	show_similar_topics = current_user == package.author or \
+			package.checkPerm(current_user, Permission.APPROVE_NEW)
+
+	similar_topics = None if not show_similar_topics else \
+			ForumTopic.query \
+				.filter_by(name=package.name) \
+				.filter(ForumTopic.topic_id != package.forums) \
+				.filter(~ db.exists().where(Package.forums==ForumTopic.topic_id)) \
+				.order_by(db.asc(ForumTopic.name), db.asc(ForumTopic.title)) \
+				.all()
+
+	releases = getReleases(package)
+	requests = [r for r in package.requests if r.status == 0]
+
+	review_thread = package.review_thread
+	if review_thread is not None and not review_thread.checkPerm(current_user, Permission.SEE_THREAD):
+		review_thread = None
+
+	topic_error = None
+	topic_error_lvl = "warning"
+	if not package.approved and package.forums is not None:
+		errors = []
+		if Package.query.filter_by(forums=package.forums, soft_deleted=False).count() > 1:
+			errors.append("<b>Error: Another package already uses this forum topic!</b>")
+			topic_error_lvl = "error"
+
+		topic = ForumTopic.query.get(package.forums)
+		if topic is not None:
+			if topic.author != package.author:
+				errors.append("<b>Error: Forum topic author doesn't match package author.</b>")
+				topic_error_lvl = "error"
+
+			if topic.wip:
+				errors.append("Warning: Forum topic is in WIP section, make sure package meets playability standards.")
+		elif package.type != PackageType.TXP:
+			errors.append("Warning: Forum topic not found. This may happen if the topic has only just been created.")
+
+		topic_error = "<br />".join(errors)
+
+
+	threads = Thread.query.filter_by(package_id=package.id)
+	if not current_user.is_authenticated:
+		threads = threads.filter_by(private=False)
+	elif not current_user.rank.atLeast(UserRank.EDITOR) and not current_user == package.author:
+		threads = threads.filter(or_(Thread.private == False, Thread.author == current_user))
+
+
+	return render_template("packages/view.html", \
+			package=package, releases=releases, requests=requests, \
+			alternatives=alternatives, similar_topics=similar_topics, \
+			review_thread=review_thread, topic_error=topic_error, topic_error_lvl=topic_error_lvl, \
+			threads=threads.all())
 
 
 @app.route("/packages/<author>/<name>/download/")
@@ -100,16 +227,18 @@ def package_download_page(package):
 
 
 class PackageForm(FlaskForm):
-	name          = StringField("Name", [InputRequired(), Length(1, 20), Regexp("^[a-z0-9_]", 0, "Lower case letters (a-z), digits (0-9), and underscores (_) only")])
-	title         = StringField("Title", [InputRequired(), Length(3, 50)])
-	shortDesc     = StringField("Short Description", [InputRequired(), Length(1,200)])
-	desc          = TextAreaField("Long Description", [Optional(), Length(0,10000)])
+	name          = StringField("Name (Technical)", [InputRequired(), Length(1, 20), Regexp("^[a-z0-9_]", 0, "Lower case letters (a-z), digits (0-9), and underscores (_) only")])
+	title         = StringField("Title (Human-readable)", [InputRequired(), Length(3, 50)])
+	shortDesc     = StringField("Short Description (Plaintext)", [InputRequired(), Length(1,200)])
+	desc          = TextAreaField("Long Description (Markdown)", [Optional(), Length(0,10000)])
 	type          = SelectField("Type", [InputRequired()], choices=PackageType.choices(), coerce=PackageType.coerce, default=PackageType.MOD)
-	license       = QuerySelectField("License", [InputRequired()], query_factory=lambda: License.query, get_pk=lambda a: a.id, get_label=lambda a: a.name)
+	license       = QuerySelectField("License", [InputRequired()], query_factory=lambda: License.query.order_by(db.asc(License.name)), get_pk=lambda a: a.id, get_label=lambda a: a.name)
+	media_license = QuerySelectField("Media License", [InputRequired()], query_factory=lambda: License.query.order_by(db.asc(License.name)), get_pk=lambda a: a.id, get_label=lambda a: a.name)
+	provides_str  = StringField("Provides (mods included in package)", [Optional(), Length(0,1000)])
 	tags          = QuerySelectMultipleField('Tags', query_factory=lambda: Tag.query.order_by(db.asc(Tag.name)), get_pk=lambda a: a.id, get_label=lambda a: a.title)
-	harddeps      = QuerySelectMultipleField('Dependencies', query_factory=lambda: Package.query.filter_by(soft_deleted=False,approved=True).join(User).order_by(db.asc(Package.title), db.asc(User.display_name)), get_pk=lambda a: a.id, get_label=lambda a: a.title + " by " + a.author.display_name)
-	softdeps      = QuerySelectMultipleField('Soft Dependencies', query_factory=lambda: Package.query.filter_by(soft_deleted=False,approved=True).join(User).order_by(db.asc(Package.title), db.asc(User.display_name)), get_pk=lambda a: a.id, get_label=lambda a: a.title + " by " + a.author.display_name)
-	repo          = StringField("Repo URL", [Optional(), URL()])
+	harddep_str   = StringField("Hard Dependencies", [Optional(), Length(0,1000)])
+	softdep_str   = StringField("Soft Dependencies", [Optional(), Length(0,1000)])
+	repo          = StringField("VCS Repository URL", [Optional(), URL()])
 	website       = StringField("Website URL", [Optional(), URL()])
 	issueTracker  = StringField("Issue Tracker URL", [Optional(), URL()])
 	forums	      = IntegerField("Forum Topic ID", [Optional(), NumberRange(0,999999)])
@@ -146,6 +275,18 @@ def create_edit_package_page(author=None, name=None):
 		form = PackageForm(formdata=request.form, obj=package)
 
 	# Initial form class from post data and default data
+	if request.method == "GET":
+		if package is None:
+			form.name.data   = request.args.get("bname")
+			form.title.data  = request.args.get("title")
+			form.repo.data   = request.args.get("repo")
+			form.forums.data = request.args.get("forums")
+		else:
+			deps = package.dependencies
+			form.harddep_str.data  = ",".join([str(x) for x in deps if not x.optional])
+			form.softdep_str.data  = ",".join([str(x) for x in deps if     x.optional])
+			form.provides_str.data = MetaPackage.ListToSpec(package.provides)
+
 	if request.method == "POST" and form.validate():
 		wasNew = False
 		if not package:
@@ -166,21 +307,52 @@ def create_edit_package_page(author=None, name=None):
 
 		form.populate_obj(package) # copy to row
 
+		mpackage_cache = {}
+		package.provides.clear()
+		mpackages = MetaPackage.SpecToList(form.provides_str.data, mpackage_cache)
+		for m in mpackages:
+			package.provides.append(m)
+
+		Dependency.query.filter_by(depender=package).delete()
+		deps = Dependency.SpecToList(package, form.harddep_str.data, mpackage_cache)
+		for dep in deps:
+			dep.optional = False
+			db.session.add(dep)
+
+		deps = Dependency.SpecToList(package, form.softdep_str.data, mpackage_cache)
+		for dep in deps:
+			dep.optional = True
+			db.session.add(dep)
+
+		if wasNew and package.type == PackageType.MOD and not package.name in mpackage_cache:
+			m = MetaPackage.GetOrCreate(package.name, mpackage_cache)
+			package.provides.append(m)
+
 		package.tags.clear()
 		for tag in form.tags.raw_data:
 			package.tags.append(Tag.query.get(tag))
 
 		db.session.commit() # save
 
-		if wasNew and package.canImportScreenshot():
+		next_url = package.getDetailsURL()
+		if wasNew and package.repo is not None:
 			task = importRepoScreenshot.delay(package.id)
-			return redirect(url_for("check_task", id=task.id, r=package.getDetailsURL()))
+			next_url = url_for("check_task", id=task.id, r=next_url)
 
-		return redirect(package.getDetailsURL())
+		if wasNew and ("WTFPL" in package.license.name or "WTFPL" in package.media_license.name):
+			next_url = url_for("flatpage", path="help/wtfpl", r=next_url)
+
+		return redirect(next_url)
+
+	package_query = Package.query.filter_by(approved=True, soft_deleted=False)
+	if package is not None:
+		package_query = package_query.filter(Package.id != package.id)
 
 	enableWizard = name is None and request.method != "POST"
 	return render_template("packages/create_edit.html", package=package, \
-			form=form, author=author, enable_wizard=enableWizard)
+			form=form, author=author, enable_wizard=enableWizard, \
+			packages=package_query.all(), \
+			mpackages=MetaPackage.query.order_by(db.asc(MetaPackage.name)).all())
 
 @app.route("/packages/<author>/<name>/approve/", methods=["POST"])
 @login_required
@@ -227,4 +399,4 @@ def delete_package_page(package):
 
 	return redirect(url)
 
-from . import todo, screenshots, editrequests, releases
+from . import todo, screenshots, releases

app/views/packages/editrequests.py

@@ -58,8 +58,13 @@ def create_edit_editrequest_page(package, id=None):
 		edited_package = Package(package)
 		erequest.applyAll(edited_package)
 
-
 	form = EditRequestForm(request.form, obj=edited_package)
+	if request.method == "GET":
+		deps = edited_package.dependencies
+		form.harddep_str.data  = ",".join([str(x) for x in deps if not x.optional])
+		form.softdep_str.data  = ",".join([str(x) for x in deps if     x.optional])
+		form.provides_str.data = MetaPackage.ListToSpec(edited_package.provides)
+
 	if request.method == "POST" and form.validate():
 		if erequest is None:
 			erequest = EditRequest()

app/views/packages/releases.py

@@ -30,16 +30,14 @@ from wtforms import *
 from wtforms.validators import *
 
 class CreatePackageReleaseForm(FlaskForm):
-	name	   = StringField("Name")
-	title	   = StringField("Title")
+	title	   = StringField("Title", [InputRequired(), Length(1, 30)])
 	uploadOpt  = RadioField ("Method", choices=[("upload", "File Upload")], default="upload")
 	vcsLabel   = StringField("VCS Commit or Branch", default="master")
 	fileUpload = FileField("File Upload")
 	submit	   = SubmitField("Save")
 
 class EditPackageReleaseForm(FlaskForm):
-	name     = StringField("Name")
-	title    = StringField("Title")
+	title    = StringField("Title", [InputRequired(), Length(1, 30)])
 	url      = StringField("URL", [URL])
 	task_id  = StringField("Task ID")
 	approved = BooleanField("Is Approved")
@@ -54,8 +52,8 @@ def create_release_page(package):
 
 	# Initial form class from post data and default data
 	form = CreatePackageReleaseForm()
-	if package.canMakeReleaseFromVCS():
-		form["uploadOpt"].choices = [("vcs", "From VCS Commit or Branch"), ("upload", "File Upload")]
+	if package.repo is not None:
+		form["uploadOpt"].choices = [("vcs", "From Git Commit or Branch"), ("upload", "File Upload")]
 		if request.method != "POST":
 			form["uploadOpt"].data = "vcs"
 
@@ -93,6 +91,23 @@ def create_release_page(package):
 
 	return render_template("packages/release_new.html", package=package, form=form)
 
+@app.route("/packages/<author>/<name>/releases/<id>/download/")
+@is_package_page
+def download_release_page(package, id):
+	release = PackageRelease.query.get(id)
+	if release is None or release.package != package:
+		abort(404)
+
+	if release is None:
+		if "application/zip" in request.accept_mimetypes and \
+				not "text/html" in request.accept_mimetypes:
+			return "", 204
+		else:
+			flash("No download available.", "error")
+			return redirect(package.getDetailsURL())
+	else:
+		return redirect(release.url, code=300)
+
 @app.route("/packages/<author>/<name>/releases/<id>/", methods=["GET", "POST"])
 @login_required
 @is_package_page

app/views/packages/screenshots.py

@@ -54,7 +54,7 @@ def create_screenshot_page(package, id=None):
 		if uploadedPath is not None:
 			ss = PackageScreenshot()
 			ss.package = package
-			ss.title   = form["title"].data
+			ss.title   = form["title"].data or "Untitled"
 			ss.url     = uploadedPath
 			db.session.add(ss)
 
@@ -91,7 +91,7 @@ def edit_screenshot_page(package, id):
 			wasApproved = screenshot.approved
 
 			if canEdit:
-				screenshot.title = form["title"].data
+				screenshot.title = form["title"].data or "Untitled"
 
 			if canApprove:
 				screenshot.approved = form["approved"].data

app/views/packages/todo.py

@@ -40,6 +40,25 @@ def todo_page():
 	if canApproveScn:
 		screenshots = PackageScreenshot.query.filter_by(approved=False).all()
 
-	return render_template("todo.html", title="Reports and Work Queue",
+
+	topics_to_add = ForumTopic.query \
+			.filter(~ db.exists().where(Package.forums==ForumTopic.topic_id)) \
+			.count()
+
+	return render_template("todo/list.html", title="Reports and Work Queue",
 		packages=packages, releases=releases, screenshots=screenshots,
-		canApproveNew=canApproveNew, canApproveRel=canApproveRel, canApproveScn=canApproveScn)
+		canApproveNew=canApproveNew, canApproveRel=canApproveRel, canApproveScn=canApproveScn,
+		topics_to_add=topics_to_add)
+
+
+@app.route("/todo/topics/")
+@login_required
+def todo_topics_page():
+	total = ForumTopic.query.count()
+
+	topics = ForumTopic.query \
+			.filter(~ db.exists().where(Package.forums==ForumTopic.topic_id)) \
+			.order_by(db.asc(ForumTopic.wip), db.asc(ForumTopic.name), db.asc(ForumTopic.title)) \
+			.all()
+
+	return render_template("todo/topics.html", topics=topics, total=total)

app/views/tagseditor.py

@@ -0,0 +1,57 @@
+# Content DB
+# Copyright (C) 2018  rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+
+from flask import *
+from flask_user import *
+from app import app
+from app.models import *
+from flask_wtf import FlaskForm
+from wtforms import *
+from wtforms.validators import *
+from app.utils import rank_required
+
+@app.route("/tags/")
+@rank_required(UserRank.MODERATOR)
+def tag_list_page():
+	return render_template("admin/tags/list.html", tags=Tag.query.order_by(db.asc(Tag.title)).all())
+
+class TagForm(FlaskForm):
+	title	 = StringField("Title", [InputRequired(), Length(3,100)])
+	name     = StringField("Name", [Optional(), Length(1, 20), Regexp("^[a-z0-9_]", 0, "Lower case letters (a-z), digits (0-9), and underscores (_) only")])
+	submit   = SubmitField("Save")
+
+@app.route("/tags/new/", methods=["GET", "POST"])
+@app.route("/tags/<name>/edit/", methods=["GET", "POST"])
+@rank_required(UserRank.MODERATOR)
+def createedit_tag_page(name=None):
+	tag = None
+	if name is not None:
+		tag = Tag.query.filter_by(name=name).first()
+		if tag is None:
+			abort(404)
+
+	form = TagForm(formdata=request.form, obj=tag)
+	if request.method == "POST" and form.validate():
+		if tag is None:
+			tag = Tag(form.title.data)
+			db.session.add(tag)
+		else:
+			form.populate_obj(tag)
+		db.session.commit()
+		return redirect(url_for("createedit_tag_page", name=tag.name))
+
+	return render_template("admin/tags/edit.html", tag=tag, form=form)

app/views/threads.py

@@ -0,0 +1,190 @@
+# Content DB
+# Copyright (C) 2018  rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+
+from flask import *
+from flask_user import *
+from app import app
+from app.models import *
+from app.utils import triggerNotif, clearNotifications
+
+from flask_wtf import FlaskForm
+from wtforms import *
+from wtforms.validators import *
+
+@app.route("/threads/")
+def threads_page():
+	query = Thread.query
+	if not Permission.SEE_THREAD.check(current_user):
+		query = query.filter_by(private=False)
+	return render_template("threads/list.html", threads=query.all())
+
+
+@app.route("/threads/<int:id>/subscribe/", methods=["POST"])
+@login_required
+def thread_subscribe_page(id):
+	thread = Thread.query.get(id)
+	if thread is None or not thread.checkPerm(current_user, Permission.SEE_THREAD):
+		abort(404)
+
+	if current_user in thread.watchers:
+		flash("Already subscribed!", "success")
+	else:
+		flash("Subscribed to thread", "success")
+		thread.watchers.append(current_user)
+		db.session.commit()
+
+	return redirect(url_for("thread_page", id=id))
+
+
+@app.route("/threads/<int:id>/unsubscribe/", methods=["POST"])
+@login_required
+def thread_unsubscribe_page(id):
+	thread = Thread.query.get(id)
+	if thread is None or not thread.checkPerm(current_user, Permission.SEE_THREAD):
+		abort(404)
+
+	if current_user in thread.watchers:
+		flash("Unsubscribed!", "success")
+		thread.watchers.remove(current_user)
+		db.session.commit()
+	else:
+		flash("Not subscribed to thread", "success")
+
+	return redirect(url_for("thread_page", id=id))
+
+
+@app.route("/threads/<int:id>/", methods=["GET", "POST"])
+def thread_page(id):
+	clearNotifications(url_for("thread_page", id=id))
+
+	thread = Thread.query.get(id)
+	if thread is None or not thread.checkPerm(current_user, Permission.SEE_THREAD):
+		abort(404)
+
+	if current_user.is_authenticated and request.method == "POST":
+		comment = request.form["comment"]
+
+		if len(comment) <= 500 and len(comment) > 3:
+			reply = ThreadReply()
+			reply.author = current_user
+			reply.comment = comment
+			db.session.add(reply)
+
+			thread.replies.append(reply)
+			if not current_user in thread.watchers:
+				thread.watchers.append(current_user)
+
+			msg = None
+			if thread.package is None:
+				msg = "New comment on '{}'".format(thread.title)
+			else:
+				msg = "New comment on '{}' on package {}".format(thread.title, thread.package.title)
+
+
+			for user in thread.watchers:
+				if user != current_user:
+					triggerNotif(user, current_user, msg, url_for("thread_page", id=thread.id))
+
+			db.session.commit()
+
+			return redirect(url_for("thread_page", id=id))
+
+		else:
+			flash("Comment needs to be between 3 and 500 characters.")
+
+	return render_template("threads/view.html", thread=thread)
+
+
+class ThreadForm(FlaskForm):
+	title	= StringField("Title", [InputRequired(), Length(3,100)])
+	comment = TextAreaField("Comment", [InputRequired(), Length(10, 500)])
+	private = BooleanField("Private")
+	submit  = SubmitField("Open Thread")
+
+@app.route("/threads/new/", methods=["GET", "POST"])
+@login_required
+def new_thread_page():
+	form = ThreadForm(formdata=request.form)
+
+	package = None
+	if "pid" in request.args:
+		package = Package.query.get(int(request.args.get("pid")))
+		if package is None:
+			flash("Unable to find that package!", "error")
+
+	# Don't allow making threads on approved packages for now
+	if package is None:
+		abort(403)
+
+	def_is_private   = request.args.get("private") or False
+	if not package.approved:
+		def_is_private = True
+	allow_change     = package.approved
+	is_review_thread = package is not None and not package.approved
+
+	# Check that user can make the thread
+	if not package.checkPerm(current_user, Permission.CREATE_THREAD):
+		flash("Unable to create thread!", "error")
+		return redirect(url_for("home_page"))
+
+	# Only allow creating one thread when not approved
+	elif is_review_thread and package.review_thread is not None:
+		flash("A review thread already exists!", "error")
+		if request.method == "GET":
+			return redirect(url_for("thread_page", id=package.review_thread.id))
+
+	# Set default values
+	elif request.method == "GET":
+		form.private.data = def_is_private
+		form.title.data   = request.args.get("title") or ""
+
+	# Validate and submit
+	elif request.method == "POST" and form.validate():
+		thread = Thread()
+		thread.author  = current_user
+		thread.title   = form.title.data
+		thread.private = form.private.data if allow_change else def_is_private
+		thread.package = package
+		db.session.add(thread)
+
+		thread.watchers.append(current_user)
+		if package is not None and package.author != current_user:
+			thread.watchers.append(package.author)
+
+		reply = ThreadReply()
+		reply.thread  = thread
+		reply.author  = current_user
+		reply.comment = form.comment.data
+		db.session.add(reply)
+
+		thread.replies.append(reply)
+
+		db.session.commit()
+
+		if is_review_thread:
+			package.review_thread = thread
+
+		if package is not None:
+			triggerNotif(package.author, current_user,
+					"New thread '{}' on package {}".format(thread.title, package.title), url_for("thread_page", id=thread.id))
+
+		db.session.commit()
+
+		return redirect(url_for("thread_page", id=thread.id))
+
+
+	return render_template("threads/new.html", form=form, allow_private_change=allow_change)

app/views/thumbnails.py

@@ -0,0 +1,71 @@
+# Content DB
+# Copyright (C) 2018  rubenwardy
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+
+from flask import *
+from app import app
+
+import os
+from PIL import Image
+
+ALLOWED_RESOLUTIONS=[(350,233)]
+
+def mkdir(path):
+	if not os.path.isdir(path):
+		os.mkdir(path)
+
+mkdir("app/public/thumbnails/")
+
+def resize_and_crop(img_path, modified_path, size):
+	img = Image.open(img_path)
+
+	# Get current and desired ratio for the images
+	img_ratio = img.size[0] / float(img.size[1])
+	ratio = size[0] / float(size[1])
+
+	# Is more portrait than target, scale and crop
+	if ratio > img_ratio:
+		img = img.resize((int(size[0]), int(size[0] * img.size[1] / img.size[0])),
+				Image.BICUBIC)
+		box = (0, (img.size[1] - size[1]) / 2, img.size[0], (img.size[1] + size[1]) / 2)
+		img = img.crop(box)
+
+	# Is more landscape than target, scale and crop
+	elif ratio < img_ratio:
+		img = img.resize((int(size[1] * img.size[0] / img.size[1]), int(size[1])),
+				Image.BICUBIC)
+		box = ((img.size[0] - size[0]) / 2, 0, (img.size[0] + size[0]) / 2, img.size[1])
+		img = img.crop(box)
+
+	# Is exactly the same ratio as target
+	else:
+		img = img.resize(size, Image.BICUBIC)
+
+	img.save(modified_path)
+
+@app.route("/thumbnails/<img>")
+@app.route("/thumbnails/<int:w>x<int:h>/<img>")
+def make_thumbnail(img, w=350, h=233):
+	if not (w, h) in ALLOWED_RESOLUTIONS:
+		abort(403)
+
+	mkdir("app/public/thumbnails/{}x{}/".format(w, h))
+
+	cache_filepath  = "public/thumbnails/{}x{}/{}".format(w, h, img)
+	source_filepath = "public/uploads/" + img
+
+	resize_and_crop("app/" + source_filepath, "app/" + cache_filepath, (w, h))
+	return send_file(cache_filepath)

app/views/users.py

@@ -25,15 +25,16 @@ from flask_wtf import FlaskForm
 from flask_user.forms import RegisterForm
 from wtforms import *
 from wtforms.validators import *
-from app.utils import rank_required, randomString
+from app.utils import rank_required, randomString, loginUser
 from app.tasks.forumtasks import checkForumAccount
 from app.tasks.emails import sendVerifyEmail
+from app.tasks.phpbbparser import getProfile
 
 # Define the User profile form
 class UserProfileForm(FlaskForm):
-	display_name = StringField("Display name", [InputRequired(), Length(2, 20)])
-	email = StringField("Email")
-	rank = SelectField("Rank", [InputRequired()], choices=UserRank.choices(), coerce=UserRank.coerce, default=UserRank.NEW_MEMBER)
+	display_name = StringField("Display name", [Optional(), Length(2, 20)])
+	email = StringField("Email", [Optional(), Email()])
+	rank = SelectField("Rank", [Optional()], choices=UserRank.choices(), coerce=UserRank.coerce, default=UserRank.NEW_MEMBER)
 	submit = SubmitField("Save")
 
 @app.route("/users/", methods=["GET"])
@@ -90,12 +91,81 @@ def user_profile_page(username):
 			# Redirect to home page
 			return redirect(url_for("user_profile_page", username=username))
 
+	packages = user.packages.filter_by(soft_deleted=False)
+	if not current_user.is_authenticated or (user != current_user and not current_user.canAccessTodoList()):
+		packages = packages.filter_by(approved=True)
+	packages = packages.order_by(db.asc(Package.title))
+
+	topics_to_add = None
+	if current_user == user or user.checkPerm(current_user, Permission.CHANGE_AUTHOR):
+		topics_to_add = ForumTopic.query \
+					.filter_by(author_id=user.id) \
+					.filter(~ db.exists().where(Package.forums==ForumTopic.topic_id)) \
+					.order_by(db.asc(ForumTopic.name), db.asc(ForumTopic.title)) \
+					.all()
+
 	# Process GET or invalid POST
 	return render_template("users/user_profile_page.html",
-			user=user, form=form)
+			user=user, form=form, packages=packages, topics_to_add=topics_to_add)
+
+class SetPasswordForm(FlaskForm):
+	email = StringField("Email", [Optional(), Email()])
+	password = PasswordField("New password", [InputRequired(), Length(2, 20)])
+	password2 = PasswordField("Verify password", [InputRequired(), Length(2, 20)])
+	submit = SubmitField("Save")
+
+@app.route("/user/set-password/", methods=["GET", "POST"])
+@login_required
+def set_password_page():
+	if current_user.password is not None:
+		return redirect(url_for("user.change_password"))
+
+	form = SetPasswordForm(request.form)
+	if current_user.email == None:
+		form.email.validators = [InputRequired(), Email()]
+
+	if request.method == "POST" and form.validate():
+		one = form.password.data
+		two = form.password2.data
+		if one == two:
+			# Hash password
+			hashed_password = user_manager.hash_password(form.password.data)
+
+			# Change password
+			user_manager.update_password(current_user, hashed_password)
+
+			# Send 'password_changed' email
+			if user_manager.enable_email and user_manager.send_password_changed_email and current_user.email:
+				emails.send_password_changed_email(current_user)
+
+			# Send password_changed signal
+			signals.user_changed_password.send(current_app._get_current_object(), user=current_user)
+
+			# Prepare one-time system message
+			flash('Your password has been changed successfully.', 'success')
+
+			newEmail = form["email"].data
+			if newEmail != current_user.email and newEmail.strip() != "":
+				token = randomString(32)
+
+				ver = UserEmailVerification()
+				ver.user = current_user
+				ver.token = token
+				ver.email = newEmail
+				db.session.add(ver)
+				db.session.commit()
+
+				task = sendVerifyEmail.delay(newEmail, token)
+				return redirect(url_for("check_task", id=task.id, r=url_for("user_profile_page", username=current_user.username)))
+			else:
+				return redirect(url_for("user_profile_page", username=current_user.username))
+		else:
+			flash("Passwords do not match", "error")
+
+	return render_template("users/set_password.html", form=form, optional=request.args.get("optional"))
 
 
-@app.route("/users/claim/", methods=["GET", "POST"])
+@app.route("/user/claim/", methods=["GET", "POST"])
 def user_claim_page():
 	username = request.args.get("username")
 	if username is None:
@@ -116,8 +186,15 @@ def user_claim_page():
 		if user is not None and method == "github":
 			return redirect(url_for("github_signin_page"))
 
+	token = None
+	if "forum_token" in session:
+		token = session["forum_token"]
+	else:
+		token = randomString(32)
+		session["forum_token"] = token
+
 	if request.method == "POST":
-		ctype    = request.form.get("claim_type")
+		ctype	= request.form.get("claim_type")
 		username = request.form.get("username")
 
 		if username is None or len(username.strip()) < 2:
@@ -126,12 +203,41 @@ def user_claim_page():
 			task = checkForumAccount.delay(username)
 			return redirect(url_for("check_task", id=task.id, r=url_for("user_claim_page", username=username, method="github")))
 		elif ctype == "forum":
-			token = request.form.get("token")
-			flash("Unimplemented", "error")
+			user = User.query.filter_by(forums_username=username).first()
+			if user is not None and user.rank.atLeast(UserRank.NEW_MEMBER):
+				flash("That user has already been claimed!", "error")
+				return redirect(url_for("user_claim_page"))
+
+			# Get signature
+			sig = None
+			try:
+				profile = getProfile("https://forum.minetest.net", username)
+				sig = profile.signature
+			except IOError:
+				flash("Unable to get forum signature - does the user exist?", "error")
+				return redirect(url_for("user_claim_page", username=username))
+
+			# Look for key
+			if token in sig:
+				if user is None:
+					user = User(username)
+					user.forums_username = username
+					db.session.add(user)
+					db.session.commit()
+
+				if loginUser(user):
+					return redirect(url_for("set_password_page"))
+				else:
+					flash("Unable to login as user", "error")
+					return redirect(url_for("user_claim_page", username=username))
+
+			else:
+				flash("Could not find the key in your signature!", "error")
+				return redirect(url_for("user_claim_page", username=username))
 		else:
 			flash("Unknown claim type", "error")
 
-	return render_template("users/claim.html", username=username, key=randomString(32))
+	return render_template("users/claim.html", username=username, key=token)
 
 @app.route("/users/verify/")
 def verify_email_page():

config.example.cfg

@@ -22,3 +22,4 @@ MAIL_DEFAULT_SENDER=""
 MAIL_SERVER=""
 MAIL_PORT=587
 MAIL_USE_TLS=True
+MAIL_UTILS_ERROR_SEND_TO=[""]

migrations/versions/11b6ef362f98_.py

@@ -0,0 +1,28 @@
+"""empty message
+
+Revision ID: 11b6ef362f98
+Revises: 9fc23495713b
+Create Date: 2018-07-04 01:01:45.440662
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '11b6ef362f98'
+down_revision = '9fc23495713b'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('package', sa.Column('score', sa.Float(), nullable=False, server_default="0.0"))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('package', 'score')
+    # ### end Alembic commands ###

migrations/versions/28a427cbd4cf_.py

@@ -0,0 +1,29 @@
+"""empty message
+
+Revision ID: 28a427cbd4cf
+Revises: e9f534df23a8
+Create Date: 2018-06-03 01:47:33.006039
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import sqlalchemy.types as ty
+
+
+# revision identifiers, used by Alembic.
+revision = '28a427cbd4cf'
+down_revision = 'e9f534df23a8'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

migrations/versions/44e138485931_.py

@@ -0,0 +1,28 @@
+"""empty message
+
+Revision ID: 44e138485931
+Revises: 9e2ac631efb0
+Create Date: 2018-07-28 14:45:28.879331
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '44e138485931'
+down_revision = '9e2ac631efb0'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('package_release', sa.Column('commit_hash', sa.String(length=41), nullable=True, server_default=None))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('package_release', 'commit_hash')
+    # ### end Alembic commands ###

migrations/versions/4e482c47e519_.py

@@ -0,0 +1,39 @@
+"""empty message
+
+Revision ID: 4e482c47e519
+Revises: 900758871713
+Create Date: 2018-05-27 22:38:16.507155
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '4e482c47e519'
+down_revision = '900758871713'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('dependency',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('depender_id', sa.Integer(), nullable=True),
+    sa.Column('package_id', sa.Integer(), nullable=True),
+    sa.Column('meta_package_id', sa.Integer(), nullable=True),
+    sa.Column('optional', sa.Boolean(), nullable=False),
+    sa.ForeignKeyConstraint(['depender_id'], ['package.id'], ),
+    sa.ForeignKeyConstraint(['meta_package_id'], ['meta_package.id'], ),
+    sa.ForeignKeyConstraint(['package_id'], ['package.id'], ),
+    sa.PrimaryKeyConstraint('id'),
+    sa.UniqueConstraint('depender_id', 'package_id', 'meta_package_id', name='_dependency_uc')
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('dependency')
+    # ### end Alembic commands ###

migrations/versions/605b3d74ada1_.py

@@ -0,0 +1,55 @@
+"""empty message
+
+Revision ID: 605b3d74ada1
+Revises: 28a427cbd4cf
+Create Date: 2018-06-11 22:50:36.828818
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '605b3d74ada1'
+down_revision = '28a427cbd4cf'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('thread',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('package_id', sa.Integer(), nullable=True),
+    sa.Column('author_id', sa.Integer(), nullable=False),
+    sa.Column('title', sa.String(length=100), nullable=False),
+    sa.Column('private', sa.Boolean(), server_default='0', nullable=True),
+    sa.Column('created_at', sa.DateTime(), nullable=False),
+    sa.ForeignKeyConstraint(['author_id'], ['user.id'], ),
+    sa.ForeignKeyConstraint(['package_id'], ['package.id'], ),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table('thread_reply',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('thread_id', sa.Integer(), nullable=False),
+    sa.Column('comment', sa.String(length=500), nullable=False),
+    sa.Column('author_id', sa.Integer(), nullable=False),
+    sa.Column('created_at', sa.DateTime(), nullable=False),
+    sa.ForeignKeyConstraint(['author_id'], ['user.id'], ),
+    sa.ForeignKeyConstraint(['thread_id'], ['thread.id'], ),
+    sa.PrimaryKeyConstraint('id')
+    )
+
+    op.add_column('package', sa.Column('review_thread_id', sa.Integer(), nullable=True))
+    op.create_foreign_key(None, 'package', 'thread', ['review_thread_id'], ['id'])
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint(None, 'package', type_='foreignkey')
+    op.drop_constraint(None, 'package', type_='foreignkey')
+    op.drop_column('package', 'review_thread_id')
+    op.drop_table('thread_reply')
+    op.drop_table('thread')
+    # ### end Alembic commands ###

migrations/versions/900758871713_.py

@@ -0,0 +1,57 @@
+"""empty message
+
+Revision ID: 900758871713
+Revises: ea5a023711e0
+Create Date: 2018-05-27 16:36:44.258935
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '900758871713'
+down_revision = 'ea5a023711e0'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('meta_package',
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.Column('name', sa.String(length=100), nullable=False),
+        sa.PrimaryKeyConstraint('id'),
+        sa.UniqueConstraint('name')
+    )
+    op.create_table('provides',
+        sa.Column('package_id', sa.Integer(), nullable=False),
+        sa.Column('metapackage_id', sa.Integer(), nullable=False),
+        sa.ForeignKeyConstraint(['metapackage_id'], ['meta_package.id'], ),
+        sa.ForeignKeyConstraint(['package_id'], ['package.id'], ),
+        sa.PrimaryKeyConstraint('package_id', 'metapackage_id')
+    )
+    op.drop_table('harddeps')
+    op.drop_table('softdeps')
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('softdeps',
+        sa.Column('package_id', sa.INTEGER(), nullable=False),
+        sa.Column('dependency_id', sa.INTEGER(), nullable=False),
+        sa.ForeignKeyConstraint(['dependency_id'], ['package.id'], ),
+        sa.ForeignKeyConstraint(['package_id'], ['package.id'], ),
+        sa.PrimaryKeyConstraint('package_id', 'dependency_id')
+    )
+    op.create_table('harddeps',
+        sa.Column('package_id', sa.INTEGER(), nullable=False),
+        sa.Column('dependency_id', sa.INTEGER(), nullable=False),
+        sa.ForeignKeyConstraint(['dependency_id'], ['package.id'], ),
+        sa.ForeignKeyConstraint(['package_id'], ['package.id'], ),
+        sa.PrimaryKeyConstraint('package_id', 'dependency_id')
+    )
+    op.drop_table('provides')
+    op.drop_table('meta_package')
+    # ### end Alembic commands ###

migrations/versions/9e2ac631efb0_.py

@@ -0,0 +1,28 @@
+"""empty message
+
+Revision ID: 9e2ac631efb0
+Revises: 11b6ef362f98
+Create Date: 2018-07-06 23:16:50.507010
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '9e2ac631efb0'
+down_revision = '11b6ef362f98'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('forum_topic', sa.Column('wip', sa.Boolean(), nullable=False, server_default="0"))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('forum_topic', 'wip')
+    # ### end Alembic commands ###

migrations/versions/9fc23495713b_.py

@@ -0,0 +1,55 @@
+"""empty message
+
+Revision ID: 9fc23495713b
+Revises: de004661c5e1
+Create Date: 2018-07-04 00:03:20.123285
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '9fc23495713b'
+down_revision = 'de004661c5e1'
+branch_labels = None
+depends_on = None
+from sqlalchemy.dialects.postgresql import ENUM
+
+type_enum = ENUM('MOD', 'GAME', 'TXP', name='packagetype', create_type=False)
+
+def upgrade():
+    type_enum.create(op.get_bind(), checkfirst=True)
+
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('krock_forum_topic')
+    op.create_table('forum_topic',
+    sa.Column('topic_id', sa.Integer(), autoincrement=False, nullable=False),
+    sa.Column('author_id', sa.Integer(), nullable=False),
+    sa.Column('type', type_enum, nullable=True),
+    sa.Column('title', sa.String(length=200), nullable=False),
+    sa.Column('name', sa.String(length=30), nullable=True),
+    sa.Column('link', sa.String(length=200), nullable=True),
+    sa.Column('posts', sa.Integer(), nullable=False),
+    sa.Column('views', sa.Integer(), nullable=False),
+    sa.Column('created_at', sa.DateTime(), nullable=False),
+    sa.ForeignKeyConstraint(['author_id'], ['user.id'], ),
+    sa.PrimaryKeyConstraint('topic_id')
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('forum_topic')
+    op.create_table('krock_forum_topic',
+    sa.Column('topic_id', sa.Integer(), autoincrement=False, nullable=False),
+    sa.Column('author_id', sa.Integer(), nullable=False),
+    sa.Column('ttype', sa.Integer(), nullable=False),
+    sa.Column('title', sa.String(length=200), nullable=False),
+    sa.Column('name', sa.String(length=30), nullable=True),
+    sa.Column('link', sa.String(length=50), nullable=True),
+    sa.ForeignKeyConstraint(['author_id'], ['user.id'], ),
+    sa.PrimaryKeyConstraint('topic_id')
+    )
+    # ### end Alembic commands ###

migrations/versions/aa6d21889d22_.py

@@ -0,0 +1,34 @@
+"""empty message
+
+Revision ID: aa6d21889d22
+Revises: b254f55eadd2
+Create Date: 2018-05-29 18:28:28.540416
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'aa6d21889d22'
+down_revision = 'b254f55eadd2'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column('user', 'password',
+               existing_type=sa.VARCHAR(length=255),
+               nullable=True,
+               existing_server_default=sa.text("''"))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column('user', 'password',
+               existing_type=sa.VARCHAR(length=255),
+               nullable=False,
+               existing_server_default=sa.text("''"))
+    # ### end Alembic commands ###

migrations/versions/aa6d7b595a94_.py

@@ -0,0 +1,35 @@
+"""empty message
+
+Revision ID: aa6d7b595a94
+Revises: aa6d21889d22
+Create Date: 2018-05-29 20:09:56.647358
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'aa6d7b595a94'
+down_revision = 'aa6d21889d22'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('package', sa.Column('media_license_id', sa.Integer()))
+    op.execute('UPDATE package SET media_license_id=license_id')
+    op.alter_column('package', 'media_license_id', nullable=False)
+    op.alter_column('package', 'license_id', existing_type=sa.INTEGER(),  nullable=False)
+    op.create_foreign_key(None, 'package', 'license', ['media_license_id'], ['id'])
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column('package', 'license_id',
+               existing_type=sa.INTEGER(),
+               nullable=True)
+    op.drop_column('package', 'media_license_id')
+    # ### end Alembic commands ###

migrations/versions/adad68a5e370_.py

@@ -0,0 +1,37 @@
+"""empty message
+
+Revision ID: adad68a5e370
+Revises: d0bec9e5698e
+Create Date: 2018-06-02 18:23:18.123340
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'adad68a5e370'
+down_revision = 'd0bec9e5698e'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('krock_forum_topic',
+    sa.Column('topic_id', sa.Integer(), autoincrement=False, nullable=False),
+    sa.Column('author_id', sa.Integer(), nullable=False),
+    sa.Column('ttype', sa.Integer(), nullable=False),
+    sa.Column('title', sa.String(length=200), nullable=False),
+    sa.Column('name', sa.String(length=30), nullable=True),
+    sa.Column('link', sa.String(length=50), nullable=True),
+    sa.ForeignKeyConstraint(['author_id'], ['user.id'], ),
+    sa.PrimaryKeyConstraint('topic_id')
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('krock_forum_topic')
+    # ### end Alembic commands ###

migrations/versions/b254f55eadd2_.py

@@ -0,0 +1,29 @@
+"""empty message
+
+Revision ID: b254f55eadd2
+Revises: 4e482c47e519
+Create Date: 2018-05-27 23:51:11.008936
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'b254f55eadd2'
+down_revision = '4e482c47e519'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    conn = op.get_bind()
+    conn.execute("ALTER TYPE userrank ADD VALUE 'TRUSTED_MEMBER'")
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

migrations/versions/d0bec9e5698e_.py

@@ -0,0 +1,28 @@
+"""empty message
+
+Revision ID: d0bec9e5698e
+Revises: aa6d7b595a94
+Create Date: 2018-05-29 21:23:43.847738
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'd0bec9e5698e'
+down_revision = 'aa6d7b595a94'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('license', sa.Column('is_foss', sa.Boolean(), nullable=False, server_default="true"))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('license', 'is_foss')
+    # ### end Alembic commands ###

migrations/versions/de004661c5e1_.py

@@ -0,0 +1,34 @@
+"""empty message
+
+Revision ID: de004661c5e1
+Revises: 605b3d74ada1
+Create Date: 2018-06-11 23:38:38.611039
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'de004661c5e1'
+down_revision = '605b3d74ada1'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('watchers',
+    sa.Column('user_id', sa.Integer(), nullable=False),
+    sa.Column('thread_id', sa.Integer(), nullable=False),
+    sa.ForeignKeyConstraint(['thread_id'], ['thread.id'], ),
+    sa.ForeignKeyConstraint(['user_id'], ['user.id'], ),
+    sa.PrimaryKeyConstraint('user_id', 'thread_id')
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('watchers')
+    # ### end Alembic commands ###

migrations/versions/e9f534df23a8_.py

@@ -0,0 +1,34 @@
+"""empty message
+
+Revision ID: e9f534df23a8
+Revises: adad68a5e370
+Create Date: 2018-06-02 18:30:54.234366
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'e9f534df23a8'
+down_revision = 'adad68a5e370'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column('krock_forum_topic', 'link',
+               existing_type=sa.VARCHAR(length=50),
+               type_=sa.String(length=200),
+               existing_nullable=False)
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column('package_release', 'link',
+               existing_type=sa.String(length=200),
+               type_=sa.VARCHAR(length=50),
+               existing_nullable=False)
+    # ### end Alembic commands ###

requirements.txt

@@ -1,4 +1,4 @@
-Flask>=0.12.2
+Flask>=0.12.2,<1.0
 Flask-SQLAlchemy>=2.3
 Flask-Login>=0.4.1
 Flask-User==0.6.19
@@ -6,9 +6,11 @@ Flask-Menu>=0.7.0
 Flask-Markdown>=0.3
 GitHub-Flask>=3.2.0
 pyScss==1.3.4
-celery==4.0.2
+celery==4.1.1
 redis==2.10.6
 beautifulsoup4==4.6.0
 lxml==4.2.1
 Flask-FlatPages==0.6
 Flask-Migrate==2.1.1
+pillow==5.1.0
+GitPython==2.1.10

setup.py

@@ -23,6 +23,7 @@ if not "FLASK_CONFIG" in os.environ:
 test_data = len(sys.argv) >= 2 and sys.argv[1].strip() == "-t"
 
 from app.models import *
+from app.utils import make_flask_user_password
 
 def defineDummyData(licenses, tags, ruben):
 	ez = User("Shara")
@@ -36,6 +37,7 @@ def defineDummyData(licenses, tags, ruben):
 
 	jeija = User("Jeija")
 	jeija.github_username = "Jeija"
+	jeija.forums_username = "Jeija"
 	db.session.add(jeija)
 
 
@@ -255,7 +257,6 @@ No warranty is provided, express or implied, for any part of the project.
 	mod.title = "Sweet Foods"
 	mod.license = licenses["CC0"]
 	mod.type = PackageType.MOD
-	mod.harddeps.append(food)
 	mod.author = ruben
 	mod.tags.append(tags["player_effects"])
 	mod.repo = "https://github.com/rubenwardy/food_sweet/"
@@ -263,6 +264,7 @@ No warranty is provided, express or implied, for any part of the project.
 	mod.forums = 9039
 	mod.shortDesc = "Adds sweet food"
 	mod.desc = "This is the long desc"
+	food_sweet = mod
 	db.session.add(mod)
 
 	game1 = Package()
@@ -314,6 +316,23 @@ Uses the CTF PvP Engine.
 	rel.approved = True
 	db.session.add(rel)
 
+	db.session.commit()
+
+	metas = {}
+	for package in Package.query.filter_by(type=PackageType.MOD).all():
+		meta = None
+		try:
+			meta = metas[package.name]
+		except KeyError:
+			meta = MetaPackage(package.name)
+			db.session.add(meta)
+			metas[package.name] = meta
+		package.provides.append(meta)
+
+	dep = Dependency(food_sweet, meta=metas["food"])
+	db.session.add(dep)
+
+
 
 delete_db = len(sys.argv) >= 2 and sys.argv[1].strip() == "-d"
 if delete_db and os.path.isfile("db.sqlite"):
@@ -324,6 +343,8 @@ db.create_all()
 print("Filling database...")
 
 ruben = User("rubenwardy")
+ruben.active = True
+ruben.password = make_flask_user_password("tuckfrump")
 ruben.github_username = "rubenwardy"
 ruben.forums_username = "rubenwardy"
 ruben.rank = UserRank.ADMIN
@@ -341,11 +362,16 @@ for tag in ["Inventory", "Mapgen", "Building", \
 licenses = {}
 for license in ["GPLv2.1", "GPLv3", "LGPLv2.1", "LGPLv3", "AGPLv2.1", "AGPLv3",
 				"Apache", "BSD 3-Clause", "BSD 2-Clause", "CC0", "CC-BY-SA",
-				"CC-BY", "CC-BY-NC-SA", "MIT", "ZLib"]:
+				"CC-BY", "MIT", "ZLib", "Other (Free)"]:
 	row = License(license)
 	licenses[row.name] = row
 	db.session.add(row)
 
+for license in ["CC-BY-NC-SA", "Other (Non-free)"]:
+	row = License(license, False)
+	licenses[row.name] = row
+	db.session.add(row)
+
 if test_data:
 	defineDummyData(licenses, tags, ruben)