Add basic dependency resolution

Fixes #130

app/models.py

@@ -213,6 +213,9 @@ class Notification(db.Model):
 	url       = db.Column(db.String(200), nullable=True)
 
 	def __init__(self, us, cau, titl, ur):
+		if len(titl) > 100:
+			title = title[:99] + "…"
+
 		self.user   = us
 		self.causer = cau
 		self.title  = titl
@@ -395,7 +398,7 @@ class Package(db.Model):
 	forums       = db.Column(db.Integer,     nullable=True)
 
 	provides = db.relationship("MetaPackage", secondary=provides, lazy="subquery",
-			backref=db.backref("packages", lazy="dynamic"))
+			backref=db.backref("packages", lazy="dynamic", order_by=db.desc("score")))
 
 	dependencies = db.relationship("Dependency", backref="depender", lazy="dynamic", foreign_keys=[Dependency.depender_id])
 
@@ -438,6 +441,13 @@ class Package(db.Model):
 		else:
 			return "ready"
 
+	def getAsDictionaryKey(self):
+		return {
+			"name": self.name,
+			"author": self.author.display_name,
+			"type": self.type.toName(),
+		}
+
 	def getAsDictionaryShort(self, base_url, version=None, protonum=None):
 		tnurl = self.getThumbnailURL(1)
 		release = self.getDownloadRelease(version=version, protonum=protonum)

app/tasks/forumtasks.py

@@ -99,11 +99,19 @@ def parseTitle(title):
 def getLinksFromModSearch():
 	links = {}
 
-	contents = urllib.request.urlopen("https://krock-works.uk.to/minetest/modList.php").read().decode("utf-8")
-	for x in json.loads(contents):
-		link = x.get("link")
-		if link is not None:
-			links[int(x["topicId"])] = link
+	try:
+		contents = urllib.request.urlopen("https://krock-works.uk.to/minetest/modList.php").read().decode("utf-8")
+		for x in json.loads(contents):
+			try:
+				link = x.get("link")
+				if link is not None:
+					links[int(x["topicId"])] = link
+			except ValueError:
+				pass
+
+	except urllib.error.URLError:
+		print("Unable to open krocks mod search!")
+		return links
 
 	return links
 

app/tasks/importtasks.py

@@ -29,6 +29,10 @@ from app.utils import randomString
 
 class GithubURLMaker:
 	def __init__(self, url):
+		self.baseUrl = None
+		self.user = None
+		self.repo = None
+
 		# Rewrite path
 		import re
 		m = re.search("^\/([^\/]+)\/([^\/]+)\/?$", url.path)
@@ -51,6 +55,9 @@ class GithubURLMaker:
 	def getScreenshotURL(self):
 		return self.baseUrl + "/screenshot.png"
 
+	def getModConfURL(self):
+		return self.baseUrl + "/mod.conf"
+
 	def getCommitsURL(self, branch):
 		return "https://api.github.com/repos/{}/{}/commits?sha={}" \
 				.format(self.user, self.repo, urllib.parse.quote_plus(branch))
@@ -292,15 +299,17 @@ def cloneRepo(urlstr, ref=None, recursive=False):
 		gitUrl = generateGitURL(urlstr)
 		print("Cloning from " + gitUrl)
 		repo = git.Repo.clone_from(gitUrl, gitDir, \
-				progress=None, env=None, depth=1, recursive=recursive, kill_after_timeout=15)
+				progress=None, env=None, depth=1, recursive=recursive, kill_after_timeout=15, b=ref)
 
-		if ref is not None:
-			repo.create_head("myhead", ref).checkout()
 		return gitDir, repo
+
 	except GitCommandError as e:
 		# This is needed to stop the backtrace being weird
 		err = e.stderr
 
+	except gitdb.exc.BadName as e:
+		err = "Unable to find the reference " + (ref or "?") + "\n" + e.stderr
+
 	raise TaskError(err.replace("stderr: ", "") \
 			.replace("Cloning into '" + gitDir + "'...", "") \
 			.strip())
@@ -339,8 +348,11 @@ def makeVCSReleaseFromGithub(id, branch, release, url):
 		raise TaskError("Invalid github repo URL")
 
 	commitsURL = urlmaker.getCommitsURL(branch)
-	contents = urllib.request.urlopen(commitsURL).read().decode("utf-8")
-	commits = json.loads(contents)
+	try:
+		contents = urllib.request.urlopen(commitsURL).read().decode("utf-8")
+		commits = json.loads(contents)
+	except HTTPError:
+		raise TaskError("Unable to get commits for Github repository. Either the repository or reference doesn't exist.")
 
 	if len(commits) == 0 or not "sha" in commits[0]:
 		raise TaskError("No commits found")
@@ -349,7 +361,6 @@ def makeVCSReleaseFromGithub(id, branch, release, url):
 	release.task_id     = None
 	release.commit_hash = commits[0]["sha"]
 	release.approve(release.package.author)
-	print(release.url)
 	db.session.commit()
 
 	return release.url

app/utils.py

@@ -50,7 +50,7 @@ def doFileUpload(file, fileType, fileTypeDesc):
 	if fileType == "image":
 		allowedExtensions = ["jpg", "jpeg", "png"]
 		isImage = True
-	elif filetype == "zip":
+	elif fileType == "zip":
 		allowedExtensions = ["zip"]
 	else:
 		raise Exception("Invalid fileType")

app/views/api.py

@@ -32,12 +32,42 @@ def api_packages_page():
 			for package in query.all()]
 	return jsonify(pkgs)
 
+
 @app.route("/api/packages/<author>/<name>/")
 @is_package_page
 def api_package_page(package):
 	return jsonify(package.getAsDictionary(app.config["BASE_URL"]))
 
 
+@app.route("/api/packages/<author>/<name>/dependencies/")
+@is_package_page
+def api_package_deps_page(package):
+	ret = []
+
+	for dep in package.dependencies:
+		name = None
+		fulfilled_by = None
+
+		if dep.package:
+			name = dep.package.name
+			fulfilled_by = [ dep.package.getAsDictionaryKey() ]
+
+		elif dep.meta_package:
+			name = dep.meta_package.name
+			fulfilled_by = [ pkg.getAsDictionaryKey() for pkg in dep.meta_package.packages]
+
+		else:
+			raise "Malformed dependency"
+
+		ret.append({
+			"name": name,
+			"is_optional": dep.optional,
+			"packages": fulfilled_by
+		})
+
+	return jsonify(ret)
+
+
 @app.route("/api/topics/")
 def api_topics_page():
 	qb     = QueryBuilder(request.args)

app/views/packages/packages.py

@@ -180,9 +180,9 @@ class PackageForm(FlaskForm):
 	tags          = QuerySelectMultipleField('Tags', query_factory=lambda: Tag.query.order_by(db.asc(Tag.name)), get_pk=lambda a: a.id, get_label=lambda a: a.title)
 	harddep_str   = StringField("Hard Dependencies", [Optional()])
 	softdep_str   = StringField("Soft Dependencies", [Optional()])
-	repo          = StringField("VCS Repository URL", [Optional(), URL()])
-	website       = StringField("Website URL", [Optional(), URL()])
-	issueTracker  = StringField("Issue Tracker URL", [Optional(), URL()])
+	repo          = StringField("VCS Repository URL", [Optional(), URL()], filters = [lambda x: x or None])
+	website       = StringField("Website URL", [Optional(), URL()], filters = [lambda x: x or None])
+	issueTracker  = StringField("Issue Tracker URL", [Optional(), URL()], filters = [lambda x: x or None])
 	forums	      = IntegerField("Forum Topic ID", [Optional(), NumberRange(0,999999)])
 	submit	      = SubmitField("Save")
 

app/views/packages/releases.py

@@ -54,7 +54,7 @@ class CreatePackageReleaseForm(FlaskForm):
 class EditPackageReleaseForm(FlaskForm):
 	title    = StringField("Title", [InputRequired(), Length(1, 30)])
 	url      = StringField("URL", [URL])
-	task_id  = StringField("Task ID")
+	task_id  = StringField("Task ID", filters = [lambda x: x or None])
 	approved = BooleanField("Is Approved")
 	min_rel  = QuerySelectField("Minimum Minetest Version", [InputRequired()],
 			query_factory=lambda: get_mt_releases(False), get_pk=lambda a: a.id, get_label=lambda a: a.name)
@@ -164,7 +164,7 @@ def edit_release_page(package, id):
 		if package.checkPerm(current_user, Permission.CHANGE_RELEASE_URL):
 			release.url = form["url"].data
 			release.task_id = form["task_id"].data
-			if release.task_id.strip() == "":
+			if release.task_id is not None:
 				release.task_id = None
 
 		if canApprove:

app/views/users/users.py

@@ -31,9 +31,9 @@ from app.tasks.phpbbparser import getProfile
 # Define the User profile form
 class UserProfileForm(FlaskForm):
 	display_name = StringField("Display name", [Optional(), Length(2, 20)])
-	email = StringField("Email", [Optional(), Email()])
-	website_url = StringField("Website URL", [Optional(), URL()])
-	donate_url = StringField("Donation URL", [Optional(), URL()])
+	email = StringField("Email", [Optional(), Email()], filters = [lambda x: x or None])
+	website_url = StringField("Website URL", [Optional(), URL()], filters = [lambda x: x or None])
+	donate_url = StringField("Donation URL", [Optional(), URL()], filters = [lambda x: x or None])
 	rank = SelectField("Rank", [Optional()], choices=UserRank.choices(), coerce=UserRank.coerce, default=UserRank.NEW_MEMBER)
 	submit = SubmitField("Save")
 
@@ -162,8 +162,8 @@ def send_email_page(username):
 
 class SetPasswordForm(FlaskForm):
 	email = StringField("Email", [Optional(), Email()])
-	password = PasswordField("New password", [InputRequired(), Length(2, 20)])
-	password2 = PasswordField("Verify password", [InputRequired(), Length(2, 20)])
+	password = PasswordField("New password", [InputRequired(), Length(2, 100)])
+	password2 = PasswordField("Verify password", [InputRequired(), Length(2, 100)])
 	submit = SubmitField("Save")
 
 @app.route("/user/set-password/", methods=["GET", "POST"])

docker-compose.yml

@@ -33,5 +33,6 @@ services:
       - config.env
     volumes:
       - "./data/uploads:/home/cdb/app/public/uploads"
+      - "./app:/home/cdb/app"
     depends_on:
       - redis

requirements.txt

@@ -12,9 +12,10 @@ GitHub-Flask~=3.2
 SQLAlchemy-Searchable==1.0.3
 
 beautifulsoup4~=4.6
-celery~=4.2
+celery==4.1.1
+kombu==4.2.0
 GitPython~=2.1
 lxml~=4.2
 pillow~=5.3
 pyScss~=1.3
-redis~=3.0
+redis==2.10.6