Move Github import to backend

Fixes #41

app/__init__.py

@@ -12,5 +12,5 @@ menu.Menu(app=app)
 markdown.Markdown(app, extensions=["fenced_code"], safe_mode=True, output_format="html5")
 github = GitHub(app)
 
-from . import models
+from . import models, tasks
 from .views import *

app/static/package_create.js

@@ -1,20 +1,4 @@
 $(function() {
-	function readConfig(text) {
-		var retval = {}
-
-		const lines = text.split("\n")
-		for (var i = 0; i < lines.length; i++) {
-			const idx = lines[i].indexOf("=")
-			if (idx > 0) {
-				const name = lines[i].substring(0, idx - 1).trim()
-				const value = lines[i].substring(idx + 1).trim()
-				retval[name] = value
-			}
-		}
-
-		return retval
-	}
-
 	function finish() {
 		$(".pkg_wiz_1").hide()
 		$(".pkg_wiz_2").hide()
@@ -22,6 +6,49 @@ $(function() {
 		$(".pkg_meta").show()
 	}
 
+	function getJSON(url) {
+		return new Promise(function(resolve, reject) {
+			fetch(url).then(function(response) {
+				response.text().then(function(txt) {
+					resolve(JSON.parse(txt))
+				}).catch(reject)
+			}).catch(reject)
+		})
+	}
+
+	function performTask(url) {
+		return new Promise(function(resolve, reject) {
+			getJSON(url).then(function(startResult) {
+				console.log(startResult)
+				if (typeof startResult.poll_url == "string") {
+					var tries = 0;
+					function retry() {
+						tries++;
+						if (tries > 10) {
+							reject("timeout")
+						} else {
+							console.log("Polling task in " + (tries*100) + "ms")
+							setTimeout(step, tries*100)
+						}
+					}
+					function step() {
+						getJSON(startResult.poll_url).then(function(res) {
+							if (res.status == "SUCCESS") {
+								console.log("Got result")
+								resolve(res.result)
+							} else {
+								retry()
+							}
+						}).catch(retry)
+					}
+					retry()
+				} else {
+					reject("Start task didn't return string!")
+				}
+			}).catch(reject)
+		})
+	}
+
 	function repoIsSupported(url) {
 		try {
 			return URI(url).hostname() == "github.com"
@@ -30,60 +57,6 @@ $(function() {
 		}
 	}
 
-	function getFile(url) {
-		return new Promise(function(resolve, reject) {
-			fetch(url).then(function(response) {
-				response.text().then(resolve).catch(reject)
-			}).catch(reject)
-		})
-	}
-
-	function getInfo(baseUrl) {
-		return new Promise(function(resolve, reject) {
-			getFile(baseUrl + "/mod.conf").then(function(text) {
-				var config = readConfig(text)
-
-				if (config["name"]) {
-					$("#name").val(config["name"])
-				}
-
-				if (config["description"]) {
-					const desc = config["description"]
-					const idx = desc.indexOf(".")
-					$("#shortDesc").val((idx < 5 || idx > 100) ? desc.substring(0, 100) : desc.substring(0, idx))
-					$("#desc").val(desc)
-				}
-
-				resolve()
-			}).catch(function() {
-				reject()
-			})
-		})
-	}
-
-	function importInfo(urlstr) {
-		// Convert to HTTPs
-		try {
-			var url = URI(urlstr).scheme("https")
-				.username("")
-				.password("")
-		} catch(e) {
-			return Promise.reject(e)
-		}
-		// Change domain
-		url = url.hostname("raw.githubusercontent.com")
-
-		// Rewrite path
-		const re = /^\/([^\/]+)\/([^\/]+)\/?$/
-		const results = re.exec(url.path())
-		if (results == null || results.length != 3) {
-			return Promise.reject("Unable to parse URL - please provide a direct URL to the repo")
-		}
-		url.path("/" + results[1] + "/" + results[2].replace(".git", "") + "/master")
-
-		return getInfo(url.toString())
-	}
-
 	$(".pkg_meta").hide()
 	$(".pkg_wiz_1").show()
 	$("#pkg_wiz_1_next").click(function() {
@@ -93,11 +66,22 @@ $(function() {
 			$(".pkg_wiz_2").show()
 			$(".pkg_repo").hide()
 
-			importInfo(repoURL).then(finish).catch(function(x) {
-				alert(x)
+			performTask("/tasks/getmeta/new/?url=" + encodeURI(repoURL)).then(function(result) {
+				console.log(result)
+				$("#name").val(result.name)
+				const desc = result.description || ""
+				if (desc.length > 0) {
+					const idx = desc.indexOf(".")
+					$("#shortDesc").val((idx < 5 || idx > 100) ? desc.substring(0, Math.min(desc.length, 100)) : desc.substring(0, idx))
+					$("#desc").val(desc)
+				}
+				finish()
+			}).catch(function(e) {
+				alert(e)
 				$(".pkg_wiz_1").show()
 				$(".pkg_wiz_2").hide()
 				$(".pkg_repo").show()
+				// finish()
 			})
 		} else {
 			finish()

app/tasks/__init__.py

@@ -0,0 +1,44 @@
+import flask
+from flask.ext.sqlalchemy import SQLAlchemy
+from celery import Celery
+from app import app
+from app.models import *
+
+class FlaskCelery(Celery):
+	def __init__(self, *args, **kwargs):
+		super(FlaskCelery, self).__init__(*args, **kwargs)
+		self.patch_task()
+
+		if 'app' in kwargs:
+			self.init_app(kwargs['app'])
+
+	def patch_task(self):
+		TaskBase = self.Task
+		_celery = self
+
+		class ContextTask(TaskBase):
+			abstract = True
+
+			def __call__(self, *args, **kwargs):
+				if flask.has_app_context():
+					return TaskBase.__call__(self, *args, **kwargs)
+				else:
+					with _celery.app.app_context():
+						return TaskBase.__call__(self, *args, **kwargs)
+
+		self.Task = ContextTask
+
+	def init_app(self, app):
+		self.app = app
+		self.config_from_object(app.config)
+
+def make_celery(app):
+	celery = FlaskCelery(app.import_name, backend=app.config['CELERY_RESULT_BACKEND'],
+					broker=app.config['CELERY_BROKER_URL'])
+
+	celery.init_app(app)
+	return celery
+
+celery = make_celery(app)
+
+from . import importtasks

app/tasks/importtasks.py

@@ -0,0 +1,68 @@
+import flask
+from flask.ext.sqlalchemy import SQLAlchemy
+import urllib.request
+from urllib.parse import urlparse
+
+from app import app
+from app.models import *
+from app.tasks import celery
+
+class GithubURLMaker:
+	def __init__(self, url):
+		# Rewrite path
+		import re
+		m = re.search("^\/([^\/]+)\/([^\/]+)\/?$", url.path)
+		if m is None:
+			return
+
+		user = m.group(1)
+		repo = m.group(2)
+		self.baseUrl = "https://raw.githubusercontent.com/" + user + "/" + repo.replace(".git", "") + "/master"
+
+	def isValid(self):
+		return self.baseUrl is not None
+
+	def getModConfURL(self):
+		return self.baseUrl + "/mod.conf"
+
+def parseConf(string):
+	retval = {}
+	for line in string.split("\n"):
+		idx = line.find("=")
+		if idx > 0:
+			key   = line[:idx-1].strip()
+			value = line[idx+1:].strip()
+			retval[key] = value
+
+	return retval
+
+@celery.task()
+def getMeta(urlstr):
+	url = urlparse(urlstr)
+
+	urlmaker = None
+	if url.netloc == "github.com":
+		urlmaker = GithubURLMaker(url)
+
+	if not urlmaker.isValid():
+		print("Error! Url maker not valid")
+		return
+
+	print(urlmaker.getModConfURL())
+
+	result = {}
+
+	try:
+		contents = urllib.request.urlopen(urlmaker.getModConfURL()).read().decode("utf-8")
+		conf = parseConf(contents)
+		for key in ["name", "description"]:
+			try:
+				result[key] = conf[key]
+			except KeyError:
+				pass
+
+		print(conf)
+	except OSError:
+		print("mod.conf does not exist")
+
+	return result

app/views/__init__.py

@@ -30,4 +30,4 @@ def home_page():
 	packages = Package.query.filter_by(approved=True).all()
 	return render_template("index.html", packages=packages)
 
-from . import users, githublogin, packages, sass
+from . import users, githublogin, packages, sass, api

app/views/api.py

@@ -0,0 +1,35 @@
+from flask import *
+from flask_user import *
+from flask.ext import menu
+from app import app
+from app.models import *
+from app.tasks import celery
+from app.tasks.importtasks import getMeta
+# from celery.result import AsyncResult
+
+from .utils import *
+
+@app.route("/tasks/getmeta/new/")
+def new_getmeta_page():
+	aresult = getMeta.delay(request.args.get("url"))
+	return jsonify({
+		"poll_url": url_for("check_task", id=aresult.id),
+	})
+
+@app.route("/tasks/<id>/")
+def check_task(id):
+    result = celery.AsyncResult(id)
+    status = result.status
+    traceback = result.traceback
+    result = result.result
+    if isinstance(result, Exception):
+        return jsonify({
+            'status': status,
+            'error': str(result),
+            # 'traceback': traceback,
+        })
+    else:
+        return jsonify({
+            'status': status,
+            'result': result,
+        })