Split importtasks.py
This commit is contained in:
rubenwardy
@@ -15,116 +15,19 @@
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
|
||||
import os, git, tempfile, shutil, gitdb, contextlib, datetime
|
||||
import os, shutil, gitdb
|
||||
from zipfile import ZipFile
|
||||
from git import GitCommandError
|
||||
from git_archive_all import GitArchiver
|
||||
from urllib.error import HTTPError
|
||||
import urllib.request
|
||||
from urllib.parse import urlsplit
|
||||
from zipfile import ZipFile
|
||||
|
||||
from kombu import uuid
|
||||
|
||||
from app.models import *
|
||||
from app.tasks import celery, TaskError
|
||||
from app.utils import randomString, getExtension, post_bot_message, addSystemNotification, addSystemAuditLog
|
||||
from app.utils import randomString, post_bot_message, addSystemNotification, addSystemAuditLog
|
||||
from app.utils.git import clone_repo, get_latest_tag, get_latest_commit, get_temp_dir
|
||||
from .minetestcheck import build_tree, MinetestCheckError, ContentType
|
||||
|
||||
|
||||
def generateGitURL(urlstr):
|
||||
scheme, netloc, path, query, frag = urlsplit(urlstr)
|
||||
|
||||
return "http://:@" + netloc + path + query
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def get_temp_dir():
|
||||
temp = os.path.join(tempfile.gettempdir(), randomString(10))
|
||||
yield temp
|
||||
shutil.rmtree(temp)
|
||||
|
||||
|
||||
# Clones a repo from an unvalidated URL.
|
||||
# Returns a tuple of path and repo on sucess.
|
||||
# Throws `TaskError` on failure.
|
||||
# Caller is responsible for deleting returned directory.
|
||||
@contextlib.contextmanager
|
||||
def clone_repo(urlstr, ref=None, recursive=False):
|
||||
gitDir = os.path.join(tempfile.gettempdir(), randomString(10))
|
||||
|
||||
err = None
|
||||
try:
|
||||
gitUrl = generateGitURL(urlstr)
|
||||
print("Cloning from " + gitUrl)
|
||||
|
||||
if ref is None:
|
||||
repo = git.Repo.clone_from(gitUrl, gitDir,
|
||||
progress=None, env=None, depth=1, recursive=recursive, kill_after_timeout=15)
|
||||
else:
|
||||
assert ref != ""
|
||||
|
||||
repo = git.Repo.init(gitDir)
|
||||
origin = repo.create_remote("origin", url=gitUrl)
|
||||
assert origin.exists()
|
||||
origin.fetch()
|
||||
repo.git.checkout(ref)
|
||||
|
||||
for submodule in repo.submodules:
|
||||
submodule.update(init=True)
|
||||
|
||||
yield repo
|
||||
shutil.rmtree(gitDir)
|
||||
return
|
||||
|
||||
except GitCommandError as e:
|
||||
# This is needed to stop the backtrace being weird
|
||||
err = e.stderr
|
||||
|
||||
except gitdb.exc.BadName as e:
|
||||
err = "Unable to find the reference " + (ref or "?") + "\n" + e.stderr
|
||||
|
||||
raise TaskError(err.replace("stderr: ", "") \
|
||||
.replace("Cloning into '" + gitDir + "'...", "") \
|
||||
.strip())
|
||||
|
||||
|
||||
def get_commit_hash(git_url, ref_name=None):
|
||||
git_url = generateGitURL(git_url)
|
||||
|
||||
if ref_name:
|
||||
ref_name = "refs/heads/" + ref_name
|
||||
else:
|
||||
ref_name = "HEAD"
|
||||
|
||||
g = git.cmd.Git()
|
||||
|
||||
remote_refs = {}
|
||||
for ref in g.ls_remote(git_url).split('\n'):
|
||||
hash_ref_list = ref.split('\t')
|
||||
remote_refs[hash_ref_list[1]] = hash_ref_list[0]
|
||||
|
||||
return remote_refs.get(ref_name)
|
||||
|
||||
|
||||
def get_latest_tag(git_url):
|
||||
with get_temp_dir() as git_dir:
|
||||
repo = git.Repo.init(git_dir)
|
||||
origin = repo.create_remote("origin", url=git_url)
|
||||
origin.fetch()
|
||||
|
||||
refs = repo.git.ls_remote(tags=True, sort="creatordate").split('\n')
|
||||
refs = [ref for ref in refs if ref.strip() != ""]
|
||||
if len(refs) == 0:
|
||||
return None, None
|
||||
|
||||
last_ref = refs[-1]
|
||||
hash_ref_list = last_ref.split('\t')
|
||||
|
||||
tag = hash_ref_list[1].replace("refs/tags/", "")
|
||||
commit_hash = repo.git.rev_parse(tag + "^{}")
|
||||
return tag, commit_hash
|
||||
|
||||
|
||||
@celery.task()
|
||||
def getMeta(urlstr, author):
|
||||
with clone_repo(urlstr, recursive=True) as repo:
|
||||
@@ -290,38 +193,12 @@ def importRepoScreenshot(id):
|
||||
return None
|
||||
|
||||
|
||||
@celery.task(bind=True)
|
||||
def importForeignDownloads(self, id):
|
||||
release = PackageRelease.query.get(id)
|
||||
if release is None:
|
||||
raise TaskError("No such release!")
|
||||
elif release.package is None:
|
||||
raise TaskError("No package attached to release")
|
||||
elif not release.url.startswith("http"):
|
||||
return
|
||||
|
||||
try:
|
||||
ext = getExtension(release.url)
|
||||
filename = randomString(10) + "." + ext
|
||||
filepath = os.path.join(app.config["UPLOAD_DIR"], filename)
|
||||
urllib.request.urlretrieve(release.url, filepath)
|
||||
|
||||
release.url = "/uploads/" + filename
|
||||
db.session.commit()
|
||||
|
||||
except urllib.error.URLError:
|
||||
db.session.rollback()
|
||||
release.task_id = self.request.id
|
||||
release.approved = False
|
||||
db.session.commit()
|
||||
|
||||
|
||||
def check_update_config_impl(package):
|
||||
config = package.update_config
|
||||
|
||||
if config.trigger == PackageUpdateTrigger.COMMIT:
|
||||
tag = None
|
||||
commit = get_commit_hash(package.repo, package.update_config.ref)
|
||||
commit = get_latest_commit(package.repo, package.update_config.ref)
|
||||
elif config.trigger == PackageUpdateTrigger.TAG:
|
||||
tag, commit = get_latest_tag(package.repo)
|
||||
else:
|
||||
|
||||
Reference in New Issue
Block a user