From 017f5c746e894f9d87d927c848386459ea332378 Mon Sep 17 00:00:00 2001 From: Michael Wood Date: Fri, 28 Nov 2014 20:12:18 +0000 Subject: toaster: Add import layer feature. This feature allows users to import layers from git into their current project and associate it with the release of the current project and the dependencies for the newly imported layer with existing layers. It will also resolve the child dependencies of the dependencies added. [YOCTO #6595] Signed-off-by: Michael Wood Signed-off-by: Alexandru DAMIAN --- lib/toaster/toastergui/static/js/importlayer.js | 277 +++++++++++++++++++++ lib/toaster/toastergui/static/js/projectapp.js | 6 + lib/toaster/toastergui/templates/importlayer.html | 109 +++++--- .../toastergui/templates/layers_dep_modal.html | 68 +++++ lib/toaster/toastergui/urls.py | 2 + lib/toaster/toastergui/views.py | 107 +++++++- 6 files changed, 533 insertions(+), 36 deletions(-) create mode 100644 lib/toaster/toastergui/static/js/importlayer.js create mode 100644 lib/toaster/toastergui/templates/layers_dep_modal.html (limited to 'lib/toaster/toastergui') diff --git a/lib/toaster/toastergui/static/js/importlayer.js b/lib/toaster/toastergui/static/js/importlayer.js new file mode 100644 index 000000000..e2bc1ab60 --- /dev/null +++ b/lib/toaster/toastergui/static/js/importlayer.js @@ -0,0 +1,277 @@ +"use strict" + +function importLayerPageInit (ctx) { + + var layerDepBtn = $("#add-layer-dependency-btn"); + var importAndAddBtn = $("#import-and-add-btn"); + var layerNameInput = $("#layer-name"); + var vcsURLInput = $("#layer-git-repo-url"); + var gitRefInput = $("#layer-git-ref"); + var layerDepInput = $("#layer-dependency"); + var layerNameCtrl = $("#layer-name-ctrl"); + var duplicatedLayerName = $("#duplicated-layer-name-hint"); + + var layerDeps = {}; + var layerDepsDeps = {}; + var currentLayerDepSelection; + var validLayerName = /^(\w|-)+$/; + + $("#new-project-button").hide(); + + libtoaster.makeTypeahead(layerDepInput, ctx.xhrDataTypeaheadUrl, { type : "layers", project_id: ctx.projectId, include_added: "true" }, function(item){ + currentLayerDepSelection = item; + + layerDepBtn.removeAttr("disabled"); + }); + + + /* We automatically add "openembedded-core" layer for convenience as a + * dependency as pretty much all layers depend on this one + */ + $.getJSON(ctx.xhrDataTypeaheadUrl, { type : "layers", project_id: ctx.projectId, include_added: "true" , value: "openembedded-core" }, function(layer) { + if (layer.list.length == 1) { + currentLayerDepSelection = layer.list[0]; + layerDepBtn.click(); + } + }); + + layerDepBtn.click(function(){ + if (currentLayerDepSelection == undefined) + return; + + layerDeps[currentLayerDepSelection.id] = currentLayerDepSelection; + + /* Make a list item for the new layer dependency */ + var newLayerDep = $("
    • "); + + newLayerDep.data('layer-id', currentLayerDepSelection.id); + newLayerDep.children("span").tooltip(); + + var link = newLayerDep.children("a"); + link.attr("href", ctx.layerDetailsUrl+String(currentLayerDepSelection.id)); + link.text(currentLayerDepSelection.name); + link.tooltip({title: currentLayerDepSelection.tooltip, placement: "right"}); + + var trashItem = newLayerDep.children("span"); + trashItem.click(function () { + var toRemove = $(this).parent().data('layer-id'); + delete layerDeps[toRemove]; + $(this).parent().fadeOut(function (){ + $(this).remove(); + }); + }); + + $("#layer-deps-list").append(newLayerDep); + + libtoaster.getLayerDepsForProject(ctx.xhrDataTypeaheadUrl, ctx.projectId, currentLayerDepSelection.id, function (data){ + /* These are the dependencies of the layer added as a dependency */ + if (data.list.length > 0) { + currentLayerDepSelection.url = ctx.layerDetailsUrl+currentLayerDepSelection.id; + layerDeps[currentLayerDepSelection.id].deps = data.list + } + + /* Clear the current selection */ + layerDepInput.val(""); + currentLayerDepSelection = undefined; + layerDepBtn.attr("disabled","disabled"); + }, null); + }); + + importAndAddBtn.click(function(){ + /* arrray of all layer dep ids includes parent and child deps */ + var allDeps = []; + /* temporary object to use to do a reduce on the dependencies for each + * layer dependency added + */ + var depDeps = {}; + + /* the layers that have dependencies have an extra property "deps" + * look in this for each layer and reduce this to a unquie object + * of deps. + */ + for (var key in layerDeps){ + if (layerDeps[key].hasOwnProperty('deps')){ + for (var dep in layerDeps[key].deps){ + var layer = layerDeps[key].deps[dep]; + depDeps[layer.id] = layer; + } + } + allDeps.push(layerDeps[key].id); + } + + /* we actually want it as an array so convert it now */ + var depDepsArray = []; + for (var key in depDeps) + depDepsArray.push (depDeps[key]); + + if (depDepsArray.length > 0) { + var layer = { name: layerNameInput.val(), url: "#", id: -1 }; + show_layer_deps_modal(ctx.projectId, layer, depDepsArray, function(selected){ + /* Add the accepted dependencies to the allDeps array */ + if (selected.length > 0){ + allDeps.concat (selected); + } + import_and_add (); + }); + } else { + import_and_add (); + } + + function import_and_add () { + /* convert to a csv of all the deps to be added */ + var layerDepsCsv = allDeps.join(","); + + var layerData = { + name: layerNameInput.val(), + vcs_url: vcsURLInput.val(), + git_ref: gitRefInput.val(), + summary: $("#layer-summary").val(), + dir_path: $("#layer-subdir").val(), + project_id: ctx.projectId, + layer_deps: layerDepsCsv, + }; + + $.ajax({ + type: "POST", + url: ctx.xhrImportLayerUrl, + data: layerData, + headers: { 'X-CSRFToken' : $.cookie('csrftoken')}, + success: function (data) { + if (data.error != "ok") { + show_error_message(data, layerData); + console.log(data.error); + } else { + /* Success layer import now go to the project page */ + window.location.replace(ctx.projectPageUrl+'#/layerimported='+layerData.name); + } + }, + error: function (data) { + console.log("Call failed"); + console.log(data); + } + }); + } + }); + + function show_error_message(error, layerData) { + + var errorMsg = $("#import-error").fadeIn(); + var errorType = error.error; + var body = errorMsg.children("span"); + var title = errorMsg.children("h3"); + var optionsList = errorMsg.children("ul"); + var invalidLayerRevision = $("#invalid-layer-revision-hint"); + var layerRevisionCtrl = $("#layer-revision-ctrl"); + + /* remove any existing items */ + optionsList.children().each(function(){ $(this).remove(); }); + body.text(""); + title.text(""); + invalidLayerRevision.hide(); + layerNameCtrl.removeClass("error"); + layerRevisionCtrl.removeClass("error"); + + switch (errorType){ + case 'hint-layer-version-exists': + title.text("This layer already exists"); + body.html("A layer "+layerData.name+" already exists with this Git repository URL and this revision. You can:"); + optionsList.append("
  • Import "+layerData.name+" with a different revision
    • "); + optionsList.append("
  • or change the revision of the existing layer
    • "); + + layerRevisionCtrl.addClass("error"); + + invalidLayerRevision.html("A layer "+layerData.name+" already exists with this revision.
    You can import "+layerData.name+" with a different revision"); + invalidLayerRevision.show(); + break; + + case 'hint-layer-exists-with-different-url': + title.text("This layer already exists"); + body.html("A layer "+layerData.name+" already exists with a different Git repository URL:

    "+error.current_url+"

    You Can:"); + optionsList.append("
  • Import the layer under a different name
    • "); + optionsList.append("
  • or change the Git repository URL of the existing layer
    • "); + duplicatedLayerName.html("A layer "+layerData.name+" already exists with a different Git repository URL.
    To import this layer give it a different name."); + duplicatedLayerName.show(); + layerNameCtrl.addClass("error"); + break; + + case 'hint-layer-exists': + title.text("This layer already exists"); + body.html("A layer "+layerData.name+" already exists: You Can:"); + optionsList.append("
  • Import the layer under a different name
    • "); + break; + default: + title.text("Error") + body.text(data.error); + } + } + + function enable_import_btn (enabled) { + var importAndAddHint = $("#import-and-add-hint"); + + if (enabled) { + importAndAddBtn.removeAttr("disabled"); + importAndAddHint.hide(); + return; + } + + importAndAddBtn.attr("disabled", "disabled"); + importAndAddHint.show(); + } + + function check_form() { + var valid = false; + var inputs = $("input:required"); + + for (var i=0; iselect targets you want to build.", "alert-success"); }); + _cmdExecuteWithParam("/layerimported", function (layer) { + $scope.displayAlert($scope.zone2alerts, + "You have imported " + layer + + " and added it to your project.", "alert-success"); + }); + _cmdExecuteWithParam("/targetbuild=", function (targets) { var oldTargetName = $scope.targetName; $scope.targetName = targets.split(",").join(" "); diff --git a/lib/toaster/toastergui/templates/importlayer.html b/lib/toaster/toastergui/templates/importlayer.html index 7e48eac66..913f951c2 100644 --- a/lib/toaster/toastergui/templates/importlayer.html +++ b/lib/toaster/toastergui/templates/importlayer.html @@ -1,68 +1,115 @@ {% extends "baseprojectpage.html" %} {% load projecttags %} {% load humanize %} +{% load static %} {% block localbreadcrumb %} -
  • Layers
    • +
  • Import layer
    • {% endblock %} {% block projectinfomain %} + + + +

    Import layer

    + + {% include "layers_dep_modal.html" %}
    {% if project %} The layer you are importing must be compatible with {{project.release.name}} ({{project.release.description}}), which is the release you are using in this project. {% endif %} -
    - Layer repository information +
    + Layer repository information + + +
    + +
    + + + +
    + +
    + - -
    Layer dependencies (optional) - + -
    -
    - Import and add to project - Just import for the moment - To import a layer, you need to enter a repository URL, a branch, tag or commit and a layer name +
    + + To import a layer, you need to enter a repository URL, a branch, tag or commit and a layer name
    - + {% endblock %} diff --git a/lib/toaster/toastergui/templates/layers_dep_modal.html b/lib/toaster/toastergui/templates/layers_dep_modal.html new file mode 100644 index 000000000..821bbda29 --- /dev/null +++ b/lib/toaster/toastergui/templates/layers_dep_modal.html @@ -0,0 +1,68 @@ + + + + diff --git a/lib/toaster/toastergui/urls.py b/lib/toaster/toastergui/urls.py index b60f7614a..6e1b0ab91 100644 --- a/lib/toaster/toastergui/urls.py +++ b/lib/toaster/toastergui/urls.py @@ -76,6 +76,7 @@ urlpatterns = patterns('toastergui.views', url(r'^layers/$', 'layers', name='layers'), url(r'^layer/(?P\d+)/$', 'layerdetails', name='layerdetails'), + url(r'^layer/$', 'layerdetails', name='layerdetails'), url(r'^targets/$', 'targets', name='targets'), url(r'^machines/$', 'machines', name='machines'), @@ -92,6 +93,7 @@ urlpatterns = patterns('toastergui.views', url(r'^xhr_projectedit/(?P\d+)/$', 'xhr_projectedit', name='xhr_projectedit'), url(r'^xhr_datatypeahead/$', 'xhr_datatypeahead', name='xhr_datatypeahead'), + url(r'^xhr_importlayer/$', 'xhr_importlayer', name='xhr_importlayer'), # default redirection diff --git a/lib/toaster/toastergui/views.py b/lib/toaster/toastergui/views.py index 434e1180b..ec055d392 100755 --- a/lib/toaster/toastergui/views.py +++ b/lib/toaster/toastergui/views.py @@ -30,6 +30,7 @@ from orm.models import Task_Dependency, Recipe_Dependency, Package, Package_File from orm.models import Target_Installed_Package, Target_File, Target_Image_File, BuildArtifact from django.views.decorators.cache import cache_control from django.core.urlresolvers import reverse +from django.core.exceptions import MultipleObjectsReturned from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger from django.http import HttpResponseBadRequest, HttpResponseNotFound from django.utils import timezone @@ -2016,8 +2017,9 @@ if toastermain.settings.MANAGED: "name" : x.layercommit.layer.name, "giturl": x.layercommit.layer.vcs_url, "url": x.layercommit.layer.layer_index_url, - "layerdetailurl": reverse("layerdetails", args=(x.layercommit.layer.pk,)), - "branch" : { "name" : x.layercommit.up_branch.name, "layersource" : x.layercommit.up_branch.layer_source.name}}, + "layerdetailurl": reverse("layerdetails", args=(x.layercommit.pk,)), + # This branch name is actually the release + "branch" : { "name" : x.layercommit.commit, "layersource" : x.layercommit.up_branch.layer_source.name}}, prj.projectlayer_set.all().order_by("id")), "targets" : map(lambda x: {"target" : x.target, "task" : x.task, "pk": x.pk}, prj.projecttarget_set.all()), "freqtargets": freqtargets, @@ -2164,7 +2166,7 @@ if toastermain.settings.MANAGED: def _lv_to_dict(x): - return {"id": x.pk, "name": x.layer.name, + return {"id": x.pk, "name": x.layer.name, "tooltip": x.layer.vcs_url+" | "+x.commit, "detail": "(" + x.layer.vcs_url + (")" if x.up_branch == None else " | "+x.up_branch.name+")"), "giturl": x.layer.vcs_url, "layerdetailurl" : reverse('layerdetails', args=(x.pk,))} @@ -2174,8 +2176,9 @@ if toastermain.settings.MANAGED: # all layers for the current project queryset_all = prj.compatible_layerversions().filter(layer__name__icontains=request.GET.get('value','')) - # but not layers with equivalent layers already in project - queryset_all = queryset_all.exclude(pk__in = [x.id for x in prj.projectlayer_equivalent_set()])[:8] + # but not layers with equivalent layers already in project + if not request.GET.has_key('include_added'): + queryset_all = queryset_all.exclude(pk__in = [x.id for x in prj.projectlayer_equivalent_set()])[:8] # and show only the selected layers for this project final_list = set([x.get_equivalents_wpriority(prj)[0] for x in queryset_all]) @@ -2243,6 +2246,100 @@ if toastermain.settings.MANAGED: return HttpResponse(jsonfilter({"error":str(e) + "\n" + traceback.format_exc()}), content_type = "application/json") + def xhr_importlayer(request): + if (not request.POST.has_key('vcs_url') or + not request.POST.has_key('name') or + not request.POST.has_key('git_ref') or + not request.POST.has_key('project_id')): + return HttpResponse(jsonfilter({"error": "Missing parameters; requires vcs_url, name, git_ref and project_id"}), content_type = "application/json") + + # Rudimentary check for any possible html tags + if "<" in request.POST: + return HttpResponse(jsonfilter({"error": "Invalid character <"}), content_type = "application/json") + + prj = Project.objects.get(pk=request.POST['project_id']) + + # Strip trailing/leading whitespace from all values + # put into a new dict because POST one is immutable + post_data = dict() + for key,val in request.POST.iteritems(): + post_data[key] = val.strip() + + + # We need to know what release the current project is so that we + # can set the imported layer's up_branch_id + prj_branch_name = Release.objects.get(pk=prj.release_id).branch_name + up_branch, branch_created = Branch.objects.get_or_create(name=prj_branch_name, layer_source_id=LayerSource.TYPE_IMPORTED) + + layer_source = LayerSource.objects.get(sourcetype=LayerSource.TYPE_IMPORTED) + try: + layer, layer_created = Layer.objects.get_or_create(name=post_data['name']) + except MultipleObjectsReturned: + return HttpResponse(jsonfilter({"error": "hint-layer-exists"}), content_type = "application/json") + + if layer: + if layer_created: + layer.layer_source = layer_source + layer.vcs_url = post_data['vcs_url'] + if post_data.has_key('summary'): + layer.summary = layer.description = post_data['summary'] + + layer.up_date = timezone.now() + layer.save() + else: + # We have an existing layer by this name, let's see if the git + # url is the same, if it is then we can just create a new layer + # version for this layer. Otherwise we need to bail out. + if layer.vcs_url != post_data['vcs_url']: + return HttpResponse(jsonfilter({"error": "hint-layer-exists-with-different-url" , "current_url" : layer.vcs_url, "current_id": layer.id }), content_type = "application/json") + + + layer_version, version_created = Layer_Version.objects.get_or_create(layer_source=layer_source, layer=layer, project=prj, up_branch_id=up_branch.id,branch=post_data['git_ref'], commit=post_data['git_ref'], dirpath=post_data['dir_path']) + + if layer_version: + if not version_created: + return HttpResponse(jsonfilter({"error": "hint-layer-version-exists", "existing_layer_version": layer_version.id }), content_type = "application/json") + + layer_version.up_date = timezone.now() + layer_version.save() + + # Add the dependencies specified for this new layer + if (post_data.has_key("layer_deps") and + version_created and + len(post_data["layer_deps"]) > 0): + for layer_dep_id in post_data["layer_deps"].split(","): + + layer_dep_obj = Layer_Version.objects.get(pk=layer_dep_id) + LayerVersionDependency.objects.get_or_create(layer_version=layer_version, depends_on=layer_dep_obj) + # Now add them to the project, we could get an execption + # if the project now contains the exact + # dependency already (like modified on another page) + try: + ProjectLayer.objects.get_or_create(layercommit=layer_dep_obj, project=prj) + except: + pass + + + # If an old layer version exists in our project then remove it + for prj_layers in ProjectLayer.objects.filter(project=prj): + dup_layer_v = Layer_Version.objects.filter(id=prj_layers.layercommit_id, layer_id=layer.id) + if len(dup_layer_v) >0 : + prj_layers.delete() + + # finally add the imported layer (version id) to the project + ProjectLayer.objects.create(layercommit=layer_version, project=prj,optional=1) + + else: + # We didn't create a layer version so back out now and clean up. + if layer_created: + layer.delete() + + return HttpResponse(jsonfilter({"error": "Uncaught error: Could not create layer version"}), content_type = "application/json") + + + return HttpResponse(jsonfilter({"error": "ok"}), content_type = "application/json") + + def importlayer(request): template = "importlayer.html" -- cgit 1.2.3-korg