diff --git a/.gitignore b/.gitignore index ab4e3db..682d70a 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ .idea/ .build/ *.iml +config.js node_modules/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..484f294 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,16 @@ +# Pull base image. +FROM mhart/alpine-node-auto:6.2.0 + +ADD ./ /app + +# Define working directory. +WORKDIR /app + +# Installing app +RUN npm install --production + +# Define default command. +CMD ["/bin/sh", "endpoint.sh"] + +# Expose ports. +EXPOSE 3000 \ No newline at end of file diff --git a/config/development.json b/config/development.json index 3a595c7..faff7bd 100644 --- a/config/development.json +++ b/config/development.json @@ -1,12 +1,12 @@ { "port": 1337, "redis": { - "host": "localhost", + "host": "redis", "port": 6379 }, "errorPages": { "404": "errors/404", "not-connected": "errors/not-connected" }, - "development": true + "production": false } diff --git a/config/production.json b/config/production.json index 72bf1a2..054e573 100644 --- a/config/production.json +++ b/config/production.json @@ -1,7 +1,7 @@ { "port": 1337, "redis": { - "host": "localhost", + "host": "redis", "port": 6379 }, "errorPages": { diff --git a/controllers/active.js b/controllers/active.js index c022efe..7b70a52 100644 --- a/controllers/active.js +++ b/controllers/active.js @@ -11,10 +11,12 @@ module.exports = function (app) { redisModel.getStatus("active").done(function(active){ redisModel.getJobsInList(active).done(function(keys){ redisModel.formatKeys(keys).done(function(formattedKeys){ - redisModel.getProgressForKeys(formattedKeys).done(function(keyList){ - redisModel.getStatusCounts().done(function(countObject){ - var model = { keys: keyList, counts: countObject, active: true, type: "Active" }; - dfd.resolve(model); + redisModel.getDataForKeys(formattedKeys).done(function(keyList) { + redisModel.getProgressForKeys(keyList).done(function (keyList) { + redisModel.getStatusCounts().done(function (countObject) { + var model = {keys: keyList, counts: countObject, active: true, type: "Active"}; + dfd.resolve(model); + }); }); }); }); diff --git a/controllers/complete.js b/controllers/complete.js index 86ab320..7c7803e 100644 --- a/controllers/complete.js +++ b/controllers/complete.js @@ -10,10 +10,12 @@ module.exports = function (app) { var dfd = q.defer(); redisModel.getStatus("complete").done(function(completed){ redisModel.getJobsInList(completed).done(function(keys){ - redisModel.formatKeys(keys).done(function(keyList){ - redisModel.getStatusCounts().done(function(countObject){ - var model = { keys: keyList, counts: countObject, complete: true, type: "Complete" }; - dfd.resolve(model); + redisModel.formatKeys(keys).done(function(formattedKeys){ + redisModel.getDataForKeys(formattedKeys).done(function(keyList) { + redisModel.getStatusCounts().done(function(countObject){ + var model = { keys: keyList, counts: countObject, complete: true, type: "Complete" }; + dfd.resolve(model); + }); }); }); }); diff --git a/controllers/delayed.js b/controllers/delayed.js index f7dfc39..afa8e80 100644 --- a/controllers/delayed.js +++ b/controllers/delayed.js @@ -12,9 +12,11 @@ module.exports = function (app) { redisModel.getJobsInList(delayed).done(function(keys){ redisModel.formatKeys(keys).done(function(formattedKeys){ redisModel.getDelayTimeForKeys(formattedKeys).done(function(keyList){ - redisModel.getStatusCounts().done(function(countObject){ - var model = { keys: keyList, counts: countObject, delayed: true, type: "Delayed" }; - dfd.resolve(model); + redisModel.getDataForKeys(keyList).done(function(keyList) { + redisModel.getStatusCounts().done(function (countObject) { + var model = {keys: keyList, counts: countObject, delayed: true, type: "Delayed"}; + dfd.resolve(model); + }); }); }); }); diff --git a/controllers/failed.js b/controllers/failed.js index 061d981..1e2efe3 100644 --- a/controllers/failed.js +++ b/controllers/failed.js @@ -10,10 +10,12 @@ module.exports = function (app) { var dfd = q.defer(); redisModel.getStatus("failed").done(function(failed){ redisModel.getJobsInList(failed).done(function(keys){ - redisModel.formatKeys(keys).done(function(keyList){ - redisModel.getStatusCounts().done(function(countObject){ - var model = { keys: keyList, counts: countObject, failed: true, type: "Failed"}; - dfd.resolve(model); + redisModel.formatKeys(keys).done(function(formattedKeys){ + redisModel.getDataForKeys(formattedKeys).done(function(keyList) { + redisModel.getStatusCounts().done(function (countObject) { + var model = {keys: keyList, counts: countObject, failed: true, type: "Failed"}; + dfd.resolve(model); + }); }); }); }); diff --git a/controllers/pending.js b/controllers/pending.js index 399b522..442d342 100644 --- a/controllers/pending.js +++ b/controllers/pending.js @@ -10,10 +10,12 @@ module.exports = function (app) { var dfd = q.defer(); redisModel.getStatus("wait").done(function(active){ redisModel.getJobsInList(active).done(function(keys){ - redisModel.formatKeys(keys).done(function(keyList){ - redisModel.getStatusCounts().done(function(countObject){ - var model = { keys: keyList, counts: countObject, pending: true, type: "Pending" }; - dfd.resolve(model); + redisModel.formatKeys(keys).done(function(formattedKeys){ + redisModel.getDataForKeys(formattedKeys).done(function(keyList) { + redisModel.getStatusCounts().done(function (countObject) { + var model = {keys: keyList, counts: countObject, pending: true, type: "Pending"}; + dfd.resolve(model); + }); }); }); }); diff --git a/endpoint.sh b/endpoint.sh new file mode 100755 index 0000000..28f2bac --- /dev/null +++ b/endpoint.sh @@ -0,0 +1 @@ +node index.js \ No newline at end of file diff --git a/lib/setupAndMiddleware.js b/lib/setupAndMiddleware.js index bf376c9..10a8cbb 100644 --- a/lib/setupAndMiddleware.js +++ b/lib/setupAndMiddleware.js @@ -1,6 +1,7 @@ var redisAdapter = require('./redisConnector'), express = require('express'); var bodyParser = require('body-parser'); +var morgan = require('morgan'); module.exports = function(app, options){ options = options || {}; @@ -13,6 +14,9 @@ module.exports = function(app, options){ if (!options.redis){ throw new Error('No redis configuration options passed to matador'); } + + app.use(morgan('dev')); + //Connect to redis redisAdapter.connect(options.redis); var redisConnectionEnforcer = require('./enforceConnection')(options); diff --git a/models/redis.js b/models/redis.js index 89300be..46bbcc0 100644 --- a/models/redis.js +++ b/models/redis.js @@ -225,11 +225,12 @@ var formatKeys = function(keys){ explodedKeys[1] = queue; explodedKeys[2] = arr[arr.length-1]; var status = "stuck"; - if(activeJobs.keys[explodedKeys[1]] && activeJobs.keys[explodedKeys[1]].indexOf(explodedKeys[2]) !== -1) status = "active"; - else if(completedJobs.keys[explodedKeys[1]] && completedJobs.keys[explodedKeys[1]].indexOf(explodedKeys[2]) !== -1) status = "complete"; - else if(failedJobs.keys[explodedKeys[1]] && failedJobs.keys[explodedKeys[1]].indexOf(explodedKeys[2]) !== -1) status = "failed"; - else if(pendingJobs.keys[explodedKeys[1]] && pendingJobs.keys[explodedKeys[1]].indexOf(explodedKeys[2]) !== -1) status = "pending"; - else if(delayedJobs.keys[explodedKeys[1]] && delayedJobs.keys[explodedKeys[1]].indexOf(explodedKeys[2]) !== -1) status = "delayed"; + console.log(activeJobs, explodedKeys[1], typeof activeJobs.keys[explodedKeys[1]]); + if(activeJobs.keys[explodedKeys[1]] && typeof activeJobs.keys[explodedKeys[1]].indexOf === "function" && activeJobs.keys[explodedKeys[1]].indexOf(explodedKeys[2]) !== -1) status = "active"; + else if(completedJobs.keys[explodedKeys[1]] && typeof completedJobs.keys[explodedKeys[1]].indexOf === "function" && completedJobs.keys[explodedKeys[1]].indexOf(explodedKeys[2]) !== -1) status = "complete"; + else if(failedJobs.keys[explodedKeys[1]] && typeof failedJobs.keys[explodedKeys[1]].indexOf === "function" && failedJobs.keys[explodedKeys[1]].indexOf(explodedKeys[2]) !== -1) status = "failed"; + else if(pendingJobs.keys[explodedKeys[1]] && typeof pendingJobs.keys[explodedKeys[1]].indexOf === "function" && pendingJobs.keys[explodedKeys[1]].indexOf(explodedKeys[2]) !== -1) status = "pending"; + else if(delayedJobs.keys[explodedKeys[1]] && typeof delayedJobs.keys[explodedKeys[1]].indexOf === "function" && delayedJobs.keys[explodedKeys[1]].indexOf(explodedKeys[2]) !== -1) status = "delayed"; keyList.push({id: explodedKeys[2], type: explodedKeys[1], status: status}); } @@ -413,6 +414,36 @@ var getProgressForKeys = function(keys){ return dfd.promise; }; +// var dfd = q.defer(); +// if(!id) dfd.resolve({success: false, message: "There was no ID provided."}); +// if(!type) dfd.resolve({success: false, message: "There was no type provided."}); +// +// var firstPartOfKey = "bull:"+type+":"; +// var multi = []; +// redis.hgetall(firstPartOfKey+id, function(err, data){ +// if(err){ +// dfd.resolve({success: false, message: err}); +// }else{ +// dfd.resolve({success: true, message: data}); +// } +// }); +// return dfd.promise; + +var getDataForKeys = function(keys){ + var dfd = q.defer(); + var multi = []; + for(var i = 0, ii = keys.length; i < ii; i++){ + multi.push(["hgetall", "bull:"+keys[i].type+":"+keys[i].id]); + } + redis.multi(multi).exec(function(err, results){ + for(var i = 0, ii = keys.length; i < ii; i++){ + keys[i].data = results[i] ? results[i].data : null; + } + dfd.resolve(keys); + }); + return dfd.promise; +}; + var getDelayTimeForKeys = function(keys){ var dfd = q.defer(); var multi = []; @@ -482,5 +513,6 @@ module.exports.makePendingById = makePendingById; //Makes a job with a specific module.exports.deleteJobByStatus = deleteJobByStatus; //Deletes all jobs in a specific status module.exports.deleteJobById = deleteJobById; //Deletes a job by ID. Requires type as the first parameter and ID as the second. module.exports.getProgressForKeys = getProgressForKeys; //Gets the progress for the keys passed in +module.exports.getDataForKeys = getDataForKeys; //Gets the progress for the keys passed in module.exports.getDelayTimeForKeys = getDelayTimeForKeys; // Gets the delay end time for the keys passed in module.exports.getQueues = getQueues //Get information about all the queues in the redis instance diff --git a/package.json b/package.json index c4701f2..ab2d6de 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "express": "^4.13.4", "less": "~2.1.2", "lodash": "~2.4.1", + "morgan": "^1.7.0", "q": "~1.1.2", "redis": "^2.5.3" }, diff --git a/public/js/DataModel.js b/public/js/DataModel.js index 0c91d4e..6441417 100644 --- a/public/js/DataModel.js +++ b/public/js/DataModel.js @@ -12,6 +12,8 @@ var DataModel = function(){ _self.keys = ko.observableArray([]); _self.memory = ko.observable({}); _self.peakMemory = ko.observable(""); + _self.refreshTimeout = ko.observable(0); + _self.fetchingUpdates = ko.observable(false); _self.autoRefreshId = null; _self.fn = { @@ -19,7 +21,9 @@ var DataModel = function(){ var pathname = window.location.pathname.replace(window.basepath, ''); var refreshUrl = window.basepath + '/api' + (pathname !== '/' ? pathname : ''); + var paused = true; var refresh = function(){ + _self.fetchingUpdates(true); $.getJSON(refreshUrl).done(function(data){ _self.complete(" ("+data.counts.complete+")"); _self.failed(" ("+data.counts.failed+")"); @@ -32,13 +36,33 @@ var DataModel = function(){ _self.memory(data.memory.usage); _self.peakMemory(data.memory.peak.human); } + _self.fetchingUpdates(false); + paused = false; }); }; + + var counter = 0; + var refreshTimer = function(pollInterval) { + if (paused) return; + + if (counter % pollInterval === 0) { + paused = true; + _self.refreshTimeout(1); + refresh(); + } else { + _self.refreshTimeout(((counter % pollInterval) / pollInterval)); + } + + counter++; + }; + if(force){ clearInterval(_self.autoRefreshId); - refresh(); + paused = false; + refreshTimer(0); } - _self.autoRefreshId = setInterval(refresh, 2500); + + _self.autoRefreshId = setInterval(refreshTimer, 100, 20); } } diff --git a/public/js/RedisHandler.js b/public/js/RedisHandler.js index affd583..8a85c1a 100644 --- a/public/js/RedisHandler.js +++ b/public/js/RedisHandler.js @@ -23,7 +23,7 @@ var RedisHandler = function(){ var type = o.type; _self.util.notyConfirm("Are you sure you want to delete the job of type "+ type + " with ID #"+id+"?", function(){ _self.util.blockUI(); - $.getJSON(window.basepath + "/api/jobs/delete/id/"+type+"/"+id).done(function(response){ + $.getJSON(window.basepath + "/api/jobs/delete/id/"+ encodeURIComponent(type) +"/"+id).done(function(response){ _self.util.handleAjaxResponse(response); dataModel.fn.refreshViewModel(true); }).always(function(){ @@ -32,6 +32,7 @@ var RedisHandler = function(){ }); }, deleteByStatus: function(status, obj){ + obj = obj || {}; var queueName = obj.name; status = status.toLowerCase(); var statusDisplay = status; @@ -63,7 +64,7 @@ var RedisHandler = function(){ var type = o.type; _self.util.notyConfirm("Are you sure you want make the job of type "+ type + " with ID #"+id+" pending? This will put this job in the queue to be run again.", function(){ _self.util.blockUI(); - $.getJSON(window.basepath + "/api/jobs/pending/id/"+type+"/"+id).done(function(response){ + $.getJSON(window.basepath + "/api/jobs/pending/id/"+encodeURIComponent(type)+"/"+id).done(function(response){ _self.util.handleAjaxResponse(response); dataModel.fn.refreshViewModel(true); }).always(function(){ @@ -75,7 +76,7 @@ var RedisHandler = function(){ var id = o.id; var type = o.type; _self.util.blockUI(); - $.getJSON(window.basepath + "/api/jobs/info/"+type+"/"+id).done(function(response){ + $.getJSON(window.basepath + "/api/jobs/info/"+encodeURIComponent(type)+"/"+id).done(function(response){ if(response.success === false){ _self.util.handleAjaxResponse(response); }else{ @@ -87,11 +88,19 @@ var RedisHandler = function(){ response.message.data = JSON.parse(response.message.data); var data = JSON.stringify(response.message.data, null, 2); var stacktrace = response.message.stacktrace; + var returnvalue = response.message.returnvalue; + + try { + returnvalue = JSON.stringify(JSON.parse(returnvalue), null, 2); + } catch (e) { + console.warn('Cannot prettify returnvalue', e); + } var message = '
Job ID: ' + id +
                         '\nType: ' + type +
                         '\nStatus: ' + o.status +
-                        '\n\nData: ' + data;
+                        '\n\nData: ' + data +
+                        '\n\nReturn value: ' + returnvalue;
 
                     if (stacktrace){
                         message = message + '\n\nStack Trace: \n' + stacktrace + '' + '
'; diff --git a/public/templates/jobList.dust b/public/templates/jobList.dust index fa9d528..3a00bb3 100644 --- a/public/templates/jobList.dust +++ b/public/templates/jobList.dust @@ -30,7 +30,10 @@ Matador - + +
+
+ diff --git a/public/templates/layouts/master.dust b/public/templates/layouts/master.dust index d005e49..200db3b 100644 --- a/public/templates/layouts/master.dust +++ b/public/templates/layouts/master.dust @@ -20,7 +20,8 @@ - + + @@ -73,6 +74,10 @@
  • Delayed
  • Queues
  • New Job
    • +
  • + +
    Fetching updates...
    +
    • diff --git a/public/templates/newJob.dust b/public/templates/newJob.dust index 1854336..56b1bce 100644 --- a/public/templates/newJob.dust +++ b/public/templates/newJob.dust @@ -15,7 +15,7 @@ Matador

    - +