Finalise the join in a network and add the ability to quit it

This commit is contained in:
Chocobozzz 2016-01-23 18:31:58 +01:00
parent 2cc8ebf134
commit 45239549bf
14 changed files with 444 additions and 101 deletions

View file

@ -21,11 +21,12 @@ Thanks to [webtorrent](https://github.com/feross/webtorrent), we can make P2P (t
- [ ] Frontend
- [X] Simple frontend (All elements are generated by jQuery)
- [ ] AngularJS frontend
- [ ] Join a network
- [X] Join a network
- [X] Generate a RSA key
- [X] Ask for the friend list of other pods and make friend with them
- [ ] Get the list of the videos owned by a pod when making friend with it
- [ ] Post the list of its own videos when making friend with another pod
- [X] Get the list of the videos owned by a pod when making friend with it
- [X] Post the list of its own videos when making friend with another pod
- [X] Quit a network
- [X] Upload a video
- [X] Seed the video
- [X] Send the meta data to all other friends

View file

@ -28,7 +28,12 @@
PodsDB.findOne({ url: req.body.signature.url }, function (err, pod) {
if (err) {
logger.error('Cannot get signed url in decryptBody.', { error: err })
res.sendStatus(500)
return res.sendStatus(500)
}
if (pod === null) {
logger.error('Unknown pod %s.', req.body.signature.url)
return res.sendStatus(403)
}
logger.debug('Decrypting body from %s.', req.body.signature.url)
@ -43,7 +48,7 @@
delete req.body.key
} else {
logger.error('Signature is not okay in decryptBody for %s.', req.body.signature.url)
res.sendStatus(500)
return res.sendStatus(403)
}
next()

View file

@ -31,6 +31,10 @@
makeFriends()
})
$('#panel_quit_friends').on('click', function () {
quitFriends()
})
$('#search-video').on('keyup', function (e) {
var search = $(this).val()
@ -60,6 +64,17 @@
})
}
function quitFriends () {
$.ajax({
url: '/api/v1/pods/quitfriends',
type: 'GET',
dataType: 'json',
success: function () {
alert('Quit friends!')
}
})
}
function printVideos (videos) {
$content.empty()
@ -72,8 +87,21 @@
var $video_name = $('<span></span>').addClass('video_name').text(video.name)
var $video_pod = $('<span></span>').addClass('video_pod_url').text(video.podUrl)
var $remove = $('<span></span>').addClass('span_action glyphicon glyphicon-remove')
var $header = $('<div></div>').append([ $video_name, $video_pod, $remove ])
var $header = $('<div></div>').append([ $video_name, $video_pod ])
if (video.namePath !== null) {
var $remove = $('<span></span>').addClass('span_action glyphicon glyphicon-remove')
// Remove the video
$remove.on('click', function () {
// TODO
if (!confirm('Are you sure ?')) return
removeVideo(video)
})
$header.append($remove)
}
var $video_description = $('<div></div>').addClass('video_description').text(video.description)
@ -82,14 +110,6 @@
getVideo(video)
})
// Remove the video
$remove.on('click', function () {
// TODO
if (!confirm('Are you sure ?')) return
removeVideo(video)
})
if (!video.magnetUri) {
$remove.css('display', 'none')
}

View file

@ -6,6 +6,7 @@
var middleware = require('../../../middlewares')
var miscMiddleware = middleware.misc
var reqValidator = middleware.reqValidators.pods
var secureRequest = middleware.reqValidators.remote.secureRequest
var pods = require('../../../src/pods')
function listPods (req, res, next) {
@ -24,8 +25,33 @@
})
}
function removePods (req, res, next) {
pods.remove(req.body.signature.url, function (err) {
if (err) return next(err)
res.sendStatus(204)
})
}
function makeFriends (req, res, next) {
pods.makeFriends(function (err) {
pods.hasFriends(function (err, has_friends) {
if (err) return next(err)
if (has_friends === true) {
// We need to quit our friends before make new ones
res.sendStatus(409)
} else {
pods.makeFriends(function (err) {
if (err) return next(err)
res.sendStatus(204)
})
}
})
}
function quitFriends (req, res, next) {
pods.quitFriends(function (err) {
if (err) return next(err)
res.sendStatus(204)
@ -34,7 +60,10 @@
router.get('/', miscMiddleware.cache(false), listPods)
router.get('/makefriends', miscMiddleware.cache(false), makeFriends)
router.get('/quitfriends', miscMiddleware.cache(false), quitFriends)
router.post('/', reqValidator.podsAdd, miscMiddleware.cache(false), addPods)
// Post because this is a secured request
router.post('/remove', secureRequest, miscMiddleware.decryptBody, removePods)
module.exports = router
})()

View file

@ -22,7 +22,7 @@
videos.removeRemotes(req.body.signature.url, pluck(req.body.data, 'magnetUri'), function (err) {
if (err) return next(err)
res.status(204)
res.sendStatus(204)
})
}

View file

@ -43,7 +43,9 @@
}
// { url }
// TODO: check if the pod is not already a friend
pods.add = function (data, callback) {
var videos = require('./videos')
logger.info('Adding pod: %s', data.url)
var params = {
@ -58,13 +60,38 @@
return callback(err)
}
videos.addRemotes(data.videos)
fs.readFile(utils.certDir + 'peertube.pub', 'utf8', function (err, cert) {
if (err) {
logger.error('Cannot read cert file.', { error: err })
return callback(err)
}
return callback(null, { cert: cert })
videos.listOwned(function (err, videos_list) {
if (err) {
logger.error('Cannot get the list of owned videos.', { error: err })
return callback(err)
}
return callback(null, { cert: cert, videos: videos_list })
})
})
})
}
pods.remove = function (url, callback) {
var videos = require('./videos')
logger.info('Removing %s pod.', url)
videos.removeAllRemotesOf(url, function (err) {
if (err) logger.error('Cannot remove all remote videos of %s.', url)
PodsDB.remove({ url: url }, function (err) {
if (err) return callback(err)
logger.info('%s pod removed.', url)
callback(null)
})
})
}
@ -82,6 +109,7 @@
}
pods.makeFriends = function (callback) {
var videos = require('./videos')
var pods_score = {}
logger.info('Make friends!')
@ -137,43 +165,109 @@
}
function makeRequestsToWinningPods (cert, pods_list) {
var data = {
url: http + '://' + host + ':' + port,
publicKey: cert
// Stop pool requests
poolRequests.deactivate()
// Flush pool requests
poolRequests.forceSend()
// Get the list of our videos to send to our new friends
videos.listOwned(function (err, videos_list) {
if (err) throw err
var data = {
url: http + '://' + host + ':' + port,
publicKey: cert,
videos: videos_list
}
utils.makeMultipleRetryRequest(
{ method: 'POST', path: '/api/' + constants.API_VERSION + '/pods/', data: data },
pods_list,
function eachRequest (err, response, body, url, pod, callback_each_request) {
// We add the pod if it responded correctly with its public certificate
if (!err && response.statusCode === 200) {
pods.add({ url: pod.url, publicKey: body.cert, score: constants.FRIEND_BASE_SCORE }, function (err) {
if (err) logger.error('Error with adding %s pod.', pod.url, { error: err })
videos.addRemotes(body.videos, function (err) {
if (err) logger.error('Error with adding videos of pod.', pod.url, { error: err })
logger.debug('Adding remote videos from %s.', pod.url, { videos: body.videos })
return callback_each_request()
})
})
} else {
logger.error('Error with adding %s pod.', pod.url, { error: err || new Error('Status not 200') })
return callback_each_request()
}
},
function endRequests (err) {
// Now we made new friends, we can re activate the pool of requests
poolRequests.activate()
if (err) {
logger.error('There was some errors when we wanted to make friends.', { error: err })
return callback(err)
}
logger.debug('makeRequestsToWinningPods finished.')
return callback(null)
}
)
})
}
}
pods.quitFriends = function (callback) {
// Stop pool requests
poolRequests.deactivate()
// Flush pool requests
poolRequests.forceSend()
PodsDB.find(function (err, pods) {
if (err) return callback(err)
var request = {
method: 'POST',
path: '/api/' + constants.API_VERSION + '/pods/remove',
sign: true,
encrypt: true,
data: {
url: 'me' // Fake data
}
}
utils.makeMultipleRetryRequest(
{ method: 'POST', path: '/api/' + constants.API_VERSION + '/pods/', data: data },
// Announce we quit them
utils.makeMultipleRetryRequest(request, pods, function () {
PodsDB.remove(function (err) {
poolRequests.activate()
pods_list,
if (err) return callback(err)
function eachRequest (err, response, body, url, pod, callback_each_request) {
// We add the pod if it responded correctly with its public certificate
if (!err && response.statusCode === 200) {
pods.add({ url: pod.url, publicKey: body.cert, score: constants.FRIEND_BASE_SCORE }, function (err) {
if (err) {
logger.error('Error with adding %s pod.', pod.url, { error: err })
}
logger.info('Broke friends, so sad :(')
return callback_each_request()
})
} else {
logger.error('Error with adding %s pod.', pod.url, { error: err || new Error('Status not 200') })
return callback_each_request()
}
},
var videos = require('./videos')
videos.removeAllRemotes(function (err) {
if (err) return callback(err)
function endRequests (err) {
if (err) {
logger.error('There was some errors when we wanted to make friends.', { error: err })
return callback(err)
}
logger.info('Removed all remote videos.')
callback(null)
})
})
})
})
}
logger.debug('makeRequestsToWinningPods finished.')
return callback(null)
}
)
}
pods.hasFriends = function (callback) {
PodsDB.count(function (err, count) {
if (err) return callback(err)
var has_friends = (count !== 0)
callback(null, has_friends)
})
}
module.exports = pods

View file

@ -6,9 +6,11 @@
var constants = require('./constants')
var logger = require('./logger')
var database = require('./database')
var pluck = require('lodash-node/compat/collection/pluck')
var PoolRequestsDB = database.PoolRequestsDB
var PodsDB = database.PodsDB
var utils = require('./utils')
var VideosDB = database.VideosDB
var poolRequests = {}
@ -90,11 +92,26 @@
}
function removeBadPods () {
PodsDB.remove({ score: 0 }, function (err, result) {
PodsDB.find({ score: 0 }, { _id: 1, url: 1 }, function (err, pods) {
if (err) throw err
var number_removed = result.result.n
if (number_removed !== 0) logger.info('Removed %d pod.', number_removed)
if (pods.length === 0) return
var urls = pluck(pods, 'url')
var ids = pluck(pods, '_id')
VideosDB.remove({ podUrl: { $in: urls } }, function (err, r) {
if (err) logger.error('Cannot remove videos from a pod that we removing.', { error: err })
var videos_removed = r.result.n
logger.info('Removed %d videos.', videos_removed)
PodsDB.remove({ _id: { $in: ids } }, function (err, r) {
if (err) logger.error('Cannot remove bad pods.', { error: err })
var pods_removed = r.result.n
logger.info('Removed %d pods.', pods_removed)
})
})
})
}
@ -126,9 +143,9 @@
utils.makeMultipleRetryRequest(params, pods, callbackEachPodFinished, callbackAllPodsFinished)
function callbackEachPodFinished (err, response, body, url, pod, callback_each_pod_finished) {
if (err || response.statusCode !== 200) {
if (err || (response.statusCode !== 200 && response.statusCode !== 204)) {
bad_pods.push(pod._id)
logger.error('Error sending secure request to %s pod.', url, { error: err })
logger.error('Error sending secure request to %s pod.', url, { error: err || new Error('Status code not 20x') })
} else {
good_pods.push(pod._id)
}
@ -180,5 +197,10 @@
clearInterval(timer)
}
poolRequests.forceSend = function () {
logger.info('Force pool requests sending.')
makePoolRequests()
}
module.exports = poolRequests
})()

View file

@ -56,7 +56,7 @@
utils.makeMultipleRetryRequest = function (all_data, pods, callbackEach, callback) {
if (!callback) {
callback = callbackEach
callbackEach = function () {}
callbackEach = null
}
var url = http + '://' + host + ':' + port
@ -71,9 +71,13 @@
// Make a request for each pod
async.each(pods, function (pod, callback_each_async) {
function callbackEachRetryRequest (err, response, body, url, pod) {
callbackEach(err, response, body, url, pod, function () {
if (callbackEach !== null) {
callbackEach(err, response, body, url, pod, function () {
callback_each_async()
})
} else {
callback_each_async()
})
}
}
var params = {

View file

@ -43,6 +43,18 @@
})
}
videos.listOwned = function (callback) {
// If namePath is not null this is *our* video
VideosDB.find({ namePath: { $ne: null } }, function (err, videos_list) {
if (err) {
logger.error('Cannot get list of the videos.', { error: err })
return callback(err)
}
return callback(null, videos_list)
})
}
videos.add = function (data, callback) {
var video_file = data.video
var video_data = data.data
@ -131,6 +143,8 @@
// Use the magnet Uri because the _id field is not the same on different servers
videos.removeRemotes = function (fromUrl, magnetUris, callback) {
if (callback === undefined) callback = function () {}
VideosDB.find({ magnetUri: { $in: magnetUris } }, function (err, videos) {
if (err || !videos) {
logger.error('Cannot find the torrent URI of these remote videos.')
@ -155,14 +169,34 @@
return callback(err)
}
logger.info('Removed remote videos from %s.', fromUrl)
callback(null)
})
})
})
}
videos.removeAllRemotes = function (callback) {
VideosDB.remove({ namePath: null }, function (err) {
if (err) return callback(err)
callback(null)
})
}
videos.removeAllRemotesOf = function (fromUrl, callback) {
VideosDB.remove({ podUrl: fromUrl }, function (err) {
if (err) return callback(err)
callback(null)
})
}
// { name, magnetUri, podUrl }
// TODO: avoid doublons
videos.addRemotes = function (videos, callback) {
if (callback === undefined) callback = function () {}
var to_add = []
async.each(videos, function (video, callback_each) {

View file

@ -1,6 +1,7 @@
;(function () {
'use strict'
var async = require('async')
var chai = require('chai')
var expect = chai.expect
@ -10,8 +11,12 @@
var apps = []
var urls = []
function makeFriend (pod_number, callback) {
return utils.makeFriend(urls[pod_number - 1], callback)
function makeFriends (pod_number, callback) {
return utils.makeFriends(urls[pod_number - 1], callback)
}
function quitFriends (pod_number, callback) {
return utils.quitFriends(urls[pod_number - 1], callback)
}
function getFriendsList (pod_number, end) {
@ -26,7 +31,11 @@
return utils.uploadVideo(urls[pod_number - 1], name, description, fixture, callback)
}
beforeEach(function (done) {
function getVideos (pod_number, callback) {
return utils.getVideosList(urls[pod_number - 1], callback)
}
before(function (done) {
this.timeout(30000)
utils.runMultipleServers(6, function (apps_run, urls_run) {
apps = apps_run
@ -35,7 +44,7 @@
})
})
afterEach(function (done) {
after(function (done) {
apps.forEach(function (app) {
process.kill(-app.pid)
})
@ -53,11 +62,11 @@
this.timeout(20000)
// Pod 3 makes friend with the first one
makeFriend(3, function () {
makeFriends(3, function () {
// Pod 4 makes friend with the second one
makeFriend(4, function () {
makeFriends(4, function () {
// Now if the fifth wants to make friends with the third et the first
makeFriend(5, function () {
makeFriends(5, function () {
setTimeout(function () {
// It should have 0 friends
getFriendsList(5, function (err, res) {
@ -73,13 +82,30 @@
})
})
it('Should quit all friends', function (done) {
this.timeout(10000)
quitFriends(1, function () {
quitFriends(2, function () {
async.each([ 1, 2, 3, 4, 5, 6 ], function (i, callback) {
getFriendsList(i, function (err, res) {
if (err) throw err
expect(res.body.length).to.equal(0)
callback()
})
}, function () {
done()
})
})
})
})
it('Should make friends with the pods 1, 2, 3', function (done) {
this.timeout(150000)
// Pods 1, 2, 3 and 4 become friends (yes this is beautiful)
makeFriend(2, function () {
makeFriend(1, function () {
makeFriend(4, function () {
makeFriends(2, function () {
makeFriends(1, function () {
makeFriends(4, function () {
// Kill the server 4
apps[3].kill()
@ -99,7 +125,7 @@
expect(res.body.length).to.equal(3)
// Pod 6 ask pod 1, 2 and 3
makeFriend(6, function () {
makeFriends(6, function () {
getFriendsList(6, function (err, res) {
if (err) throw err
@ -125,5 +151,45 @@
})
})
})
it('Should pod 1 quit friends', function (done) {
this.timeout(25000)
// Upload a video on server 3 for aditionnal tests
uploadVideo(3, function () {
setTimeout(function () {
quitFriends(1, function () {
// Remove pod 1 from pod 2
getVideos(1, function (err, res) {
if (err) throw err
expect(res.body).to.be.an('array')
expect(res.body.length).to.equal(2)
getVideos(2, function (err, res) {
if (err) throw err
expect(res.body).to.be.an('array')
expect(res.body.length).to.equal(3)
done()
})
})
})
}, 15000)
})
})
it('Should make friends between pod 1 and 2 and exchange their videos', function (done) {
this.timeout(20000)
makeFriends(1, function () {
setTimeout(function () {
getVideos(1, function (err, res) {
if (err) throw err
expect(res.body).to.be.an('array')
expect(res.body.length).to.equal(5)
done()
})
}, 5000)
})
})
})
})()

View file

@ -9,6 +9,29 @@
var utils = require('./utils')
describe('Test basic friends', function () {
function testMadeFriends (urls, url_to_test, callback) {
var friends = []
for (var i = 0; i < urls.length; i++) {
if (urls[i] === url_to_test) continue
friends.push(urls[i])
}
utils.getFriendsList(url_to_test, function (err, res) {
if (err) throw err
var result = res.body
var result_urls = [ result[0].url, result[1].url ]
expect(result).to.be.an('array')
expect(result.length).to.equal(2)
expect(result_urls[0]).to.not.equal(result_urls[1])
var error_string = 'Friends url do not correspond for ' + url_to_test
expect(friends).to.contain(result_urls[0], error_string)
expect(friends).to.contain(result_urls[1], error_string)
callback()
})
}
var apps = []
var urls = []
@ -41,29 +64,6 @@
it('Should make friends', function (done) {
this.timeout(10000)
function testMadeFriends (urls, url_to_test, callback) {
var friends = []
for (var i = 0; i < urls.length; i++) {
if (urls[i] === url_to_test) continue
friends.push(urls[i])
}
utils.getFriendsList(url_to_test, function (err, res) {
if (err) throw err
var result = res.body
var result_urls = [ result[0].url, result[1].url ]
expect(result).to.be.an('array')
expect(result.length).to.equal(2)
expect(result_urls[0]).to.not.equal(result_urls[1])
var error_string = 'Friends url do not correspond for ' + url_to_test
expect(friends).to.contain(result_urls[0], error_string)
expect(friends).to.contain(result_urls[1], error_string)
callback()
})
}
var path = '/api/v1/pods/makefriends'
// The second pod make friend with the third
@ -118,8 +118,48 @@
})
})
// TODO
it('Should not be able to make friends again')
it('Should not be allowed to make friend again', function (done) {
utils.makeFriends(urls[1], 409, done)
})
it('Should quit friends of pod 2', function (done) {
utils.quitFriends(urls[1], function () {
utils.getFriendsList(urls[1], function (err, res) {
if (err) throw err
var result = res.body
expect(result).to.be.an('array')
expect(result.length).to.equal(0)
// Other pods shouldn't have pod 2 too
async.each([ urls[0], urls[2] ], function (url, callback) {
utils.getFriendsList(url, function (err, res) {
if (err) throw err
var result = res.body
expect(result).to.be.an('array')
expect(result.length).to.equal(1)
expect(result[0].url).not.to.be.equal(urls[1])
callback()
})
}, function (err) {
if (err) throw err
done()
})
})
})
})
it('Should allow pod 2 to make friend again', function (done) {
utils.makeFriends(urls[1], function () {
async.each(urls, function (url, callback) {
testMadeFriends(urls, url, callback)
}, function (err) {
if (err) throw err
done()
})
})
})
after(function (done) {
apps.forEach(function (app) {

View file

@ -22,12 +22,12 @@
urls = urls_run
// The second pod make friend with the third
utils.makeFriend(urls[1], function (err, res) {
utils.makeFriends(urls[1], function (err, res) {
if (err) throw err
// Wait for the request between pods
setTimeout(function () {
utils.makeFriend(urls[0], function (err, res) {
utils.makeFriends(urls[0], function (err, res) {
if (err) throw err
webtorrent.create({ host: 'client', port: '1' }, function () {

View file

@ -34,9 +34,32 @@
.end(end)
}
function makeFriend (url, callback) {
function makeFriends (url, expected_status, callback) {
if (!callback) {
callback = expected_status
expected_status = 204
}
var path = '/api/v1/pods/makefriends'
// The first pod make friend with the third
request(url)
.get(path)
.set('Accept', 'application/json')
.expect(expected_status)
.end(function (err, res) {
if (err) throw err
// Wait for the request between pods
setTimeout(function () {
callback()
}, 1000)
})
}
function quitFriends (url, callback) {
var path = '/api/v1/pods/quitfriends'
// The first pod make friend with the third
request(url)
.get(path)
@ -152,7 +175,8 @@
flushTests: flushTests,
getFriendsList: getFriendsList,
getVideosList: getVideosList,
makeFriend: makeFriend,
makeFriends: makeFriends,
quitFriends: quitFriends,
removeVideo: removeVideo,
runMultipleServers: runMultipleServers,
runServer: runServer,

View file

@ -1,13 +1,17 @@
menu(class='col-md-2')
div(id='panel_get_videos' class='panel_button')
span(class='glyphicon glyphicon-list')
| Get videos
div(id='panel_upload_video' class='panel_button')
span(class='glyphicon glyphicon-cloud-upload')
| Upload a video
div(id='panel_make_friends' class='panel_button')
span(class='glyphicon glyphicon-magnet')
| Make friends
span(class='glyphicon glyphicon-user')
| Make friends
div(id='panel_quit_friends' class='panel_button')
span(class='glyphicon glyphicon-plane')
| Quit friends