diff --git a/client/src/assets/player/peertube-player-local-storage.ts b/client/src/assets/player/peertube-player-local-storage.ts index f9a2a85fe..c406c669a 100644 --- a/client/src/assets/player/peertube-player-local-storage.ts +++ b/client/src/assets/player/peertube-player-local-storage.ts @@ -1,6 +1,8 @@ import { logger } from '@root-helpers/logger' +import { peertubeLocalStorage, peertubeSessionStorage } from '@root-helpers/peertube-web-storage' +import { randomString } from '@root-helpers/string' -function getStoredVolume () { +export function getStoredVolume () { const value = getLocalStorage('volume') if (value !== null && value !== undefined) { const valueNumber = parseFloat(value) @@ -12,38 +14,38 @@ function getStoredVolume () { return undefined } -function getStoredMute () { +export function getStoredMute () { const value = getLocalStorage('mute') if (value !== null && value !== undefined) return value === 'true' return undefined } -function getStoredTheater () { +export function getStoredTheater () { const value = getLocalStorage('theater-enabled') if (value !== null && value !== undefined) return value === 'true' return false } -function saveVolumeInStore (value: number) { +export function saveVolumeInStore (value: number) { return setLocalStorage('volume', value.toString()) } -function saveMuteInStore (value: boolean) { +export function saveMuteInStore (value: boolean) { return setLocalStorage('mute', value.toString()) } -function saveTheaterInStore (enabled: boolean) { +export function saveTheaterInStore (enabled: boolean) { return setLocalStorage('theater-enabled', enabled.toString()) } -function saveAverageBandwidth (value: number) { +export function saveAverageBandwidth (value: number) { /** used to choose the most fitting resolution */ return setLocalStorage('average-bandwidth', value.toString()) } -function getAverageBandwidthInStore () { +export function getAverageBandwidthInStore () { const value = getLocalStorage('average-bandwidth') if (value !== null && value !== undefined) { const valueNumber = parseInt(value, 10) @@ -57,25 +59,25 @@ function getAverageBandwidthInStore () { // --------------------------------------------------------------------------- -function saveLastSubtitle (language: string) { +export function saveLastSubtitle (language: string) { return setLocalStorage('last-subtitle', language) } -function getStoredLastSubtitle () { +export function getStoredLastSubtitle () { return getLocalStorage('last-subtitle') } -function savePreferredSubtitle (language: string) { +export function savePreferredSubtitle (language: string) { return setLocalStorage('preferred-subtitle', language) } -function getStoredPreferredSubtitle () { +export function getStoredPreferredSubtitle () { return getLocalStorage('preferred-subtitle') } // --------------------------------------------------------------------------- -function saveVideoWatchHistory (videoUUID: string, duration: number) { +export function saveVideoWatchHistory (videoUUID: string, duration: number) { return setLocalStorage(`video-watch-history`, JSON.stringify({ ...getStoredVideoWatchHistory(), @@ -86,7 +88,7 @@ function saveVideoWatchHistory (videoUUID: string, duration: number) { })) } -function getStoredVideoWatchHistory (videoUUID?: string) { +export function getStoredVideoWatchHistory (videoUUID?: string) { let data try { @@ -105,7 +107,7 @@ function getStoredVideoWatchHistory (videoUUID?: string) { return data } -function cleanupVideoWatch () { +export function cleanupVideoWatch () { const data = getStoredVideoWatchHistory() if (!data) return @@ -127,39 +129,36 @@ function cleanupVideoWatch () { // --------------------------------------------------------------------------- -export { - getStoredVolume, - getStoredMute, - getStoredTheater, - saveVolumeInStore, - saveMuteInStore, - saveTheaterInStore, - saveAverageBandwidth, - getAverageBandwidthInStore, - saveLastSubtitle, - getStoredLastSubtitle, - saveVideoWatchHistory, - getStoredVideoWatchHistory, - cleanupVideoWatch, - savePreferredSubtitle, - getStoredPreferredSubtitle +export function getPlayerSessionId () { + const key = 'session-id' + + let sessionId = getSessionStorage(key) + if (sessionId) return sessionId + + sessionId = randomString(32) + setSessionStorage(key, sessionId) + + return sessionId } +// --------------------------------------------------------------------------- +// Private // --------------------------------------------------------------------------- const KEY_PREFIX = 'peertube-videojs-' function getLocalStorage (key: string) { - try { - return localStorage.getItem(KEY_PREFIX + key) - } catch { - return undefined - } + return peertubeLocalStorage.getItem(KEY_PREFIX + key) } function setLocalStorage (key: string, value: string) { - try { - localStorage.setItem(KEY_PREFIX + key, value) - } catch { /* empty */ - } + peertubeLocalStorage.setItem(KEY_PREFIX + key, value) +} + +function getSessionStorage (key: string) { + return peertubeSessionStorage.getItem(KEY_PREFIX + key) +} + +function setSessionStorage (key: string, value: string) { + peertubeSessionStorage.setItem(KEY_PREFIX + key, value) } diff --git a/client/src/assets/player/shared/peertube/peertube-plugin.ts b/client/src/assets/player/shared/peertube/peertube-plugin.ts index 4ee02429f..2bfbf18fe 100644 --- a/client/src/assets/player/shared/peertube/peertube-plugin.ts +++ b/client/src/assets/player/shared/peertube/peertube-plugin.ts @@ -1,10 +1,11 @@ -import debug from 'debug' -import videojs from 'video.js' import { timeToInt } from '@peertube/peertube-core-utils' import { VideoView, VideoViewEvent } from '@peertube/peertube-models' import { logger } from '@root-helpers/logger' import { isIOS, isMobile, isSafari } from '@root-helpers/web-browser' +import debug from 'debug' +import videojs from 'video.js' import { + getPlayerSessionId, getStoredLastSubtitle, getStoredMute, getStoredVolume, @@ -371,7 +372,9 @@ class PeerTubePlugin extends Plugin { if (!this.videoViewUrl()) return Promise.resolve(true) - const body: VideoView = { currentTime, viewEvent } + const sessionId = getPlayerSessionId() + + const body: VideoView = { currentTime, viewEvent, sessionId } const headers = new Headers({ 'Content-type': 'application/json; charset=UTF-8' }) if (this.authorizationHeader()) headers.set('Authorization', this.authorizationHeader()) diff --git a/client/src/root-helpers/string.ts b/client/src/root-helpers/string.ts index f81587494..46439b535 100644 --- a/client/src/root-helpers/string.ts +++ b/client/src/root-helpers/string.ts @@ -1,7 +1,17 @@ -function capitalizeFirstLetter (str: string) { +export function capitalizeFirstLetter (str: string) { return str.charAt(0).toUpperCase() + str.slice(1) } -export { - capitalizeFirstLetter +export function randomString (length: number) { + const chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789' + const charsLength = chars.length + const randomArray = new Uint8Array(length) + + let result = '' + + for (const v of crypto.getRandomValues(randomArray)) { + result += chars[v % charsLength] + } + + return result } diff --git a/config/default.yaml b/config/default.yaml index 689ffa74c..c1cc2b233 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -381,7 +381,16 @@ views: # PeerTube buffers local video views before updating and federating the video local_buffer_update_interval: '30 minutes' - ip_view_expiration: '1 hour' + # How long does it take to count again a view from the same user + view_expiration: '1 hour' + + # Minimum amount of time the viewer has to watch the video before PeerTube adds a view + count_view_after: '10 seconds' + + # Player can send a session id string to track the user + # Since this can be spoofed by users to create fake views, you have the option to disable this feature + # If disabled, PeerTube will use the IP address to track the same user (default behavior before PeerTube 6.1) + trust_viewer_session_id: true # How often the web browser sends "is watching" information to the server # Increase the value or set null to disable it if you plan to have many viewers diff --git a/config/production.yaml.example b/config/production.yaml.example index b5bf8c6a5..1d5bcccd2 100644 --- a/config/production.yaml.example +++ b/config/production.yaml.example @@ -379,7 +379,16 @@ views: # PeerTube buffers local video views before updating and federating the video local_buffer_update_interval: '30 minutes' - ip_view_expiration: '1 hour' + # How long does it take to count again a view from the same user + view_expiration: '1 hour' + + # Minimum amount of time the viewer has to watch the video before PeerTube adds a view + count_view_after: '10 seconds' + + # Player can send a session id string to track the user + # Since this can be spoofed by users to create fake views, you have the option to disable this feature + # If disabled, PeerTube will use the IP address to track the same user (default behavior before PeerTube 6.1) + trust_viewer_session_id: true # How often the web browser sends "is watching" information to the server # Increase the value or set null to disable it if you plan to have many viewers diff --git a/config/test.yaml b/config/test.yaml index 74b4524a0..e3ac19cef 100644 --- a/config/test.yaml +++ b/config/test.yaml @@ -150,7 +150,7 @@ views: max_age: -1 local_buffer_update_interval: '5 seconds' - ip_view_expiration: '1 second' + view_expiration: '1 second' geo_ip: enabled: false diff --git a/packages/core-utils/src/common/random.ts b/packages/core-utils/src/common/random.ts index 705735d09..cb17ae021 100644 --- a/packages/core-utils/src/common/random.ts +++ b/packages/core-utils/src/common/random.ts @@ -1,8 +1,4 @@ // high excluded -function randomInt (low: number, high: number) { +export function randomInt (low: number, high: number) { return Math.floor(Math.random() * (high - low) + low) } - -export { - randomInt -} diff --git a/packages/models/src/videos/video-view.model.ts b/packages/models/src/videos/video-view.model.ts index f61211104..bebcf7b17 100644 --- a/packages/models/src/videos/video-view.model.ts +++ b/packages/models/src/videos/video-view.model.ts @@ -3,4 +3,5 @@ export type VideoViewEvent = 'seek' export interface VideoView { currentTime: number viewEvent?: VideoViewEvent + sessionId?: string } diff --git a/packages/server-commands/src/videos/views-command.ts b/packages/server-commands/src/videos/views-command.ts index 048bd3fda..d4c7e7e79 100644 --- a/packages/server-commands/src/videos/views-command.ts +++ b/packages/server-commands/src/videos/views-command.ts @@ -9,8 +9,9 @@ export class ViewsCommand extends AbstractCommand { currentTime: number viewEvent?: VideoViewEvent xForwardedFor?: string + sessionId?: string }) { - const { id, xForwardedFor, viewEvent, currentTime } = options + const { id, xForwardedFor, viewEvent, currentTime, sessionId } = options const path = '/api/v1/videos/' + id + '/views' return this.postBodyRequest({ @@ -20,7 +21,8 @@ export class ViewsCommand extends AbstractCommand { xForwardedFor, fields: { currentTime, - viewEvent + viewEvent, + sessionId }, implicitToken: false, defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 @@ -30,6 +32,7 @@ export class ViewsCommand extends AbstractCommand { async simulateView (options: OverrideCommandOptions & { id: number | string xForwardedFor?: string + sessionId?: string }) { await this.view({ ...options, currentTime: 0 }) await this.view({ ...options, currentTime: 5 }) @@ -39,6 +42,7 @@ export class ViewsCommand extends AbstractCommand { id: number | string currentTimes: number[] xForwardedFor?: string + sessionId?: string }) { let viewEvent: VideoViewEvent = 'seek' diff --git a/packages/tests/src/api/views/video-views-counter.ts b/packages/tests/src/api/views/video-views-counter.ts index 9b83176dc..504998f17 100644 --- a/packages/tests/src/api/views/video-views-counter.ts +++ b/packages/tests/src/api/views/video-views-counter.ts @@ -1,10 +1,11 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ +import { wait } from '@peertube/peertube-core-utils' +import { buildUUID } from '@peertube/peertube-node-utils' +import { PeerTubeServer, cleanupTests, stopFfmpeg, waitJobs } from '@peertube/peertube-server-commands' +import { prepareViewsServers, prepareViewsVideos, processViewsBuffer } from '@tests/shared/views.js' import { expect } from 'chai' import { FfmpegCommand } from 'fluent-ffmpeg' -import { prepareViewsServers, prepareViewsVideos, processViewsBuffer } from '@tests/shared/views.js' -import { wait } from '@peertube/peertube-core-utils' -import { cleanupTests, PeerTubeServer, stopFfmpeg, waitJobs } from '@peertube/peertube-server-commands' describe('Test video views/viewers counters', function () { let servers: PeerTubeServer[] @@ -21,7 +22,14 @@ describe('Test video views/viewers counters', function () { } } - function runTests () { + function runTests (options: { useSessionId: boolean }) { + + const generateSession = () => { + if (!options.useSessionId) return undefined + + return buildUUID() + } + describe('Test views counter on VOD', function () { let videoUUID: string @@ -35,29 +43,35 @@ describe('Test video views/viewers counters', function () { }) it('Should not view a video if watch time is below the threshold', async function () { - await servers[0].views.simulateViewer({ id: videoUUID, currentTimes: [ 1, 2 ] }) + await servers[0].views.simulateViewer({ id: videoUUID, sessionId: generateSession(), currentTimes: [ 1, 2 ] }) await processViewsBuffer(servers) await checkCounter('views', videoUUID, 0) }) it('Should view a video if watch time is above the threshold', async function () { - await servers[0].views.simulateViewer({ id: videoUUID, currentTimes: [ 1, 4 ] }) + await servers[0].views.simulateViewer({ id: videoUUID, sessionId: generateSession(), currentTimes: [ 1, 4 ] }) await processViewsBuffer(servers) await checkCounter('views', videoUUID, 1) }) - it('Should not view again this video with the same IP', async function () { - await servers[0].views.simulateViewer({ id: videoUUID, xForwardedFor: '0.0.0.1,127.0.0.1', currentTimes: [ 1, 4 ] }) - await servers[0].views.simulateViewer({ id: videoUUID, xForwardedFor: '0.0.0.1,127.0.0.1', currentTimes: [ 1, 4 ] }) + it('Should not view again this video with the same IP/session Id', async function () { + const sessionId = generateSession() + const xForwardedFor = '0.0.0.1,127.0.0.1' + + await servers[0].views.simulateViewer({ id: videoUUID, sessionId, xForwardedFor, currentTimes: [ 1, 4 ] }) + await servers[0].views.simulateViewer({ id: videoUUID, sessionId, xForwardedFor, currentTimes: [ 1, 4 ] }) await processViewsBuffer(servers) await checkCounter('views', videoUUID, 2) }) it('Should view the video from server 2 and send the event', async function () { - await servers[1].views.simulateViewer({ id: videoUUID, currentTimes: [ 1, 4 ] }) + const sessionId = generateSession() + + await servers[1].views.simulateViewer({ id: videoUUID, sessionId, currentTimes: [ 1, 4 ] }) + await waitJobs(servers) await processViewsBuffer(servers) @@ -87,19 +101,28 @@ describe('Test video views/viewers counters', function () { it('Should view twice and display 1 view/viewer', async function () { this.timeout(30000) - for (let i = 0; i < 3; i++) { - await servers[0].views.simulateViewer({ id: liveVideoId, currentTimes: [ 0, 35 ] }) - await servers[0].views.simulateViewer({ id: liveVideoId, currentTimes: [ 0, 35 ] }) - await servers[0].views.simulateViewer({ id: vodVideoId, currentTimes: [ 0, 5 ] }) - await servers[0].views.simulateViewer({ id: vodVideoId, currentTimes: [ 0, 5 ] }) + const sessionId = generateSession() - await wait(1000) + for (let i = 0; i < 3; i++) { + await servers[0].views.simulateViewer({ id: liveVideoId, sessionId, currentTimes: [ 0, 35 ] }) + await servers[0].views.simulateViewer({ id: liveVideoId, sessionId, currentTimes: [ 0, 35 ] }) + await servers[0].views.simulateViewer({ id: vodVideoId, sessionId, currentTimes: [ 0, 5 ] }) + await servers[0].views.simulateViewer({ id: vodVideoId, sessionId, currentTimes: [ 0, 5 ] }) } - await waitJobs(servers) + let doWhile = true + while (doWhile) { + try { + await checkCounter('viewers', liveVideoId, 1) + await checkCounter('viewers', vodVideoId, 1) - await checkCounter('viewers', liveVideoId, 1) - await checkCounter('viewers', vodVideoId, 1) + doWhile = false + } catch { + await wait(1000) + + doWhile = true + } + } await processViewsBuffer(servers) @@ -121,7 +144,7 @@ describe('Test video views/viewers counters', function () { await checkCounter('viewers', vodVideoId, 0) error = false - await wait(2500) + await wait(1000) } catch { error = true } @@ -131,21 +154,42 @@ describe('Test video views/viewers counters', function () { it('Should view on a remote and on local and display appropriate views/viewers', async function () { this.timeout(30000) - await servers[0].views.simulateViewer({ id: vodVideoId, xForwardedFor: '0.0.0.1,127.0.0.1', currentTimes: [ 0, 5 ] }) - await servers[0].views.simulateViewer({ id: vodVideoId, xForwardedFor: '0.0.0.1,127.0.0.1', currentTimes: [ 0, 5 ] }) - await servers[0].views.simulateViewer({ id: vodVideoId, currentTimes: [ 0, 5 ] }) - await servers[1].views.simulateViewer({ id: vodVideoId, currentTimes: [ 0, 5 ] }) - await servers[1].views.simulateViewer({ id: vodVideoId, currentTimes: [ 0, 5 ] }) + const xForwardedFor = '0.0.0.1,127.0.0.1' + const sessionId = generateSession() + const xForwardedFor2 = '0.0.0.2,127.0.0.1' + const sessionId2 = generateSession() - await servers[0].views.simulateViewer({ id: liveVideoId, currentTimes: [ 0, 35 ] }) - await servers[1].views.simulateViewer({ id: liveVideoId, currentTimes: [ 0, 35 ] }) - await servers[1].views.simulateViewer({ id: liveVideoId, currentTimes: [ 0, 35 ] }) + { + const currentTimes = [ 0, 5 ] - await wait(3000) // Throttled federation - await waitJobs(servers) + await servers[0].views.simulateViewer({ id: vodVideoId, xForwardedFor, sessionId, currentTimes }) + await servers[0].views.simulateViewer({ id: vodVideoId, xForwardedFor, sessionId, currentTimes }) + await servers[0].views.simulateViewer({ id: vodVideoId, xForwardedFor: xForwardedFor2, sessionId: sessionId2, currentTimes }) + await servers[1].views.simulateViewer({ id: vodVideoId, xForwardedFor, sessionId, currentTimes }) + await servers[1].views.simulateViewer({ id: vodVideoId, xForwardedFor, sessionId, currentTimes }) + } - await checkCounter('viewers', liveVideoId, 2) - await checkCounter('viewers', vodVideoId, 3) + { + const currentTimes = [ 0, 35 ] + + await servers[0].views.simulateViewer({ id: liveVideoId, xForwardedFor: xForwardedFor2, sessionId: sessionId2, currentTimes }) + await servers[1].views.simulateViewer({ id: liveVideoId, xForwardedFor, sessionId, currentTimes }) + await servers[1].views.simulateViewer({ id: liveVideoId, xForwardedFor, sessionId, currentTimes }) + } + + let doWhile = true + while (doWhile) { + try { + await checkCounter('viewers', liveVideoId, 2) + await checkCounter('viewers', vodVideoId, 3) + + doWhile = false + } catch { + await wait(1000) + + doWhile = true + } + } await processViewsBuffer(servers) @@ -167,7 +211,13 @@ describe('Test video views/viewers counters', function () { servers = await prepareViewsServers({ viewExpiration: '5 seconds', viewersFederationV2: false }) }) - runTests() + describe('Not using session id', function () { + runTests({ useSessionId: false }) + }) + + describe('Using session id', function () { + runTests({ useSessionId: true }) + }) after(async function () { await cleanupTests(servers) @@ -182,10 +232,74 @@ describe('Test video views/viewers counters', function () { servers = await prepareViewsServers({ viewExpiration: '5 seconds', viewersFederationV2: true }) }) - runTests() + describe('Not using session id', function () { + runTests({ useSessionId: false }) + }) + + describe('Using session id', function () { + runTests({ useSessionId: true }) + }) + + describe('View minimum duration config', function () { + + it('Should update "count_view_after" config', async function () { + this.timeout(120000) + + const { uuid } = await servers[0].videos.quickUpload({ name: 'video' }) + + { + await servers[0].views.simulateViewer({ id: uuid, sessionId: buildUUID(), currentTimes: [ 1, 2 ] }) + await processViewsBuffer(servers) + + await checkCounter('views', uuid, 0) + } + + await servers[0].kill() + await servers[0].run({ views: { videos: { count_view_after: '1 second' } } }) + + { + await servers[0].views.simulateViewer({ id: uuid, sessionId: buildUUID(), currentTimes: [ 1, 2 ] }) + await processViewsBuffer(servers) + + await checkCounter('views', uuid, 1) + } + }) + }) after(async function () { await cleanupTests(servers) }) }) + + describe('Disabling session id trusting', function () { + let videoUUID: string + + before(async function () { + this.timeout(120000) + + servers = await prepareViewsServers({ viewExpiration: '5 seconds', viewersFederationV2: true, trustViewerSessionId: false }); + + ({ uuid: videoUUID } = await servers[0].videos.quickUpload({ name: 'video' })) + await waitJobs(servers) + }) + + it('Should not take into account session id if the server does not trust it', async function () { + await servers[0].views.simulateViewer({ id: videoUUID, sessionId: buildUUID(), currentTimes: [ 1, 4 ] }) + await servers[0].views.simulateViewer({ id: videoUUID, sessionId: buildUUID(), currentTimes: [ 1, 4 ] }) + + await processViewsBuffer(servers) + await checkCounter('views', videoUUID, 1) + + const xForwardedFor = '0.0.0.1,127.0.0.1' + await servers[0].views.simulateViewer({ id: videoUUID, xForwardedFor, sessionId: buildUUID(), currentTimes: [ 1, 4 ] }) + + await processViewsBuffer(servers) + await checkCounter('views', videoUUID, 2) + }) + + after(async function () { + await cleanupTests(servers) + }) + }) + }) diff --git a/packages/tests/src/api/views/video-views-overall-stats.ts b/packages/tests/src/api/views/video-views-overall-stats.ts index 5c947bbb9..e284ff4ea 100644 --- a/packages/tests/src/api/views/video-views-overall-stats.ts +++ b/packages/tests/src/api/views/video-views-overall-stats.ts @@ -1,11 +1,12 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ -import { expect } from 'chai' -import { FfmpegCommand } from 'fluent-ffmpeg' -import { prepareViewsServers, prepareViewsVideos, processViewersStats } from '@tests/shared/views.js' -import { cleanupTests, PeerTubeServer, stopFfmpeg, waitJobs } from '@peertube/peertube-server-commands' import { wait } from '@peertube/peertube-core-utils' import { VideoStatsOverall } from '@peertube/peertube-models' +import { buildUUID } from '@peertube/peertube-node-utils' +import { PeerTubeServer, cleanupTests, stopFfmpeg, waitJobs } from '@peertube/peertube-server-commands' +import { prepareViewsServers, prepareViewsVideos, processViewersStats } from '@tests/shared/views.js' +import { expect } from 'chai' +import { FfmpegCommand } from 'fluent-ffmpeg' /** * @@ -16,39 +17,43 @@ import { VideoStatsOverall } from '@peertube/peertube-models' * * user3 started and ended in the interval * * user4 started and ended after end date */ -async function simulateComplexViewers (servers: PeerTubeServer[], videoUUID: string) { +async function simulateComplexViewers (servers: PeerTubeServer[], videoUUID: string, useSessionId: boolean) { const user0 = '8.8.8.8,127.0.0.1' const user1 = '8.8.8.8,127.0.0.1' const user2 = '8.8.8.9,127.0.0.1' const user3 = '8.8.8.10,127.0.0.1' const user4 = '8.8.8.11,127.0.0.1' - await servers[0].views.view({ id: videoUUID, currentTime: 0, xForwardedFor: user0 }) // User 0 starts + const sessionIdField = useSessionId + ? 'sessionId' + : 'xForwardedFor' + + await servers[0].views.view({ id: videoUUID, currentTime: 0, [sessionIdField]: user0 }) // User 0 starts await wait(500) - await servers[0].views.view({ id: videoUUID, currentTime: 0, xForwardedFor: user1 }) // User 1 starts - await servers[0].views.view({ id: videoUUID, currentTime: 2, xForwardedFor: user0 }) // User 0 ends + await servers[0].views.view({ id: videoUUID, currentTime: 0, [sessionIdField]: user1 }) // User 1 starts + await servers[0].views.view({ id: videoUUID, currentTime: 2, [sessionIdField]: user0 }) // User 0 ends await wait(500) const startDate = new Date().toISOString() - await servers[0].views.view({ id: videoUUID, currentTime: 0, xForwardedFor: user2 }) // User 2 starts + await servers[0].views.view({ id: videoUUID, currentTime: 0, [sessionIdField]: user2 }) // User 2 starts await wait(500) - await servers[0].views.view({ id: videoUUID, currentTime: 0, xForwardedFor: user3 }) // User 3 starts + await servers[0].views.view({ id: videoUUID, currentTime: 0, [sessionIdField]: user3 }) // User 3 starts await wait(500) - await servers[0].views.view({ id: videoUUID, currentTime: 4, xForwardedFor: user1 }) // User 1 ends + await servers[0].views.view({ id: videoUUID, currentTime: 4, [sessionIdField]: user1 }) // User 1 ends await wait(500) - await servers[0].views.view({ id: videoUUID, currentTime: 3, xForwardedFor: user3 }) // User 3 ends + await servers[0].views.view({ id: videoUUID, currentTime: 3, [sessionIdField]: user3 }) // User 3 ends await wait(500) const endDate = new Date().toISOString() - await servers[0].views.view({ id: videoUUID, currentTime: 0, xForwardedFor: user4 }) // User 4 starts - await servers[0].views.view({ id: videoUUID, currentTime: 5, xForwardedFor: user2 }) // User 2 ends + await servers[0].views.view({ id: videoUUID, currentTime: 0, [sessionIdField]: user4 }) // User 4 starts + await servers[0].views.view({ id: videoUUID, currentTime: 5, [sessionIdField]: user2 }) // User 2 ends await wait(500) - await servers[0].views.view({ id: videoUUID, currentTime: 1, xForwardedFor: user4 }) // User 4 ends + await servers[0].views.view({ id: videoUUID, currentTime: 1, [sessionIdField]: user4 }) // User 4 ends await processViewersStats(servers) @@ -58,61 +63,101 @@ async function simulateComplexViewers (servers: PeerTubeServer[], videoUUID: str describe('Test views overall stats', function () { let servers: PeerTubeServer[] - before(async function () { - this.timeout(120000) + function runTests (options: { useSessionId: boolean }) { + const { useSessionId } = options - servers = await prepareViewsServers() - }) + const generateSessionId = () => { + if (!options.useSessionId) return undefined - describe('Test watch time stats of local videos on live and VOD', function () { - let vodVideoId: string - let liveVideoId: string - let command: FfmpegCommand + return buildUUID() + } before(async function () { - this.timeout(240000); + this.timeout(120000) - ({ vodVideoId, liveVideoId, ffmpegCommand: command } = await prepareViewsVideos({ servers, live: true, vod: true })) + servers = await prepareViewsServers() }) - it('Should display overall stats of a video with no viewers', async function () { - for (const videoId of [ liveVideoId, vodVideoId ]) { - const stats = await servers[0].videoStats.getOverallStats({ videoId }) - const video = await servers[0].videos.get({ id: videoId }) + describe('Test watch time stats of local videos on live and VOD', function () { + let vodVideoId: string + let liveVideoId: string + let command: FfmpegCommand - expect(video.views).to.equal(0) - expect(stats.averageWatchTime).to.equal(0) - expect(stats.totalWatchTime).to.equal(0) - expect(stats.totalViewers).to.equal(0) - } - }) + before(async function () { + this.timeout(240000); - it('Should display overall stats with 1 viewer below the watch time limit', async function () { - this.timeout(60000) + ({ vodVideoId, liveVideoId, ffmpegCommand: command } = await prepareViewsVideos({ servers, live: true, vod: true })) + }) - for (const videoId of [ liveVideoId, vodVideoId ]) { - await servers[0].views.simulateViewer({ id: videoId, currentTimes: [ 0, 1 ] }) - } + it('Should display overall stats of a video with no viewers', async function () { + for (const videoId of [ liveVideoId, vodVideoId ]) { + const stats = await servers[0].videoStats.getOverallStats({ videoId }) + const video = await servers[0].videos.get({ id: videoId }) - await processViewersStats(servers) + expect(video.views).to.equal(0) + expect(stats.averageWatchTime).to.equal(0) + expect(stats.totalWatchTime).to.equal(0) + expect(stats.totalViewers).to.equal(0) + } + }) - for (const videoId of [ liveVideoId, vodVideoId ]) { - const stats = await servers[0].videoStats.getOverallStats({ videoId }) - const video = await servers[0].videos.get({ id: videoId }) + it('Should display overall stats with 1 viewer below the watch time limit', async function () { + this.timeout(60000) - expect(video.views).to.equal(0) - expect(stats.averageWatchTime).to.equal(1) - expect(stats.totalWatchTime).to.equal(1) - expect(stats.totalViewers).to.equal(1) - } - }) + for (const videoId of [ liveVideoId, vodVideoId ]) { + await servers[0].views.simulateViewer({ id: videoId, sessionId: generateSessionId(), currentTimes: [ 0, 1 ] }) + } - it('Should display overall stats with 2 viewers', async function () { - this.timeout(60000) + await processViewersStats(servers) - { - await servers[0].views.simulateViewer({ id: vodVideoId, currentTimes: [ 0, 3 ] }) - await servers[0].views.simulateViewer({ id: liveVideoId, currentTimes: [ 0, 35, 40 ] }) + for (const videoId of [ liveVideoId, vodVideoId ]) { + const stats = await servers[0].videoStats.getOverallStats({ videoId }) + const video = await servers[0].videos.get({ id: videoId }) + + expect(video.views).to.equal(0) + expect(stats.averageWatchTime).to.equal(1) + expect(stats.totalWatchTime).to.equal(1) + expect(stats.totalViewers).to.equal(1) + } + }) + + it('Should display overall stats with 2 viewers', async function () { + this.timeout(60000) + + { + await servers[0].views.simulateViewer({ id: vodVideoId, sessionId: generateSessionId(), currentTimes: [ 0, 3 ] }) + await servers[0].views.simulateViewer({ id: liveVideoId, sessionId: generateSessionId(), currentTimes: [ 0, 35, 40 ] }) + + await processViewersStats(servers) + + { + const stats = await servers[0].videoStats.getOverallStats({ videoId: vodVideoId }) + const video = await servers[0].videos.get({ id: vodVideoId }) + + expect(video.views).to.equal(1) + expect(stats.averageWatchTime).to.equal(2) + expect(stats.totalWatchTime).to.equal(4) + expect(stats.totalViewers).to.equal(2) + } + + { + const stats = await servers[0].videoStats.getOverallStats({ videoId: liveVideoId }) + const video = await servers[0].videos.get({ id: liveVideoId }) + + expect(video.views).to.equal(1) + expect(stats.averageWatchTime).to.equal(21) + expect(stats.totalWatchTime).to.equal(41) + expect(stats.totalViewers).to.equal(2) + } + } + }) + + it('Should display overall stats with a remote viewer below the watch time limit', async function () { + this.timeout(60000) + + for (const videoId of [ liveVideoId, vodVideoId ]) { + await servers[1].views.simulateViewer({ id: videoId, sessionId: generateSessionId(), currentTimes: [ 0, 2 ] }) + } await processViewersStats(servers) @@ -122,8 +167,8 @@ describe('Test views overall stats', function () { expect(video.views).to.equal(1) expect(stats.averageWatchTime).to.equal(2) - expect(stats.totalWatchTime).to.equal(4) - expect(stats.totalViewers).to.equal(2) + expect(stats.totalWatchTime).to.equal(6) + expect(stats.totalViewers).to.equal(3) } { @@ -131,270 +176,280 @@ describe('Test views overall stats', function () { const video = await servers[0].videos.get({ id: liveVideoId }) expect(video.views).to.equal(1) - expect(stats.averageWatchTime).to.equal(21) - expect(stats.totalWatchTime).to.equal(41) - expect(stats.totalViewers).to.equal(2) + expect(stats.averageWatchTime).to.equal(14) + expect(stats.totalWatchTime).to.equal(43) + expect(stats.totalViewers).to.equal(3) } - } + }) + + it('Should display overall stats with a remote viewer above the watch time limit', async function () { + this.timeout(60000) + + await servers[1].views.simulateViewer({ id: vodVideoId, sessionId: generateSessionId(), currentTimes: [ 0, 5 ] }) + await servers[1].views.simulateViewer({ id: liveVideoId, sessionId: generateSessionId(), currentTimes: [ 0, 45 ] }) + await processViewersStats(servers) + + { + const stats = await servers[0].videoStats.getOverallStats({ videoId: vodVideoId }) + const video = await servers[0].videos.get({ id: vodVideoId }) + + expect(video.views).to.equal(2) + expect(stats.averageWatchTime).to.equal(3) + expect(stats.totalWatchTime).to.equal(11) + expect(stats.totalViewers).to.equal(4) + } + + { + const stats = await servers[0].videoStats.getOverallStats({ videoId: liveVideoId }) + const video = await servers[0].videos.get({ id: liveVideoId }) + + expect(video.views).to.equal(2) + expect(stats.averageWatchTime).to.equal(22) + expect(stats.totalWatchTime).to.equal(88) + expect(stats.totalViewers).to.equal(4) + } + }) + + it('Should filter overall stats by date', async function () { + this.timeout(60000) + + const beforeView = new Date() + + await servers[0].views.simulateViewer({ id: vodVideoId, sessionId: generateSessionId(), currentTimes: [ 0, 3 ] }) + await processViewersStats(servers) + + { + const stats = await servers[0].videoStats.getOverallStats({ videoId: vodVideoId, startDate: beforeView.toISOString() }) + expect(stats.averageWatchTime).to.equal(3) + expect(stats.totalWatchTime).to.equal(3) + expect(stats.totalViewers).to.equal(1) + } + + { + const stats = await servers[0].videoStats.getOverallStats({ videoId: liveVideoId, endDate: beforeView.toISOString() }) + expect(stats.averageWatchTime).to.equal(22) + expect(stats.totalWatchTime).to.equal(88) + expect(stats.totalViewers).to.equal(4) + } + }) + + after(async function () { + await stopFfmpeg(command) + }) }) - it('Should display overall stats with a remote viewer below the watch time limit', async function () { - this.timeout(60000) + describe('Test watchers peak stats of local videos on VOD', function () { + let videoUUID: string + let before2Watchers: Date - for (const videoId of [ liveVideoId, vodVideoId ]) { - await servers[1].views.simulateViewer({ id: videoId, currentTimes: [ 0, 2 ] }) - } + before(async function () { + this.timeout(240000); - await processViewersStats(servers) + ({ vodVideoId: videoUUID } = await prepareViewsVideos({ servers, live: true, vod: true })) + }) - { - const stats = await servers[0].videoStats.getOverallStats({ videoId: vodVideoId }) - const video = await servers[0].videos.get({ id: vodVideoId }) + it('Should not have watchers peak', async function () { + const stats = await servers[0].videoStats.getOverallStats({ videoId: videoUUID }) - expect(video.views).to.equal(1) - expect(stats.averageWatchTime).to.equal(2) - expect(stats.totalWatchTime).to.equal(6) - expect(stats.totalViewers).to.equal(3) - } + expect(stats.viewersPeak).to.equal(0) + expect(stats.viewersPeakDate).to.be.null + }) - { - const stats = await servers[0].videoStats.getOverallStats({ videoId: liveVideoId }) - const video = await servers[0].videos.get({ id: liveVideoId }) + it('Should have watcher peak with 1 watcher', async function () { + this.timeout(60000) - expect(video.views).to.equal(1) - expect(stats.averageWatchTime).to.equal(14) - expect(stats.totalWatchTime).to.equal(43) - expect(stats.totalViewers).to.equal(3) - } + const before = new Date() + await servers[0].views.simulateViewer({ id: videoUUID, sessionId: generateSessionId(), currentTimes: [ 0, 2 ] }) + const after = new Date() + + await processViewersStats(servers) + + const stats = await servers[0].videoStats.getOverallStats({ videoId: videoUUID }) + + expect(stats.viewersPeak).to.equal(1) + expect(new Date(stats.viewersPeakDate)).to.be.above(before).and.below(after) + }) + + it('Should have watcher peak with 2 watchers', async function () { + this.timeout(60000) + + const sessionId = generateSessionId() + + before2Watchers = new Date() + await servers[0].views.view({ id: videoUUID, sessionId, currentTime: 0 }) + await servers[1].views.view({ id: videoUUID, sessionId, currentTime: 0 }) + await servers[0].views.view({ id: videoUUID, sessionId, currentTime: 2 }) + await servers[1].views.view({ id: videoUUID, sessionId, currentTime: 2 }) + const after = new Date() + + await processViewersStats(servers) + + const stats = await servers[0].videoStats.getOverallStats({ videoId: videoUUID }) + + expect(stats.viewersPeak).to.equal(2) + expect(new Date(stats.viewersPeakDate)).to.be.above(before2Watchers).and.below(after) + }) + + it('Should filter peak viewers stats by date', async function () { + { + const stats = await servers[0].videoStats.getOverallStats({ videoId: videoUUID, startDate: new Date().toISOString() }) + expect(stats.viewersPeak).to.equal(0) + expect(stats.viewersPeakDate).to.not.exist + } + + { + const stats = await servers[0].videoStats.getOverallStats({ videoId: videoUUID, endDate: before2Watchers.toISOString() }) + expect(stats.viewersPeak).to.equal(1) + expect(new Date(stats.viewersPeakDate)).to.be.below(before2Watchers) + } + }) + + it('Should complex filter peak viewers by date', async function () { + this.timeout(60000) + + const { startDate, endDate } = await simulateComplexViewers(servers, videoUUID, useSessionId) + + const expectCorrect = (stats: VideoStatsOverall) => { + expect(stats.viewersPeak).to.equal(3) + expect(new Date(stats.viewersPeakDate)).to.be.above(new Date(startDate)).and.below(new Date(endDate)) + } + + expectCorrect(await servers[0].videoStats.getOverallStats({ videoId: videoUUID, startDate, endDate })) + expectCorrect(await servers[0].videoStats.getOverallStats({ videoId: videoUUID, startDate })) + expectCorrect(await servers[0].videoStats.getOverallStats({ videoId: videoUUID, endDate })) + expectCorrect(await servers[0].videoStats.getOverallStats({ videoId: videoUUID })) + }) }) - it('Should display overall stats with a remote viewer above the watch time limit', async function () { - this.timeout(60000) + describe('Test countries/subdivisions', function () { + let videoUUID: string - await servers[1].views.simulateViewer({ id: vodVideoId, currentTimes: [ 0, 5 ] }) - await servers[1].views.simulateViewer({ id: liveVideoId, currentTimes: [ 0, 45 ] }) - await processViewersStats(servers) + it('Should not report countries/subdivisions if geoip is disabled', async function () { + this.timeout(120000) - { - const stats = await servers[0].videoStats.getOverallStats({ videoId: vodVideoId }) - const video = await servers[0].videos.get({ id: vodVideoId }) + const { uuid } = await servers[0].videos.quickUpload({ name: 'video' }) + await waitJobs(servers) - expect(video.views).to.equal(2) - expect(stats.averageWatchTime).to.equal(3) - expect(stats.totalWatchTime).to.equal(11) - expect(stats.totalViewers).to.equal(4) - } + await servers[1].views.view({ id: uuid, sessionId: generateSessionId(), xForwardedFor: '8.8.8.8,127.0.0.1', currentTime: 1 }) - { - const stats = await servers[0].videoStats.getOverallStats({ videoId: liveVideoId }) - const video = await servers[0].videos.get({ id: liveVideoId }) + await processViewersStats(servers) - expect(video.views).to.equal(2) - expect(stats.averageWatchTime).to.equal(22) - expect(stats.totalWatchTime).to.equal(88) - expect(stats.totalViewers).to.equal(4) - } - }) + const stats = await servers[0].videoStats.getOverallStats({ videoId: uuid }) + expect(stats.countries).to.have.lengthOf(0) + expect(stats.subdivisions).to.have.lengthOf(0) + }) - it('Should filter overall stats by date', async function () { - this.timeout(60000) + it('Should not report subdivisions if database URL is not provided in the configuration', async function () { + this.timeout(240000) - const beforeView = new Date() + const { uuid } = await servers[0].videos.quickUpload({ name: 'video without subdivisions' }) + await waitJobs(servers) - await servers[0].views.simulateViewer({ id: vodVideoId, currentTimes: [ 0, 3 ] }) - await processViewersStats(servers) + await Promise.all([ servers[0].kill(), servers[1].kill() ]) - { - const stats = await servers[0].videoStats.getOverallStats({ videoId: vodVideoId, startDate: beforeView.toISOString() }) - expect(stats.averageWatchTime).to.equal(3) - expect(stats.totalWatchTime).to.equal(3) - expect(stats.totalViewers).to.equal(1) - } + const config = { geo_ip: { enabled: true, city: { database_url: '' } } } + await Promise.all([ servers[0].run(config), servers[1].run(config) ]) - { - const stats = await servers[0].videoStats.getOverallStats({ videoId: liveVideoId, endDate: beforeView.toISOString() }) - expect(stats.averageWatchTime).to.equal(22) - expect(stats.totalWatchTime).to.equal(88) - expect(stats.totalViewers).to.equal(4) - } + await servers[0].views.simulateViewer({ + id: uuid, + sessionId: generateSessionId(), + xForwardedFor: '8.8.8.8,127.0.0.1', + currentTimes: [ 1, 2 ] + }) + await servers[1].views.simulateViewer({ + id: uuid, + sessionId: generateSessionId(), + xForwardedFor: '8.8.8.4,127.0.0.1', + currentTimes: [ 3, 4 ] + }) + await servers[1].views.simulateViewer({ + id: uuid, + sessionId: generateSessionId(), + xForwardedFor: '80.67.169.12,127.0.0.1', + currentTimes: [ 2, 3 ] + }) + + await processViewersStats(servers) + + const stats = await servers[0].videoStats.getOverallStats({ videoId: uuid }) + + expect(stats.countries).to.have.lengthOf(2) + expect(stats.subdivisions).to.have.lengthOf(0) + }) + + it('Should report countries/subdivisions if geoip is enabled', async function () { + this.timeout(240000) + + const { uuid } = await servers[0].videos.quickUpload({ name: 'video' }) + videoUUID = uuid + await waitJobs(servers) + + await Promise.all([ + servers[0].kill(), + servers[1].kill() + ]) + + const config = { geo_ip: { enabled: true } } + await Promise.all([ + servers[0].run(config), + servers[1].run(config) + ]) + + await servers[0].views.simulateViewer({ + id: uuid, + sessionId: generateSessionId(), + xForwardedFor: '8.8.8.8,127.0.0.1', + currentTimes: [ 1, 2 ] + }) + await servers[1].views.simulateViewer({ + id: uuid, + sessionId: generateSessionId(), + xForwardedFor: '8.8.8.4,127.0.0.1', + currentTimes: [ 3, 4 ] + }) + await servers[1].views.simulateViewer({ + id: uuid, + sessionId: generateSessionId(), + xForwardedFor: '80.67.169.12,127.0.0.1', + currentTimes: [ 2, 3 ] + }) + + await processViewersStats(servers) + + const stats = await servers[0].videoStats.getOverallStats({ videoId: uuid }) + + expect(stats.countries).to.have.lengthOf(2) + + expect(stats.countries[0].isoCode).to.equal('US') + expect(stats.countries[0].viewers).to.equal(2) + + expect(stats.countries[1].isoCode).to.equal('FR') + expect(stats.countries[1].viewers).to.equal(1) + + expect(stats.subdivisions[0].name).to.equal('California') + expect(stats.subdivisions[0].viewers).to.equal(2) + + expect(stats.subdivisions[1].name).to.equal('Brittany') + expect(stats.subdivisions[1].viewers).to.equal(1) + }) + + it('Should filter countries/subdivisions stats by date', async function () { + const stats = await servers[0].videoStats.getOverallStats({ videoId: videoUUID, startDate: new Date().toISOString() }) + expect(stats.countries).to.have.lengthOf(0) + expect(stats.subdivisions).to.have.lengthOf(0) + }) }) after(async function () { - await stopFfmpeg(command) + await cleanupTests(servers) }) + } + + describe('Not using session id', function () { + runTests({ useSessionId: false }) }) - describe('Test watchers peak stats of local videos on VOD', function () { - let videoUUID: string - let before2Watchers: Date - - before(async function () { - this.timeout(240000); - - ({ vodVideoId: videoUUID } = await prepareViewsVideos({ servers, live: true, vod: true })) - }) - - it('Should not have watchers peak', async function () { - const stats = await servers[0].videoStats.getOverallStats({ videoId: videoUUID }) - - expect(stats.viewersPeak).to.equal(0) - expect(stats.viewersPeakDate).to.be.null - }) - - it('Should have watcher peak with 1 watcher', async function () { - this.timeout(60000) - - const before = new Date() - await servers[0].views.simulateViewer({ id: videoUUID, currentTimes: [ 0, 2 ] }) - const after = new Date() - - await processViewersStats(servers) - - const stats = await servers[0].videoStats.getOverallStats({ videoId: videoUUID }) - - expect(stats.viewersPeak).to.equal(1) - expect(new Date(stats.viewersPeakDate)).to.be.above(before).and.below(after) - }) - - it('Should have watcher peak with 2 watchers', async function () { - this.timeout(60000) - - before2Watchers = new Date() - await servers[0].views.view({ id: videoUUID, currentTime: 0 }) - await servers[1].views.view({ id: videoUUID, currentTime: 0 }) - await servers[0].views.view({ id: videoUUID, currentTime: 2 }) - await servers[1].views.view({ id: videoUUID, currentTime: 2 }) - const after = new Date() - - await processViewersStats(servers) - - const stats = await servers[0].videoStats.getOverallStats({ videoId: videoUUID }) - - expect(stats.viewersPeak).to.equal(2) - expect(new Date(stats.viewersPeakDate)).to.be.above(before2Watchers).and.below(after) - }) - - it('Should filter peak viewers stats by date', async function () { - { - const stats = await servers[0].videoStats.getOverallStats({ videoId: videoUUID, startDate: new Date().toISOString() }) - expect(stats.viewersPeak).to.equal(0) - expect(stats.viewersPeakDate).to.not.exist - } - - { - const stats = await servers[0].videoStats.getOverallStats({ videoId: videoUUID, endDate: before2Watchers.toISOString() }) - expect(stats.viewersPeak).to.equal(1) - expect(new Date(stats.viewersPeakDate)).to.be.below(before2Watchers) - } - }) - - it('Should complex filter peak viewers by date', async function () { - this.timeout(60000) - - const { startDate, endDate } = await simulateComplexViewers(servers, videoUUID) - - const expectCorrect = (stats: VideoStatsOverall) => { - expect(stats.viewersPeak).to.equal(3) - expect(new Date(stats.viewersPeakDate)).to.be.above(new Date(startDate)).and.below(new Date(endDate)) - } - - expectCorrect(await servers[0].videoStats.getOverallStats({ videoId: videoUUID, startDate, endDate })) - expectCorrect(await servers[0].videoStats.getOverallStats({ videoId: videoUUID, startDate })) - expectCorrect(await servers[0].videoStats.getOverallStats({ videoId: videoUUID, endDate })) - expectCorrect(await servers[0].videoStats.getOverallStats({ videoId: videoUUID })) - }) - }) - - describe('Test countries/subdivisions', function () { - let videoUUID: string - - it('Should not report countries/subdivisions if geoip is disabled', async function () { - this.timeout(120000) - - const { uuid } = await servers[0].videos.quickUpload({ name: 'video' }) - await waitJobs(servers) - - await servers[1].views.view({ id: uuid, xForwardedFor: '8.8.8.8,127.0.0.1', currentTime: 1 }) - - await processViewersStats(servers) - - const stats = await servers[0].videoStats.getOverallStats({ videoId: uuid }) - expect(stats.countries).to.have.lengthOf(0) - expect(stats.subdivisions).to.have.lengthOf(0) - }) - - it('Should not report subdivisions if database URL is not provided in the configuration', async function () { - this.timeout(240000) - - const { uuid } = await servers[0].videos.quickUpload({ name: 'video without subdivisions' }) - await waitJobs(servers) - - await Promise.all([ servers[0].kill(), servers[1].kill() ]) - - const config = { geo_ip: { enabled: true, city: { database_url: '' } } } - await Promise.all([ servers[0].run(config), servers[1].run(config) ]) - - await servers[0].views.simulateViewer({ id: uuid, xForwardedFor: '8.8.8.8,127.0.0.1', currentTimes: [ 1, 2 ] }) - await servers[1].views.simulateViewer({ id: uuid, xForwardedFor: '8.8.8.4,127.0.0.1', currentTimes: [ 3, 4 ] }) - await servers[1].views.simulateViewer({ id: uuid, xForwardedFor: '80.67.169.12,127.0.0.1', currentTimes: [ 2, 3 ] }) - - await processViewersStats(servers) - - const stats = await servers[0].videoStats.getOverallStats({ videoId: uuid }) - - expect(stats.countries).to.have.lengthOf(2) - expect(stats.subdivisions).to.have.lengthOf(0) - }) - - it('Should report countries/subdivisions if geoip is enabled', async function () { - this.timeout(240000) - - const { uuid } = await servers[0].videos.quickUpload({ name: 'video' }) - videoUUID = uuid - await waitJobs(servers) - - await Promise.all([ - servers[0].kill(), - servers[1].kill() - ]) - - const config = { geo_ip: { enabled: true } } - await Promise.all([ - servers[0].run(config), - servers[1].run(config) - ]) - - await servers[0].views.simulateViewer({ id: uuid, xForwardedFor: '8.8.8.8,127.0.0.1', currentTimes: [ 1, 2 ] }) - await servers[1].views.simulateViewer({ id: uuid, xForwardedFor: '8.8.8.4,127.0.0.1', currentTimes: [ 3, 4 ] }) - await servers[1].views.simulateViewer({ id: uuid, xForwardedFor: '80.67.169.12,127.0.0.1', currentTimes: [ 2, 3 ] }) - - await processViewersStats(servers) - - const stats = await servers[0].videoStats.getOverallStats({ videoId: uuid }) - - expect(stats.countries).to.have.lengthOf(2) - - expect(stats.countries[0].isoCode).to.equal('US') - expect(stats.countries[0].viewers).to.equal(2) - - expect(stats.countries[1].isoCode).to.equal('FR') - expect(stats.countries[1].viewers).to.equal(1) - - expect(stats.subdivisions[0].name).to.equal('California') - expect(stats.subdivisions[0].viewers).to.equal(2) - - expect(stats.subdivisions[1].name).to.equal('Brittany') - expect(stats.subdivisions[1].viewers).to.equal(1) - }) - - it('Should filter countries/subdivisions stats by date', async function () { - const stats = await servers[0].videoStats.getOverallStats({ videoId: videoUUID, startDate: new Date().toISOString() }) - expect(stats.countries).to.have.lengthOf(0) - expect(stats.subdivisions).to.have.lengthOf(0) - }) - }) - - after(async function () { - await cleanupTests(servers) + describe('Using session id', function () { + runTests({ useSessionId: true }) }) }) diff --git a/packages/tests/src/api/views/video-views-retention-stats.ts b/packages/tests/src/api/views/video-views-retention-stats.ts index 8ffe7c702..d066bf52b 100644 --- a/packages/tests/src/api/views/video-views-retention-stats.ts +++ b/packages/tests/src/api/views/video-views-retention-stats.ts @@ -1,6 +1,7 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ import { wait } from '@peertube/peertube-core-utils' +import { buildUUID } from '@peertube/peertube-node-utils' import { PeerTubeServer, cleanupTests } from '@peertube/peertube-server-commands' import { prepareViewsServers, prepareViewsVideos, processViewersStats } from '@tests/shared/views.js' import { expect } from 'chai' @@ -17,61 +18,76 @@ describe('Test views retention stats', function () { describe('Test retention stats on VOD', function () { let vodVideoId: string - before(async function () { - this.timeout(240000); + function runTests (options: { useSessionId: boolean }) { - ({ vodVideoId } = await prepareViewsVideos({ servers, live: false, vod: true })) + const sessionField = options.useSessionId + ? 'sessionId' + : 'xForwardedFor' + + before(async function () { + this.timeout(240000); + + ({ vodVideoId } = await prepareViewsVideos({ servers, live: false, vod: true })) + }) + + it('Should display empty retention', async function () { + const { data } = await servers[0].videoStats.getRetentionStats({ videoId: vodVideoId }) + expect(data).to.have.lengthOf(6) + + for (let i = 0; i < 6; i++) { + expect(data[i].second).to.equal(i) + expect(data[i].retentionPercent).to.equal(0) + } + }) + + it('Should display appropriate retention metrics', async function () { + await servers[0].views.simulateViewer({ [sessionField]: '127.0.0.2,127.0.0.1', id: vodVideoId, currentTimes: [ 0, 1 ] }) + await servers[0].views.simulateViewer({ [sessionField]: '127.0.0.3,127.0.0.1', id: vodVideoId, currentTimes: [ 1, 3 ] }) + await servers[1].views.simulateViewer({ [sessionField]: '127.0.0.2,127.0.0.1', id: vodVideoId, currentTimes: [ 3, 4 ] }) + await servers[1].views.simulateViewer({ [sessionField]: '127.0.0.3,127.0.0.1', id: vodVideoId, currentTimes: [ 0, 1 ] }) + + // Do not take into account empty section + await servers[1].views.simulateViewer({ [sessionField]: '127.0.0.3,127.0.0.1', id: vodVideoId, currentTimes: [ 5, 5 ] }) + await servers[0].views.simulateViewer({ [sessionField]: '127.0.0.4,127.0.0.1', id: vodVideoId, currentTimes: [ 1, 1 ] }) + await servers[1].views.simulateViewer({ [sessionField]: '127.0.0.4,127.0.0.1', id: vodVideoId, currentTimes: [ 1, 1 ] }) + + await processViewersStats(servers) + + const { data } = await servers[0].videoStats.getRetentionStats({ videoId: vodVideoId }) + expect(data).to.have.lengthOf(6) + + expect(data.map(d => d.retentionPercent)).to.deep.equal([ 50, 75, 25, 50, 25, 0 ]) + }) + + it('Should display appropriate retention metrics after a server restart', async function () { + this.timeout(240000) + + const newVideo = await servers[0].videos.quickUpload({ name: 'video 2' }) + + await servers[0].views.simulateViewer({ [sessionField]: '127.0.0.2,127.0.0.1', id: newVideo.id, currentTimes: [ 0, 1 ] }) + await servers[0].views.simulateViewer({ [sessionField]: '127.0.0.3,127.0.0.1', id: newVideo.id, currentTimes: [ 1, 3 ] }) + + await wait(2500) + + await servers[0].kill() + + await servers[0].run() + + await processViewersStats(servers) + + const { data } = await servers[0].videoStats.getRetentionStats({ videoId: newVideo.id }) + expect(data).to.have.lengthOf(6) + + expect(data.map(d => d.retentionPercent)).to.deep.equal([ 50, 100, 50, 50, 0, 0 ]) + }) + } + + describe('Not using session id', function () { + runTests({ useSessionId: false }) }) - it('Should display empty retention', async function () { - const { data } = await servers[0].videoStats.getRetentionStats({ videoId: vodVideoId }) - expect(data).to.have.lengthOf(6) - - for (let i = 0; i < 6; i++) { - expect(data[i].second).to.equal(i) - expect(data[i].retentionPercent).to.equal(0) - } - }) - - it('Should display appropriate retention metrics', async function () { - await servers[0].views.simulateViewer({ xForwardedFor: '127.0.0.2,127.0.0.1', id: vodVideoId, currentTimes: [ 0, 1 ] }) - await servers[0].views.simulateViewer({ xForwardedFor: '127.0.0.3,127.0.0.1', id: vodVideoId, currentTimes: [ 1, 3 ] }) - await servers[1].views.simulateViewer({ xForwardedFor: '127.0.0.2,127.0.0.1', id: vodVideoId, currentTimes: [ 3, 4 ] }) - await servers[1].views.simulateViewer({ xForwardedFor: '127.0.0.3,127.0.0.1', id: vodVideoId, currentTimes: [ 0, 1 ] }) - - // Do not take into account empty section - await servers[1].views.simulateViewer({ xForwardedFor: '127.0.0.3,127.0.0.1', id: vodVideoId, currentTimes: [ 5, 5 ] }) - await servers[0].views.simulateViewer({ xForwardedFor: '127.0.0.4,127.0.0.1', id: vodVideoId, currentTimes: [ 1, 1 ] }) - await servers[1].views.simulateViewer({ xForwardedFor: '127.0.0.4,127.0.0.1', id: vodVideoId, currentTimes: [ 1, 1 ] }) - - await processViewersStats(servers) - - const { data } = await servers[0].videoStats.getRetentionStats({ videoId: vodVideoId }) - expect(data).to.have.lengthOf(6) - - expect(data.map(d => d.retentionPercent)).to.deep.equal([ 50, 75, 25, 50, 25, 0 ]) - }) - - it('Should display appropriate retention metrics after a server restart', async function () { - this.timeout(240000) - - const newVideo = await servers[0].videos.quickUpload({ name: 'video 2' }) - - await servers[0].views.simulateViewer({ xForwardedFor: '127.0.0.2,127.0.0.1', id: newVideo.id, currentTimes: [ 0, 1 ] }) - await servers[0].views.simulateViewer({ xForwardedFor: '127.0.0.3,127.0.0.1', id: newVideo.id, currentTimes: [ 1, 3 ] }) - - await wait(2500) - - await servers[0].kill() - - await servers[0].run() - - await processViewersStats(servers) - - const { data } = await servers[0].videoStats.getRetentionStats({ videoId: newVideo.id }) - expect(data).to.have.lengthOf(6) - - expect(data.map(d => d.retentionPercent)).to.deep.equal([ 50, 100, 50, 50, 0, 0 ]) + describe('Using session id', function () { + runTests({ useSessionId: true }) }) }) diff --git a/packages/tests/src/api/views/video-views-timeserie-stats.ts b/packages/tests/src/api/views/video-views-timeserie-stats.ts index 44fccb644..90a0dbf2a 100644 --- a/packages/tests/src/api/views/video-views-timeserie-stats.ts +++ b/packages/tests/src/api/views/video-views-timeserie-stats.ts @@ -1,10 +1,11 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ +import { VideoStatsTimeserie, VideoStatsTimeserieMetric } from '@peertube/peertube-models' +import { buildUUID } from '@peertube/peertube-node-utils' +import { PeerTubeServer, cleanupTests, stopFfmpeg } from '@peertube/peertube-server-commands' +import { prepareViewsServers, prepareViewsVideos, processViewersStats } from '@tests/shared/views.js' import { expect } from 'chai' import { FfmpegCommand } from 'fluent-ffmpeg' -import { prepareViewsServers, prepareViewsVideos, processViewersStats } from '@tests/shared/views.js' -import { VideoStatsTimeserie, VideoStatsTimeserieMetric } from '@peertube/peertube-models' -import { cleanupTests, PeerTubeServer, stopFfmpeg } from '@peertube/peertube-server-commands' function buildOneMonthAgo () { const monthAgo = new Date() @@ -80,170 +81,187 @@ describe('Test views timeserie stats', function () { expect(new Date(second.date).getTime() - new Date(first.date).getTime()).to.equal(intervalMs) } - before(async function () { - this.timeout(240000); + function runTests (options: { useSessionId: boolean }) { - ({ vodVideoId, liveVideoId, ffmpegCommand: command } = await prepareViewsVideos({ servers, live: true, vod: true })) - }) + const generateSession = () => { + if (!options.useSessionId) return undefined - it('Should display appropriate viewers metrics', async function () { - for (const videoId of [ vodVideoId, liveVideoId ]) { - await servers[0].views.simulateViewer({ id: videoId, currentTimes: [ 0, 3 ] }) - await servers[1].views.simulateViewer({ id: videoId, currentTimes: [ 0, 5 ] }) + return buildUUID() } - await processViewersStats(servers) + before(async function () { + this.timeout(240000); + + ({ vodVideoId, liveVideoId, ffmpegCommand: command } = await prepareViewsVideos({ servers, live: true, vod: true })) + }) + + it('Should display appropriate viewers metrics', async function () { + for (const videoId of [ vodVideoId, liveVideoId ]) { + await servers[0].views.simulateViewer({ id: videoId, sessionId: generateSession(), currentTimes: [ 0, 3 ] }) + await servers[1].views.simulateViewer({ id: videoId, sessionId: generateSession(), currentTimes: [ 0, 5 ] }) + } + + await processViewersStats(servers) + + for (const videoId of [ vodVideoId, liveVideoId ]) { + const result = await servers[0].videoStats.getTimeserieStats({ + videoId, + startDate: buildOneMonthAgo(), + endDate: new Date(), + metric: 'viewers' + }) + expectTimeserieData(result, 2) + } + }) + + it('Should display appropriate watch time metrics', async function () { + for (const videoId of [ vodVideoId, liveVideoId ]) { + const result = await servers[0].videoStats.getTimeserieStats({ + videoId, + startDate: buildOneMonthAgo(), + endDate: new Date(), + metric: 'aggregateWatchTime' + }) + expectTimeserieData(result, 8) + + await servers[1].views.simulateViewer({ id: videoId, sessionId: generateSession(), currentTimes: [ 0, 1 ] }) + } + + await processViewersStats(servers) + + for (const videoId of [ vodVideoId, liveVideoId ]) { + const result = await servers[0].videoStats.getTimeserieStats({ + videoId, + startDate: buildOneMonthAgo(), + endDate: new Date(), + metric: 'aggregateWatchTime' + }) + expectTimeserieData(result, 9) + } + }) + + it('Should use a custom start/end date', async function () { + const now = new Date() + const twentyDaysAgo = new Date() + twentyDaysAgo.setDate(twentyDaysAgo.getDate() - 19) - for (const videoId of [ vodVideoId, liveVideoId ]) { const result = await servers[0].videoStats.getTimeserieStats({ - videoId, - startDate: buildOneMonthAgo(), - endDate: new Date(), - metric: 'viewers' + videoId: vodVideoId, + metric: 'aggregateWatchTime', + startDate: twentyDaysAgo, + endDate: now }) - expectTimeserieData(result, 2) - } - }) - it('Should display appropriate watch time metrics', async function () { - for (const videoId of [ vodVideoId, liveVideoId ]) { + expect(result.groupInterval).to.equal('1 day') + expect(result.data).to.have.lengthOf(20) + + const first = result.data[0] + expect(new Date(first.date).toLocaleDateString()).to.equal(twentyDaysAgo.toLocaleDateString()) + + expectInterval(result, 24 * 3600 * 1000) + expectTodayLastValue(result, 9) + }) + + it('Should automatically group by months', async function () { + const now = new Date() + const heightYearsAgo = new Date() + heightYearsAgo.setFullYear(heightYearsAgo.getFullYear() - 7) + const result = await servers[0].videoStats.getTimeserieStats({ - videoId, - startDate: buildOneMonthAgo(), - endDate: new Date(), - metric: 'aggregateWatchTime' + videoId: vodVideoId, + metric: 'aggregateWatchTime', + startDate: heightYearsAgo, + endDate: now }) - expectTimeserieData(result, 8) - await servers[1].views.simulateViewer({ id: videoId, currentTimes: [ 0, 1 ] }) - } + expect(result.groupInterval).to.equal('6 months') + expect(result.data).to.have.length.above(10).and.below(200) + }) - await processViewersStats(servers) + it('Should automatically group by days', async function () { + const now = new Date() + const threeMonthsAgo = new Date() + threeMonthsAgo.setMonth(threeMonthsAgo.getMonth() - 3) - for (const videoId of [ vodVideoId, liveVideoId ]) { const result = await servers[0].videoStats.getTimeserieStats({ - videoId, - startDate: buildOneMonthAgo(), - endDate: new Date(), - metric: 'aggregateWatchTime' + videoId: vodVideoId, + metric: 'aggregateWatchTime', + startDate: threeMonthsAgo, + endDate: now }) - expectTimeserieData(result, 9) - } - }) - it('Should use a custom start/end date', async function () { - const now = new Date() - const twentyDaysAgo = new Date() - twentyDaysAgo.setDate(twentyDaysAgo.getDate() - 19) - - const result = await servers[0].videoStats.getTimeserieStats({ - videoId: vodVideoId, - metric: 'aggregateWatchTime', - startDate: twentyDaysAgo, - endDate: now + expect(result.groupInterval).to.equal('2 days') + expect(result.data).to.have.length.above(10).and.below(200) }) - expect(result.groupInterval).to.equal('1 day') - expect(result.data).to.have.lengthOf(20) + it('Should automatically group by hours', async function () { + const now = new Date() + const twoDaysAgo = new Date() + twoDaysAgo.setDate(twoDaysAgo.getDate() - 1) - const first = result.data[0] - expect(new Date(first.date).toLocaleDateString()).to.equal(twentyDaysAgo.toLocaleDateString()) + const result = await servers[0].videoStats.getTimeserieStats({ + videoId: vodVideoId, + metric: 'aggregateWatchTime', + startDate: twoDaysAgo, + endDate: now + }) - expectInterval(result, 24 * 3600 * 1000) - expectTodayLastValue(result, 9) - }) + expect(result.groupInterval).to.equal('1 hour') + expect(result.data).to.have.length.above(24).and.below(50) - it('Should automatically group by months', async function () { - const now = new Date() - const heightYearsAgo = new Date() - heightYearsAgo.setFullYear(heightYearsAgo.getFullYear() - 7) - - const result = await servers[0].videoStats.getTimeserieStats({ - videoId: vodVideoId, - metric: 'aggregateWatchTime', - startDate: heightYearsAgo, - endDate: now + expectInterval(result, 3600 * 1000) + expectTodayLastValue(result, 9) }) - expect(result.groupInterval).to.equal('6 months') - expect(result.data).to.have.length.above(10).and.below(200) - }) + it('Should automatically group by ten minutes', async function () { + const now = new Date() + const twoHoursAgo = new Date() + twoHoursAgo.setHours(twoHoursAgo.getHours() - 4) - it('Should automatically group by days', async function () { - const now = new Date() - const threeMonthsAgo = new Date() - threeMonthsAgo.setMonth(threeMonthsAgo.getMonth() - 3) + const result = await servers[0].videoStats.getTimeserieStats({ + videoId: vodVideoId, + metric: 'aggregateWatchTime', + startDate: twoHoursAgo, + endDate: now + }) - const result = await servers[0].videoStats.getTimeserieStats({ - videoId: vodVideoId, - metric: 'aggregateWatchTime', - startDate: threeMonthsAgo, - endDate: now + expect(result.groupInterval).to.equal('10 minutes') + expect(result.data).to.have.length.above(20).and.below(30) + + expectInterval(result, 60 * 10 * 1000) + expectTodayLastValue(result) }) - expect(result.groupInterval).to.equal('2 days') - expect(result.data).to.have.length.above(10).and.below(200) - }) + it('Should automatically group by one minute', async function () { + const now = new Date() + const thirtyAgo = new Date() + thirtyAgo.setMinutes(thirtyAgo.getMinutes() - 30) - it('Should automatically group by hours', async function () { - const now = new Date() - const twoDaysAgo = new Date() - twoDaysAgo.setDate(twoDaysAgo.getDate() - 1) + const result = await servers[0].videoStats.getTimeserieStats({ + videoId: vodVideoId, + metric: 'aggregateWatchTime', + startDate: thirtyAgo, + endDate: now + }) - const result = await servers[0].videoStats.getTimeserieStats({ - videoId: vodVideoId, - metric: 'aggregateWatchTime', - startDate: twoDaysAgo, - endDate: now + expect(result.groupInterval).to.equal('1 minute') + expect(result.data).to.have.length.above(20).and.below(40) + + expectInterval(result, 60 * 1000) + expectTodayLastValue(result) }) - expect(result.groupInterval).to.equal('1 hour') - expect(result.data).to.have.length.above(24).and.below(50) - - expectInterval(result, 3600 * 1000) - expectTodayLastValue(result, 9) - }) - - it('Should automatically group by ten minutes', async function () { - const now = new Date() - const twoHoursAgo = new Date() - twoHoursAgo.setHours(twoHoursAgo.getHours() - 4) - - const result = await servers[0].videoStats.getTimeserieStats({ - videoId: vodVideoId, - metric: 'aggregateWatchTime', - startDate: twoHoursAgo, - endDate: now + after(async function () { + await stopFfmpeg(command) }) + } - expect(result.groupInterval).to.equal('10 minutes') - expect(result.data).to.have.length.above(20).and.below(30) - - expectInterval(result, 60 * 10 * 1000) - expectTodayLastValue(result) + describe('Not using session id', function () { + runTests({ useSessionId: false }) }) - it('Should automatically group by one minute', async function () { - const now = new Date() - const thirtyAgo = new Date() - thirtyAgo.setMinutes(thirtyAgo.getMinutes() - 30) - - const result = await servers[0].videoStats.getTimeserieStats({ - videoId: vodVideoId, - metric: 'aggregateWatchTime', - startDate: thirtyAgo, - endDate: now - }) - - expect(result.groupInterval).to.equal('1 minute') - expect(result.data).to.have.length.above(20).and.below(40) - - expectInterval(result, 60 * 1000) - expectTodayLastValue(result) - }) - - after(async function () { - await stopFfmpeg(command) + describe('Using session id', function () { + runTests({ useSessionId: true }) }) }) diff --git a/packages/tests/src/api/views/videos-views-cleaner.ts b/packages/tests/src/api/views/videos-views-cleaner.ts index 521dd9b5e..1246b90b4 100644 --- a/packages/tests/src/api/views/videos-views-cleaner.ts +++ b/packages/tests/src/api/views/videos-views-cleaner.ts @@ -1,8 +1,7 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ -import { expect } from 'chai' -import { SQLCommand } from '@tests/shared/sql-command.js' import { wait } from '@peertube/peertube-core-utils' +import { buildUUID } from '@peertube/peertube-node-utils' import { cleanupTests, createMultipleServers, @@ -12,6 +11,8 @@ import { setAccessTokensToServers, waitJobs } from '@peertube/peertube-server-commands' +import { SQLCommand } from '@tests/shared/sql-command.js' +import { expect } from 'chai' describe('Test video views cleaner', function () { let servers: PeerTubeServer[] @@ -33,10 +34,12 @@ describe('Test video views cleaner', function () { await waitJobs(servers) - await servers[0].views.simulateView({ id: videoIdServer1 }) - await servers[1].views.simulateView({ id: videoIdServer1 }) - await servers[0].views.simulateView({ id: videoIdServer2 }) - await servers[1].views.simulateView({ id: videoIdServer2 }) + const sessionId = buildUUID() + + await servers[0].views.simulateView({ id: videoIdServer1, sessionId }) + await servers[1].views.simulateView({ id: videoIdServer1, sessionId }) + await servers[0].views.simulateView({ id: videoIdServer2, sessionId }) + await servers[1].views.simulateView({ id: videoIdServer2, sessionId }) await waitJobs(servers) diff --git a/packages/tests/src/shared/views.ts b/packages/tests/src/shared/views.ts index ef97fa442..5d70d42f6 100644 --- a/packages/tests/src/shared/views.ts +++ b/packages/tests/src/shared/views.ts @@ -33,14 +33,25 @@ async function processViewsBuffer (servers: PeerTubeServer[]) { async function prepareViewsServers (options: { viewersFederationV2?: boolean viewExpiration?: string // default 1 second + trustViewerSessionId?: boolean // default true } = {}) { - const { viewExpiration = '1 second' } = options + const { viewExpiration = '1 second', trustViewerSessionId = true } = options const env = options?.viewersFederationV2 === true ? { USE_VIEWERS_FEDERATION_V2: 'true' } : undefined - const servers = await createMultipleServers(2, { views: { videos: { ip_view_expiration: viewExpiration } } }, { env }) + const config = { + views: { + videos: { + view_expiration: viewExpiration, + trust_viewer_session_id: trustViewerSessionId, + count_view_after: '10 seconds' + } + } + } + + const servers = await createMultipleServers(2, config, { env }) await setAccessTokensToServers(servers) await setDefaultVideoChannel(servers) diff --git a/scripts/simulate-many-viewers.ts b/scripts/simulate-many-viewers.ts index 97ab968b7..f7597789f 100644 --- a/scripts/simulate-many-viewers.ts +++ b/scripts/simulate-many-viewers.ts @@ -59,7 +59,7 @@ async function prepare () { views: { videos: { local_buffer_update_interval: '30 minutes', - ip_view_expiration: '1 hour' + view_expiration: '1 hour' } } } diff --git a/server/core/controllers/api/videos/view.ts b/server/core/controllers/api/videos/view.ts index cc0534753..fd000f484 100644 --- a/server/core/controllers/api/videos/view.ts +++ b/server/core/controllers/api/videos/view.ts @@ -41,7 +41,8 @@ async function viewVideo (req: express.Request, res: express.Response) { video, ip, currentTime: body.currentTime, - viewEvent: body.viewEvent + viewEvent: body.viewEvent, + sessionId: body.sessionId }) if (successView) { diff --git a/server/core/initializers/checker-before-init.ts b/server/core/initializers/checker-before-init.ts index 9e97ca5c1..58926ae32 100644 --- a/server/core/initializers/checker-before-init.ts +++ b/server/core/initializers/checker-before-init.ts @@ -54,7 +54,7 @@ function checkMissedConfig () { 'services.twitter.username', 'followers.instance.enabled', 'followers.instance.manual_approval', 'tracker.enabled', 'tracker.private', 'tracker.reject_too_many_announces', - 'history.videos.max_age', 'views.videos.remote.max_age', 'views.videos.local_buffer_update_interval', 'views.videos.ip_view_expiration', + 'history.videos.max_age', 'views.videos.remote.max_age', 'views.videos.local_buffer_update_interval', 'views.videos.view_expiration', 'views.videos.watching_interval.anonymous', 'views.videos.watching_interval.users', 'rates_limit.api.window', 'rates_limit.api.max', 'rates_limit.login.window', 'rates_limit.login.max', 'rates_limit.signup.window', 'rates_limit.signup.max', 'rates_limit.ask_send_email.window', 'rates_limit.ask_send_email.max', diff --git a/server/core/initializers/config.ts b/server/core/initializers/config.ts index 2e03b5cc2..c6914a7e7 100644 --- a/server/core/initializers/config.ts +++ b/server/core/initializers/config.ts @@ -308,7 +308,9 @@ const CONFIG = { MAX_AGE: parseDurationToMs(config.get('views.videos.remote.max_age')) }, LOCAL_BUFFER_UPDATE_INTERVAL: parseDurationToMs(config.get('views.videos.local_buffer_update_interval')), - IP_VIEW_EXPIRATION: parseDurationToMs(config.get('views.videos.ip_view_expiration')), + VIEW_EXPIRATION: parseDurationToMs(config.get('views.videos.view_expiration')), + COUNT_VIEW_AFTER: parseDurationToMs(config.get('views.videos.count_view_after')), + TRUST_VIEWER_SESSION_ID: config.get('views.videos.trust_viewer_session_id'), WATCHING_INTERVAL: { ANONYMOUS: parseDurationToMs(config.get('views.videos.watching_interval.anonymous')), USERS: parseDurationToMs(config.get('views.videos.watching_interval.users')) diff --git a/server/core/initializers/constants.ts b/server/core/initializers/constants.ts index ab89c4a80..7e8475668 100644 --- a/server/core/initializers/constants.ts +++ b/server/core/initializers/constants.ts @@ -502,7 +502,7 @@ const CONSTRAINTS_FIELDS = { } const VIEW_LIFETIME = { - VIEW: CONFIG.VIEWS.VIDEOS.IP_VIEW_EXPIRATION, + VIEW: CONFIG.VIEWS.VIDEOS.VIEW_EXPIRATION, VIEWER_COUNTER: 60000 * 2, // 2 minutes VIEWER_STATS: 60000 * 60 // 1 hour } diff --git a/server/core/lib/redis.ts b/server/core/lib/redis.ts index 5dcd2b20b..156f20c8d 100644 --- a/server/core/lib/redis.ts +++ b/server/core/lib/redis.ts @@ -176,12 +176,12 @@ class Redis { /* ************ Views per IP ************ */ - setIPVideoView (ip: string, videoUUID: string) { - return this.setValue(this.generateIPViewKey(ip, videoUUID), '1', VIEW_LIFETIME.VIEW) + setSessionIdVideoView (ip: string, videoUUID: string) { + return this.setValue(this.generateSessionIdViewKey(ip, videoUUID), '1', VIEW_LIFETIME.VIEW) } - async doesVideoIPViewExist (ip: string, videoUUID: string) { - return this.exists(this.generateIPViewKey(ip, videoUUID)) + async doesVideoSessionIdViewExist (sessionId: string, videoUUID: string) { + return this.exists(this.generateSessionIdViewKey(sessionId, videoUUID)) } /* ************ Video views stats ************ */ @@ -281,8 +281,8 @@ class Redis { return this.getObject(viewerKey) } - setLocalVideoViewer (ip: string, videoId: number, object: any) { - const { setKey, viewerKey } = this.generateLocalVideoViewerKeys(ip, videoId) + setLocalVideoViewer (sessionId: string, videoId: number, object: any) { + const { setKey, viewerKey } = this.generateLocalVideoViewerKeys(sessionId, videoId) return Promise.all([ this.addToSet(setKey, viewerKey), @@ -338,12 +338,16 @@ class Redis { return { setKey: `local-video-views-buffer`, videoKey: `local-video-views-buffer-${videoId}` } } - generateLocalVideoViewerKeys (ip: string, videoId: number): { setKey: string, viewerKey: string } + generateLocalVideoViewerKeys (sessionId: string, videoId: number): { setKey: string, viewerKey: string } generateLocalVideoViewerKeys (): { setKey: string } - generateLocalVideoViewerKeys (ip?: string, videoId?: number) { - const anonymousIP = sha256(CONFIG.SECRETS + '-' + ip) + generateLocalVideoViewerKeys (sessionId?: string, videoId?: number) { + return { + setKey: `local-video-viewer-stats-keys`, - return { setKey: `local-video-viewer-stats-keys`, viewerKey: `local-video-viewer-stats-${anonymousIP}-${videoId}` } + viewerKey: sessionId && videoId + ? `local-video-viewer-stats-${sessionId}-${videoId}` + : undefined + } } private generateVideoViewStatsKeys (options: { videoId?: number, hour?: number }) { @@ -370,8 +374,8 @@ class Redis { return 'verify-email-registration-' + registrationId } - generateIPViewKey (ip: string, videoUUID: string) { - return `views-${videoUUID}-${sha256(CONFIG.SECRETS.PEERTUBE + '-' + ip)}` + generateSessionIdViewKey (sessionId: string, videoUUID: string) { + return `views-${videoUUID}-${sessionId}` } private generateContactFormKey (ip: string) { diff --git a/server/core/lib/views/shared/video-viewer-counters.ts b/server/core/lib/views/shared/video-viewer-counters.ts index 8a18f94db..9d92b09a4 100644 --- a/server/core/lib/views/shared/video-viewer-counters.ts +++ b/server/core/lib/views/shared/video-viewer-counters.ts @@ -1,4 +1,4 @@ -import { buildUUID, isTestOrDevInstance, isUsingViewersFederationV2, sha256 } from '@peertube/peertube-node-utils' +import { isTestOrDevInstance, isUsingViewersFederationV2 } from '@peertube/peertube-node-utils' import { exists } from '@server/helpers/custom-validators/misc.js' import { logger, loggerTagsFactory } from '@server/helpers/logger.js' import { VIEW_LIFETIME } from '@server/initializers/constants.js' @@ -28,8 +28,6 @@ export class VideoViewerCounters { private readonly viewersPerVideo = new Map() private readonly idToViewer = new Map() - private readonly salt = buildUUID() - private processingViewerCounters = false constructor () { @@ -40,13 +38,13 @@ export class VideoViewerCounters { async addLocalViewer (options: { video: MVideoImmutable - ip: string + sessionId: string }) { - const { video, ip } = options + const { video, sessionId } = options logger.debug('Adding local viewer to video viewers counter %s.', video.uuid, { ...lTags(video.uuid) }) - const viewerId = this.generateViewerId(ip, video.uuid) + const viewerId = sessionId + '-' + video.uuid const viewer = this.idToViewer.get(viewerId) if (viewer) { @@ -217,10 +215,6 @@ export class VideoViewerCounters { logger.debug('Video viewers update for %s is %d.', video.url, totalViewers, lTags()) } - private generateViewerId (ip: string, videoUUID: string) { - return sha256(this.salt + '-' + ip + '-' + videoUUID) - } - private async federateViewerIfNeeded (video: MVideoImmutable, viewer: Viewer) { // Federate the viewer if it's been a "long" time we did not const now = new Date().getTime() diff --git a/server/core/lib/views/shared/video-viewer-stats.ts b/server/core/lib/views/shared/video-viewer-stats.ts index cee804ce8..a5cbf1b9b 100644 --- a/server/core/lib/views/shared/video-viewer-stats.ts +++ b/server/core/lib/views/shared/video-viewer-stats.ts @@ -1,4 +1,3 @@ -import { Transaction } from 'sequelize' import { VideoViewEvent } from '@peertube/peertube-models' import { isTestOrDevInstance } from '@peertube/peertube-node-utils' import { GeoIP } from '@server/helpers/geo-ip.js' @@ -12,6 +11,7 @@ import { VideoModel } from '@server/models/video/video.js' import { LocalVideoViewerWatchSectionModel } from '@server/models/view/local-video-viewer-watch-section.js' import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer.js' import { MVideo, MVideoImmutable } from '@server/types/models/index.js' +import { Transaction } from 'sequelize' const lTags = loggerTagsFactory('views') @@ -37,7 +37,7 @@ export class VideoViewerStats { private processingRedisWrites = false private readonly viewerCache = new Map() - private readonly redisPendingWrites = new Map() + private readonly redisPendingWrites = new Map() constructor () { setInterval(() => this.processViewerStats(), VIEW_LIFETIME.VIEWER_STATS) @@ -50,35 +50,19 @@ export class VideoViewerStats { video: MVideoImmutable currentTime: number ip: string + sessionId: string viewEvent?: VideoViewEvent }) { - const { video, ip, viewEvent, currentTime } = options + const { video, ip, viewEvent, currentTime, sessionId } = options - logger.debug('Adding local viewer to video stats %s.', video.uuid, { currentTime, viewEvent, ...lTags(video.uuid) }) + logger.debug( + 'Adding local viewer to video stats %s.', video.uuid, + { currentTime, viewEvent, sessionId, ...lTags(video.uuid) } + ) - return this.updateLocalViewerStats({ video, viewEvent, currentTime, ip }) - } - - // --------------------------------------------------------------------------- - - async getWatchTime (videoId: number, ip: string) { - const stats: LocalViewerStats = await this.getLocalVideoViewerByIP({ ip, videoId }) - - return stats?.watchTime || 0 - } - - // --------------------------------------------------------------------------- - - private async updateLocalViewerStats (options: { - video: MVideoImmutable - ip: string - currentTime: number - viewEvent?: VideoViewEvent - }) { - const { video, ip, viewEvent, currentTime } = options const nowMs = new Date().getTime() - let stats: LocalViewerStats = await this.getLocalVideoViewerByIP({ ip, videoId: video.id }) + let stats: LocalViewerStats = await this.getLocalVideoViewer({ sessionId, videoId: video.id }) if (stats && stats.watchSections.length >= MAX_LOCAL_VIEWER_WATCH_SECTIONS) { logger.warn('Too much watch section to store for a viewer, skipping this one', { currentTime, viewEvent, ...lTags(video.uuid) }) @@ -129,9 +113,19 @@ export class VideoViewerStats { logger.debug('Set local video viewer stats for video %s.', video.uuid, { stats, ...lTags(video.uuid) }) - this.setLocalVideoViewer(ip, video.id, stats) + this.setLocalVideoViewer(sessionId, video.id, stats) } + // --------------------------------------------------------------------------- + + async getWatchTime (videoId: number, sessionId: string) { + const stats: LocalViewerStats = await this.getLocalVideoViewer({ sessionId, videoId }) + + return stats?.watchTime || 0 + } + + // --------------------------------------------------------------------------- + async processViewerStats () { if (this.processingViewersStats) return this.processingViewersStats = true @@ -213,11 +207,11 @@ export class VideoViewerStats { * */ - private getLocalVideoViewerByIP (options: { - ip: string + private getLocalVideoViewer (options: { + sessionId: string videoId: number }): Promise { - const { viewerKey } = Redis.Instance.generateLocalVideoViewerKeys(options.ip, options.videoId) + const { viewerKey } = Redis.Instance.generateLocalVideoViewerKeys(options.sessionId, options.videoId) return this.getLocalVideoViewerByKey(viewerKey) } @@ -229,11 +223,11 @@ export class VideoViewerStats { return Redis.Instance.getLocalVideoViewer({ key }) } - private setLocalVideoViewer (ip: string, videoId: number, stats: LocalViewerStats) { - const { viewerKey } = Redis.Instance.generateLocalVideoViewerKeys(ip, videoId) + private setLocalVideoViewer (sessionId: string, videoId: number, stats: LocalViewerStats) { + const { viewerKey } = Redis.Instance.generateLocalVideoViewerKeys(sessionId, videoId) this.viewerCache.set(viewerKey, stats) - this.redisPendingWrites.set(viewerKey, { ip, videoId, stats }) + this.redisPendingWrites.set(viewerKey, { sessionId, videoId, stats }) } private deleteLocalVideoViewersKeys (key: string) { @@ -248,13 +242,13 @@ export class VideoViewerStats { this.processingRedisWrites = true for (const [ key, pendingWrite ] of this.redisPendingWrites) { - const { ip, videoId, stats } = pendingWrite + const { sessionId, videoId, stats } = pendingWrite this.redisPendingWrites.delete(key) try { - await Redis.Instance.setLocalVideoViewer(ip, videoId, stats) + await Redis.Instance.setLocalVideoViewer(sessionId, videoId, stats) } catch (err) { - logger.error('Cannot write viewer into redis', { ip, videoId, stats, err }) + logger.error('Cannot write viewer into redis', { sessionId, videoId, stats, err }) } } diff --git a/server/core/lib/views/shared/video-views.ts b/server/core/lib/views/shared/video-views.ts index ca1941f68..078aca836 100644 --- a/server/core/lib/views/shared/video-views.ts +++ b/server/core/lib/views/shared/video-views.ts @@ -1,12 +1,13 @@ +import { buildUUID } from '@peertube/peertube-node-utils' import { logger, loggerTagsFactory } from '@server/helpers/logger.js' +import { VIEW_LIFETIME } from '@server/initializers/constants.js' import { sendView } from '@server/lib/activitypub/send/send-view.js' import { getCachedVideoDuration } from '@server/lib/video.js' import { getServerActor } from '@server/models/application/application.js' import { MVideo, MVideoImmutable } from '@server/types/models/index.js' -import { buildUUID } from '@peertube/peertube-node-utils' -import { Redis } from '../../redis.js' import { LRUCache } from 'lru-cache' -import { VIEW_LIFETIME } from '@server/initializers/constants.js' +import { Redis } from '../../redis.js' +import { CONFIG } from '@server/initializers/config.js' const lTags = loggerTagsFactory('views') @@ -19,19 +20,19 @@ export class VideoViews { async addLocalView (options: { video: MVideoImmutable - ip: string + sessionId: string watchTime: number }) { - const { video, ip, watchTime } = options + const { video, sessionId, watchTime } = options logger.debug('Adding local view to video %s.', video.uuid, { watchTime, ...lTags(video.uuid) }) if (!await this.hasEnoughWatchTime(video, watchTime)) return false - const viewExists = await this.doesVideoIPViewExist(ip, video.uuid) + const viewExists = await this.doesVideoSessionIdViewExist(sessionId, video.uuid) if (viewExists) return false - await this.setIPVideoView(ip, video.uuid) + await this.setSessionIdVideoView(sessionId, video.uuid) await this.addView(video) @@ -69,24 +70,25 @@ export class VideoViews { private async hasEnoughWatchTime (video: MVideoImmutable, watchTime: number) { const { duration, isLive } = await getCachedVideoDuration(video.id) - if (isLive || duration >= 30) return watchTime >= 30 + const countViewAfterSeconds = CONFIG.VIEWS.VIDEOS.COUNT_VIEW_AFTER / 1000 // Config is in ms + if (isLive || duration >= countViewAfterSeconds) return watchTime >= countViewAfterSeconds // Check more than 50% of the video is watched return duration / watchTime < 2 } - private doesVideoIPViewExist (ip: string, videoUUID: string) { - const key = Redis.Instance.generateIPViewKey(ip, videoUUID) + private doesVideoSessionIdViewExist (sessionId: string, videoUUID: string) { + const key = Redis.Instance.generateSessionIdViewKey(sessionId, videoUUID) const value = this.viewsCache.has(key) if (value === true) return Promise.resolve(true) - return Redis.Instance.doesVideoIPViewExist(ip, videoUUID) + return Redis.Instance.doesVideoSessionIdViewExist(sessionId, videoUUID) } - private setIPVideoView (ip: string, videoUUID: string) { - const key = Redis.Instance.generateIPViewKey(ip, videoUUID) + private setSessionIdVideoView (sessionId: string, videoUUID: string) { + const key = Redis.Instance.generateSessionIdViewKey(sessionId, videoUUID) this.viewsCache.set(key, true) - return Redis.Instance.setIPVideoView(ip, videoUUID) + return Redis.Instance.setSessionIdVideoView(sessionId, videoUUID) } } diff --git a/server/core/lib/views/video-views-manager.ts b/server/core/lib/views/video-views-manager.ts index 0d20d5f34..ee4dd6d78 100644 --- a/server/core/lib/views/video-views-manager.ts +++ b/server/core/lib/views/video-views-manager.ts @@ -1,6 +1,8 @@ -import { logger, loggerTagsFactory } from '@server/helpers/logger.js' -import { MVideo, MVideoImmutable } from '@server/types/models/index.js' import { VideoViewEvent } from '@peertube/peertube-models' +import { sha256 } from '@peertube/peertube-node-utils' +import { logger, loggerTagsFactory } from '@server/helpers/logger.js' +import { CONFIG } from '@server/initializers/config.js' +import { MVideo, MVideoImmutable } from '@server/types/models/index.js' import { VideoScope, VideoViewerCounters, VideoViewerStats, VideoViews, ViewerScope } from './shared/index.js' /** @@ -44,20 +46,26 @@ export class VideoViewsManager { video: MVideoImmutable currentTime: number ip: string | null + sessionId?: string viewEvent?: VideoViewEvent }) { const { video, ip, viewEvent, currentTime } = options - logger.debug('Processing local view for %s and ip %s.', video.url, ip, lTags()) + let sessionId = options.sessionId + if (!sessionId || CONFIG.VIEWS.VIDEOS.TRUST_VIEWER_SESSION_ID !== true) { + sessionId = sha256(CONFIG.SECRETS + '-' + ip) + } - await this.videoViewerStats.addLocalViewer({ video, ip, viewEvent, currentTime }) + logger.debug(`Processing local view for ${video.url}, ip ${ip} and session id ${sessionId}.`, lTags()) - const successViewer = await this.videoViewerCounters.addLocalViewer({ video, ip }) + await this.videoViewerStats.addLocalViewer({ video, ip, sessionId, viewEvent, currentTime }) + + const successViewer = await this.videoViewerCounters.addLocalViewer({ video, sessionId }) // Do it after added local viewer to fetch updated information - const watchTime = await this.videoViewerStats.getWatchTime(video.id, ip) + const watchTime = await this.videoViewerStats.getWatchTime(video.id, sessionId) - const successView = await this.videoViews.addLocalView({ video, watchTime, ip }) + const successView = await this.videoViews.addLocalView({ video, watchTime, sessionId }) return { successView, successViewer } }