mirrors/PeerTube
1
0
Fork 0
mirror of https://github.com/Chocobozzz/PeerTube.git synced 2024-05-17 04:12:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

Put private videos under a specific subdirectory

Browse source
This commit is contained in:
Chocobozzz 2022-10-12 16:09:02 +02:00 committed by Chocobozzz
parent 38a3ccc7f8
commit 3545e72c68
105 changed files with 2929 additions and 1308 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

53
client/src/app/+videos/+video-watch/video-watch.component.ts
View file

@ -20,12 +20,12 @@ import {
} from '@app/core'
import { HooksService } from '@app/core/plugins/hooks.service'
import { isXPercentInViewport, scrollToTop } from '@app/helpers'
import { Video, VideoCaptionService, VideoDetails, VideoService } from '@app/shared/shared-main'
import { Video, VideoCaptionService, VideoDetails, VideoFileTokenService, VideoService } from '@app/shared/shared-main'
import { SubscribeButtonComponent } from '@app/shared/shared-user-subscription'
import { LiveVideoService } from '@app/shared/shared-video-live'
import { VideoPlaylist, VideoPlaylistService } from '@app/shared/shared-video-playlist'
import { logger } from '@root-helpers/logger'
import { isP2PEnabled } from '@root-helpers/video'
import { isP2PEnabled, videoRequiresAuth } from '@root-helpers/video'
import { timeToInt } from '@shared/core-utils'
import {
HTMLServerConfig,
@ -78,6 +78,8 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
private nextVideoUUID = ''
private nextVideoTitle = ''
private videoFileToken: string
private currentTime: number
private paramsSub: Subscription
@ -110,6 +112,7 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
private pluginService: PluginService,
private peertubeSocket: PeerTubeSocket,
private screenService: ScreenService,
private videoFileTokenService: VideoFileTokenService,
private location: PlatformLocation,
@Inject(LOCALE_ID) private localeId: string
) { }
@ -252,12 +255,19 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
'filter:api.video-watch.video.get.result'
)
const videoAndLiveObs: Observable<{ video: VideoDetails, live?: LiveVideo }> = videoObs.pipe(
const videoAndLiveObs: Observable<{ video: VideoDetails, live?: LiveVideo, videoFileToken?: string }> = videoObs.pipe(
switchMap(video => {
if (!video.isLive) return of({ video })
if (!video.isLive) return of({ video, live: undefined })
return this.liveVideoService.getVideoLive(video.uuid)
.pipe(map(live => ({ live, video })))
}),
switchMap(({ video, live }) => {
if (!videoRequiresAuth(video)) return of({ video, live, videoFileToken: undefined })
return this.videoFileTokenService.getVideoFileToken(video.uuid)
.pipe(map(({ token }) => ({ video, live, videoFileToken: token })))
})
)
@ -266,7 +276,7 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
this.videoCaptionService.listCaptions(videoId),
this.userService.getAnonymousOrLoggedUser()
]).subscribe({
next: ([ { video, live }, captionsResult, loggedInOrAnonymousUser ]) => {
next: ([ { video, live, videoFileToken }, captionsResult, loggedInOrAnonymousUser ]) => {
const queryParams = this.route.snapshot.queryParams
const urlOptions = {
@ -283,7 +293,7 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
peertubeLink: false
}
this.onVideoFetched({ video, live, videoCaptions: captionsResult.data, loggedInOrAnonymousUser, urlOptions })
this.onVideoFetched({ video, live, videoCaptions: captionsResult.data, videoFileToken, loggedInOrAnonymousUser, urlOptions })
.catch(err => this.handleGlobalError(err))
},
@ -356,16 +366,19 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
video: VideoDetails
live: LiveVideo
videoCaptions: VideoCaption[]
videoFileToken: string
urlOptions: URLOptions
loggedInOrAnonymousUser: User
}) {
const { video, live, videoCaptions, urlOptions, loggedInOrAnonymousUser } = options
const { video, live, videoCaptions, urlOptions, videoFileToken, loggedInOrAnonymousUser } = options
this.subscribeToLiveEventsIfNeeded(this.video, video)
this.video = video
this.videoCaptions = videoCaptions
this.liveVideo = live
this.videoFileToken = videoFileToken
// Re init attributes
this.playerPlaceholderImgSrc = undefined
@ -414,6 +427,7 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
video: this.video,
videoCaptions: this.videoCaptions,
liveVideo: this.liveVideo,
videoFileToken: this.videoFileToken,
urlOptions,
loggedInOrAnonymousUser,
user: this.user
@ -561,11 +575,15 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
video: VideoDetails
liveVideo: LiveVideo
videoCaptions: VideoCaption[]
videoFileToken: string
urlOptions: CustomizationOptions & { playerMode: PlayerMode }
loggedInOrAnonymousUser: User
user?: AuthUser // Keep for plugins
}) {
const { video, liveVideo, videoCaptions, urlOptions, loggedInOrAnonymousUser } = params
const { video, liveVideo, videoCaptions, videoFileToken, urlOptions, loggedInOrAnonymousUser } = params
const getStartTime = () => {
const byUrl = urlOptions.startTime !== undefined
@ -623,13 +641,6 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
theaterButton: true,
captions: videoCaptions.length !== 0,
videoViewUrl: video.privacy.id !== VideoPrivacy.PRIVATE
? this.videoService.getVideoViewUrl(video.uuid)
: null,
authorizationHeader: this.authService.getRequestHeaderValue(),
metricsUrl: environment.apiUrl + '/api/v1/metrics/playback',
embedUrl: video.embedUrl,
embedTitle: video.name,
instanceName: this.serverConfig.instance.name,
@ -639,7 +650,17 @@ export class VideoWatchComponent implements OnInit, OnDestroy {
language: this.localeId,
serverUrl: environment.apiUrl,
metricsUrl: environment.apiUrl + '/api/v1/metrics/playback',
videoViewUrl: video.privacy.id !== VideoPrivacy.PRIVATE
? this.videoService.getVideoViewUrl(video.uuid)
: null,
authorizationHeader: () => this.authService.getRequestHeaderValue(),
serverUrl: environment.originServerUrl,
videoFileToken: () => videoFileToken,
requiresAuth: videoRequiresAuth(video),
videoCaptions: playerCaptions,

18
client/src/app/core/auth/auth-user.model.ts
View file

@ -1,7 +1,7 @@
import { Observable, of } from 'rxjs'
import { map } from 'rxjs/operators'
import { User } from '@app/core/users/user.model'
import { UserTokens } from '@root-helpers/users'
import { OAuthUserTokens } from '@root-helpers/users'
import { hasUserRight } from '@shared/core-utils/users'
import {
MyUser as ServerMyUserModel,
@ -13,33 +13,33 @@ import {
} from '@shared/models'
export class AuthUser extends User implements ServerMyUserModel {
tokens: UserTokens
oauthTokens: OAuthUserTokens
specialPlaylists: MyUserSpecialPlaylist[]
canSeeVideosLink = true
constructor (userHash: Partial<ServerMyUserModel>, hashTokens: Partial<UserTokens>) {
constructor (userHash: Partial<ServerMyUserModel>, hashTokens: Partial<OAuthUserTokens>) {
super(userHash)
this.tokens = new UserTokens(hashTokens)
this.oauthTokens = new OAuthUserTokens(hashTokens)
this.specialPlaylists = userHash.specialPlaylists
}
getAccessToken () {
return this.tokens.accessToken
return this.oauthTokens.accessToken
}
getRefreshToken () {
return this.tokens.refreshToken
return this.oauthTokens.refreshToken
}
getTokenType () {
return this.tokens.tokenType
return this.oauthTokens.tokenType
}
refreshTokens (accessToken: string, refreshToken: string) {
this.tokens.accessToken = accessToken
this.tokens.refreshToken = refreshToken
this.oauthTokens.accessToken = accessToken
this.oauthTokens.refreshToken = refreshToken
}
hasRight (right: UserRight) {

4
client/src/app/core/auth/auth.service.ts
View file

@ -5,7 +5,7 @@ import { HttpClient, HttpErrorResponse, HttpHeaders, HttpParams } from '@angular
import { Injectable } from '@angular/core'
import { Router } from '@angular/router'
import { Notifier } from '@app/core/notification/notifier.service'
import { logger, objectToUrlEncoded, peertubeLocalStorage, UserTokens } from '@root-helpers/index'
import { logger, OAuthUserTokens, objectToUrlEncoded, peertubeLocalStorage } from '@root-helpers/index'
import { HttpStatusCode, MyUser as UserServerModel, OAuthClientLocal, User, UserLogin, UserRefreshToken } from '@shared/models'
import { environment } from '../../../environments/environment'
import { RestExtractor } from '../rest/rest-extractor.service'
@ -74,7 +74,7 @@ export class AuthService {
]
}
buildAuthUser (userInfo: Partial<User>, tokens: UserTokens) {
buildAuthUser (userInfo: Partial<User>, tokens: OAuthUserTokens) {
this.user = new AuthUser(userInfo, tokens)
}

14
client/src/app/core/users/user-local-storage.service.ts
View file

@ -4,7 +4,7 @@ import { Injectable } from '@angular/core'
import { AuthService, AuthStatus } from '@app/core/auth'
import { getBoolOrDefault } from '@root-helpers/local-storage-utils'
import { logger } from '@root-helpers/logger'
import { UserLocalStorageKeys, UserTokens } from '@root-helpers/users'
import { UserLocalStorageKeys, OAuthUserTokens } from '@root-helpers/users'
import { UserRole, UserUpdateMe } from '@shared/models'
import { NSFWPolicyType } from '@shared/models/videos'
import { ServerService } from '../server'
@ -24,7 +24,7 @@ export class UserLocalStorageService {
this.setLoggedInUser(user)
this.setUserInfo(user)
this.setTokens(user.tokens)
this.setTokens(user.oauthTokens)
}
})
@ -43,7 +43,7 @@ export class UserLocalStorageService {
next: () => {
const user = this.authService.getUser()
this.setTokens(user.tokens)
this.setTokens(user.oauthTokens)
}
})
}
@ -174,14 +174,14 @@ export class UserLocalStorageService {
// ---------------------------------------------------------------------------
getTokens () {
return UserTokens.getUserTokens(this.localStorageService)
return OAuthUserTokens.getUserTokens(this.localStorageService)
}
setTokens (tokens: UserTokens) {
UserTokens.saveToLocalStorage(this.localStorageService, tokens)
setTokens (tokens: OAuthUserTokens) {
OAuthUserTokens.saveToLocalStorage(this.localStorageService, tokens)
}
flushTokens () {
UserTokens.flushLocalStorage(this.localStorageService)
OAuthUserTokens.flushLocalStorage(this.localStorageService)
}
}

5
client/src/app/helpers/utils/url.ts
View file

@ -54,8 +54,9 @@ function objectToFormData (obj: any, form?: FormData, namespace?: string) {
}
export {
objectToFormData,
getAbsoluteAPIUrl,
getAPIHost,
getAbsoluteEmbedUrl
getAbsoluteEmbedUrl,
objectToFormData
}

11
client/src/app/shared/shared-main/shared-main.module.ts
View file

@ -44,7 +44,15 @@ import {
import { PluginPlaceholderComponent, PluginSelectorDirective } from './plugins'
import { ActorRedirectGuard } from './router'
import { UserHistoryService, UserNotificationsComponent, UserNotificationService, UserQuotaComponent } from './users'
import { EmbedComponent, RedundancyService, VideoImportService, VideoOwnershipService, VideoResolver, VideoService } from './video'
import {
EmbedComponent,
RedundancyService,
VideoFileTokenService,
VideoImportService,
VideoOwnershipService,
VideoResolver,
VideoService
} from './video'
import { VideoCaptionService } from './video-caption'
import { VideoChannelService } from './video-channel'
@ -185,6 +193,7 @@ import { VideoChannelService } from './video-channel'
VideoImportService,
VideoOwnershipService,
VideoService,
VideoFileTokenService,
VideoResolver,
VideoCaptionService,

1
client/src/app/shared/shared-main/video/index.ts
View file

@ -2,6 +2,7 @@ export * from './embed.component'
export * from './redundancy.service'
export * from './video-details.model'
export * from './video-edit.model'
export * from './video-file-token.service'
export * from './video-import.service'
export * from './video-ownership.service'
export * from './video.model'

33
client/src/app/shared/shared-main/video/video-file-token.service.ts Normal file
View file

@ -0,0 +1,33 @@
import { catchError, map, of, tap } from 'rxjs'
import { HttpClient } from '@angular/common/http'
import { Injectable } from '@angular/core'
import { RestExtractor } from '@app/core'
import { VideoToken } from '@shared/models'
import { VideoService } from './video.service'
@Injectable()
export class VideoFileTokenService {
private readonly store = new Map<string, { token: string, expires: Date }>()
constructor (
private authHttp: HttpClient,
private restExtractor: RestExtractor
) {}
getVideoFileToken (videoUUID: string) {
const existing = this.store.get(videoUUID)
if (existing) return of(existing)
return this.createVideoFileToken(videoUUID)
.pipe(tap(result => this.store.set(videoUUID, { token: result.token, expires: new Date(result.expires) })))
}
private createVideoFileToken (videoUUID: string) {
return this.authHttp.post<VideoToken>(`${VideoService.BASE_VIDEO_URL}/${videoUUID}/token`, {})
.pipe(
map(({ files }) => files),
catchError(err => this.restExtractor.handleError(err))
)
}
}

5
client/src/app/shared/shared-video-miniature/video-download.component.html
View file

@ -48,10 +48,7 @@
<ng-template ngbNavContent>
<div class="nav-content">
<my-input-text
*ngIf="!isConfidentialVideo()"
[show]="true" [readonly]="true" [withCopy]="true" [withToggle]="false" [value]="getLink()"
></my-input-text>
<my-input-text [show]="true" [readonly]="true" [withCopy]="true" [withToggle]="false" [value]="getLink()"></my-input-text>
</div>
</ng-template>
</ng-container>

19
client/src/app/shared/shared-video-miniature/video-download.component.ts
View file

@ -2,11 +2,12 @@ import { mapValues, pick } from 'lodash-es'
import { firstValueFrom } from 'rxjs'
import { tap } from 'rxjs/operators'
import { Component, ElementRef, Inject, LOCALE_ID, ViewChild } from '@angular/core'
import { AuthService, HooksService, Notifier } from '@app/core'
import { HooksService } from '@app/core'
import { NgbModal, NgbModalRef } from '@ng-bootstrap/ng-bootstrap'
import { logger } from '@root-helpers/logger'
import { videoRequiresAuth } from '@root-helpers/video'
import { VideoCaption, VideoFile, VideoPrivacy } from '@shared/models'
import { BytesPipe, NumberFormatterPipe, VideoDetails, VideoService } from '../shared-main'
import { BytesPipe, NumberFormatterPipe, VideoDetails, VideoFileTokenService, VideoService } from '../shared-main'
type DownloadType = 'video' | 'subtitles'
type FileMetadata = { [key: string]: { label: string, value: string }}
@ -32,6 +33,8 @@ export class VideoDownloadComponent {
type: DownloadType = 'video'
videoFileToken: string
private activeModal: NgbModalRef
private bytesPipe: BytesPipe
@ -42,10 +45,9 @@ export class VideoDownloadComponent {
constructor (
@Inject(LOCALE_ID) private localeId: string,
private notifier: Notifier,
private modalService: NgbModal,
private videoService: VideoService,
private auth: AuthService,
private videoFileTokenService: VideoFileTokenService,
private hooks: HooksService
) {
this.bytesPipe = new BytesPipe()
@ -71,6 +73,8 @@ export class VideoDownloadComponent {
}
show (video: VideoDetails, videoCaptions?: VideoCaption[]) {
this.videoFileToken = undefined
this.video = video
this.videoCaptions = videoCaptions
@ -84,6 +88,11 @@ export class VideoDownloadComponent {
this.subtitleLanguageId = this.videoCaptions[0].language.id
}
if (videoRequiresAuth(this.video)) {
this.videoFileTokenService.getVideoFileToken(this.video.uuid)
.subscribe(({ token }) => this.videoFileToken = token)
}
this.activeModal.shown.subscribe(() => {
this.hooks.runAction('action:modal.video-download.shown', 'common')
})
@ -155,7 +164,7 @@ export class VideoDownloadComponent {
if (!file) return ''
const suffix = this.isConfidentialVideo()
? '?access_token=' + this.auth.getAccessToken()
? '?videoFileToken=' + this.videoFileToken
: ''
switch (this.downloadType) {

8
client/src/assets/player/shared/common/utils.ts
View file

@ -52,6 +52,10 @@ function getRtcConfig () {
}
}
function isSameOrigin (current: string, target: string) {
return new URL(current).origin === new URL(target).origin
}
// ---------------------------------------------------------------------------
export {
@ -60,5 +64,7 @@ export {
videoFileMaxByResolution,
videoFileMinByResolution,
bytes
bytes,
isSameOrigin
}

18
client/src/assets/player/shared/manager-options/hls-options-builder.ts
View file

@ -5,7 +5,7 @@ import { LiveVideoLatencyMode } from '@shared/models'
import { getAverageBandwidthInStore } from '../../peertube-player-local-storage'
import { P2PMediaLoader, P2PMediaLoaderPluginOptions } from '../../types'
import { PeertubePlayerManagerOptions } from '../../types/manager-options'
import { getRtcConfig } from '../common'
import { getRtcConfig, isSameOrigin } from '../common'
import { RedundancyUrlManager } from '../p2p-media-loader/redundancy-url-manager'
import { segmentUrlBuilderFactory } from '../p2p-media-loader/segment-url-builder'
import { segmentValidatorFactory } from '../p2p-media-loader/segment-validator'
@ -84,7 +84,21 @@ export class HLSOptionsBuilder {
simultaneousHttpDownloads: 1,
httpFailedSegmentTimeout: 1000,
segmentValidator: segmentValidatorFactory(this.options.p2pMediaLoader.segmentsSha256Url, this.options.common.isLive),
xhrSetup: (xhr, url) => {
if (!this.options.common.requiresAuth) return
if (!isSameOrigin(this.options.common.serverUrl, url)) return
xhr.setRequestHeader('Authorization', this.options.common.authorizationHeader())
},
segmentValidator: segmentValidatorFactory({
segmentsSha256Url: this.options.p2pMediaLoader.segmentsSha256Url,
isLive: this.options.common.isLive,
authorizationHeader: this.options.common.authorizationHeader,
requiresAuth: this.options.common.requiresAuth,
serverUrl: this.options.common.serverUrl
}),
segmentUrlBuilder: segmentUrlBuilderFactory(redundancyUrlManager),
useP2P: this.options.common.p2pEnabled,

15
client/src/assets/player/shared/manager-options/webtorrent-options-builder.ts
View file

@ -1,4 +1,5 @@
import { PeertubePlayerManagerOptions } from '../../types'
import { addQueryParams } from '../../../../../../shared/core-utils'
import { PeertubePlayerManagerOptions, WebtorrentPluginOptions } from '../../types'
export class WebTorrentOptionsBuilder {
@ -16,13 +17,23 @@ export class WebTorrentOptionsBuilder {
const autoplay = this.autoPlayValue === 'play'
const webtorrent = {
const webtorrent: WebtorrentPluginOptions = {
autoplay,
playerRefusedP2P: commonOptions.p2pEnabled === false,
videoDuration: commonOptions.videoDuration,
playerElement: commonOptions.playerElement,
videoFileToken: commonOptions.videoFileToken,
requiresAuth: commonOptions.requiresAuth,
buildWebSeedUrls: file => {
if (!commonOptions.requiresAuth) return []
return [ addQueryParams(file.fileUrl, { videoFileToken: commonOptions.videoFileToken() }) ]
},
videoFiles: webtorrentOptions.videoFiles.length !== 0
? webtorrentOptions.videoFiles
// The WebTorrent plugin won't be able to play these files, but it will fallback to HTTP mode

30
client/src/assets/player/shared/p2p-media-loader/segment-validator.ts
View file

@ -2,13 +2,22 @@ import { basename } from 'path'
import { Segment } from '@peertube/p2p-media-loader-core'
import { logger } from '@root-helpers/logger'
import { wait } from '@root-helpers/utils'
import { isSameOrigin } from '../common'
type SegmentsJSON = { [filename: string]: string | { [byterange: string]: string } }
const maxRetries = 3
function segmentValidatorFactory (segmentsSha256Url: string, isLive: boolean) {
let segmentsJSON = fetchSha256Segments(segmentsSha256Url)
function segmentValidatorFactory (options: {
serverUrl: string
segmentsSha256Url: string
isLive: boolean
authorizationHeader: () => string
requiresAuth: boolean
}) {
const { serverUrl, segmentsSha256Url, isLive, authorizationHeader, requiresAuth } = options
let segmentsJSON = fetchSha256Segments({ serverUrl, segmentsSha256Url, authorizationHeader, requiresAuth })
const regex = /bytes=(\d+)-(\d+)/
return async function segmentValidator (segment: Segment, _method: string, _peerId: string, retry = 1) {
@ -28,7 +37,7 @@ function segmentValidatorFactory (segmentsSha256Url: string, isLive: boolean) {
await wait(1000)
segmentsJSON = fetchSha256Segments(segmentsSha256Url)
segmentsJSON = fetchSha256Segments({ serverUrl, segmentsSha256Url, authorizationHeader, requiresAuth })
await segmentValidator(segment, _method, _peerId, retry + 1)
return
@ -68,8 +77,19 @@ export {
// ---------------------------------------------------------------------------
function fetchSha256Segments (url: string) {
return fetch(url)
function fetchSha256Segments (options: {
serverUrl: string
segmentsSha256Url: string
authorizationHeader: () => string
requiresAuth: boolean
}) {
const { serverUrl, segmentsSha256Url, requiresAuth, authorizationHeader } = options
const headers = requiresAuth && isSameOrigin(serverUrl, segmentsSha256Url)
? { Authorization: authorizationHeader() }
: {}
return fetch(segmentsSha256Url, { headers })
.then(res => res.json() as Promise<SegmentsJSON>)
.catch(err => {
logger.error('Cannot get sha256 segments', err)

4
client/src/assets/player/shared/peertube/peertube-plugin.ts
View file

@ -22,7 +22,7 @@ const Plugin = videojs.getPlugin('plugin')
class PeerTubePlugin extends Plugin {
private readonly videoViewUrl: string
private readonly authorizationHeader: string
private readonly authorizationHeader: () => string
private readonly videoUUID: string
private readonly startTime: number
@ -228,7 +228,7 @@ class PeerTubePlugin extends Plugin {
'Content-type': 'application/json; charset=UTF-8'
})
if (this.authorizationHeader) headers.set('Authorization', this.authorizationHeader)
if (this.authorizationHeader) headers.set('Authorization', this.authorizationHeader())
return fetch(this.videoViewUrl, { method: 'POST', body: JSON.stringify(body), headers })
}

22
client/src/assets/player/shared/webtorrent/webtorrent-plugin.ts
View file

@ -2,7 +2,7 @@ import videojs from 'video.js'
import * as WebTorrent from 'webtorrent'
import { logger } from '@root-helpers/logger'
import { isIOS } from '@root-helpers/web-browser'
import { timeToInt } from '@shared/core-utils'
import { addQueryParams, timeToInt } from '@shared/core-utils'
import { VideoFile } from '@shared/models'
import { getAverageBandwidthInStore, getStoredMute, getStoredVolume, saveAverageBandwidth } from '../../peertube-player-local-storage'
import { PeerTubeResolution, PlayerNetworkInfo, WebtorrentPluginOptions } from '../../types'
@ -38,6 +38,8 @@ class WebTorrentPlugin extends Plugin {
BANDWIDTH_AVERAGE_NUMBER_OF_VALUES: 5 // Last 5 seconds to build average bandwidth
}
private readonly buildWebSeedUrls: (file: VideoFile) => string[]
private readonly webtorrent = new WebTorrent({
tracker: {
rtcConfig: getRtcConfig()
@ -57,6 +59,9 @@ class WebTorrentPlugin extends Plugin {
private isAutoResolutionObservation = false
private playerRefusedP2P = false
private requiresAuth: boolean
private videoFileToken: () => string
private torrentInfoInterval: any
private autoQualityInterval: any
private addTorrentDelay: any
@ -81,6 +86,11 @@ class WebTorrentPlugin extends Plugin {
this.savePlayerSrcFunction = this.player.src
this.playerElement = options.playerElement
this.requiresAuth = options.requiresAuth
this.videoFileToken = options.videoFileToken
this.buildWebSeedUrls = options.buildWebSeedUrls
this.player.ready(() => {
const playerOptions = this.player.options_
@ -268,7 +278,8 @@ class WebTorrentPlugin extends Plugin {
return new CacheChunkStore(new PeertubeChunkStore(chunkLength, storeOpts), {
max: 100
})
}
},
urlList: this.buildWebSeedUrls(this.currentVideoFile)
}
this.torrent = this.webtorrent.add(magnetOrTorrentUrl, torrentOptions, torrent => {
@ -533,7 +544,12 @@ class WebTorrentPlugin extends Plugin {
// Enable error display now this is our last fallback
this.player.one('error', () => this.player.peertube().displayFatalError())
const httpUrl = this.currentVideoFile.fileUrl
let httpUrl = this.currentVideoFile.fileUrl
if (this.requiresAuth && this.videoFileToken) {
httpUrl = addQueryParams(httpUrl, { videoFileToken: this.videoFileToken() })
}
this.player.src = this.savePlayerSrcFunction
this.player.src(httpUrl)

4
client/src/assets/player/types/manager-options.ts
View file

@ -57,7 +57,7 @@ export interface CommonOptions extends CustomizationOptions {
captions: boolean
videoViewUrl: string
authorizationHeader?: string
authorizationHeader?: () => string
metricsUrl: string
@ -77,6 +77,8 @@ export interface CommonOptions extends CustomizationOptions {
videoShortUUID: string
serverUrl: string
requiresAuth: boolean
videoFileToken: () => string
errorNotifier: (message: string) => void
}

7
client/src/assets/player/types/peertube-videojs-typings.ts
View file

@ -95,7 +95,7 @@ type PeerTubePluginOptions = {
videoDuration: number
videoViewUrl: string
authorizationHeader?: string
authorizationHeader?: () => string
subtitle?: string
@ -151,6 +151,11 @@ type WebtorrentPluginOptions = {
startTime: number | string
playerRefusedP2P: boolean
requiresAuth: boolean
videoFileToken: () => string
buildWebSeedUrls: (file: VideoFile) => string[]
}
type P2PMediaLoaderPluginOptions = {

4
client/src/root-helpers/logger.ts
View file

@ -1,6 +1,6 @@
import { ClientLogCreate } from '@shared/models/server'
import { peertubeLocalStorage } from './peertube-web-storage'
import { UserTokens } from './users'
import { OAuthUserTokens } from './users'
export type LoggerHook = (message: LoggerMessage, meta?: LoggerMeta) => void
export type LoggerLevel = 'info' | 'warn' | 'error'
@ -56,7 +56,7 @@ class Logger {
})
try {
const tokens = UserTokens.getUserTokens(peertubeLocalStorage)
const tokens = OAuthUserTokens.getUserTokens(peertubeLocalStorage)
if (tokens) headers.set('Authorization', `${tokens.tokenType} ${tokens.accessToken}`)
} catch (err) {

2
client/src/root-helpers/users/index.ts
View file

@ -1,2 +1,2 @@
export * from './user-local-storage-keys'
export * from './user-tokens'
export * from './oauth-user-tokens'

8
client/src/root-helpers/users/user-tokens.ts → client/src/root-helpers/users/oauth-user-tokens.ts
View file

@ -1,11 +1,11 @@
import { UserTokenLocalStorageKeys } from './user-local-storage-keys'
export class UserTokens {
export class OAuthUserTokens {
accessToken: string
refreshToken: string
tokenType: string
constructor (hash?: Partial<UserTokens>) {
constructor (hash?: Partial<OAuthUserTokens>) {
if (hash) {
this.accessToken = hash.accessToken
this.refreshToken = hash.refreshToken
@ -25,14 +25,14 @@ export class UserTokens {
if (!accessTokenLocalStorage || !refreshTokenLocalStorage || !tokenTypeLocalStorage) return null
return new UserTokens({
return new OAuthUserTokens({
accessToken: accessTokenLocalStorage,
refreshToken: refreshTokenLocalStorage,
tokenType: tokenTypeLocalStorage
})
}
static saveToLocalStorage (localStorage: Pick<Storage, 'setItem'>, tokens: UserTokens) {
static saveToLocalStorage (localStorage: Pick<Storage, 'setItem'>, tokens: OAuthUserTokens) {
localStorage.setItem(UserTokenLocalStorageKeys.ACCESS_TOKEN, tokens.accessToken)
localStorage.setItem(UserTokenLocalStorageKeys.REFRESH_TOKEN, tokens.refreshToken)
localStorage.setItem(UserTokenLocalStorageKeys.TOKEN_TYPE, tokens.tokenType)

9
client/src/root-helpers/video.ts
View file

@ -1,4 +1,4 @@
import { HTMLServerConfig, Video } from '@shared/models'
import { HTMLServerConfig, Video, VideoPrivacy } from '@shared/models'
function buildVideoOrPlaylistEmbed (options: {
embedUrl: string
@ -26,9 +26,14 @@ function isP2PEnabled (video: Video, config: HTMLServerConfig, userP2PEnabled: b
return userP2PEnabled
}
function videoRequiresAuth (video: Video) {
return new Set([ VideoPrivacy.PRIVATE, VideoPrivacy.INTERNAL ]).has(video.privacy.id)
}
export {
buildVideoOrPlaylistEmbed,
isP2PEnabled
isP2PEnabled,
videoRequiresAuth
}
// ---------------------------------------------------------------------------

22
client/src/standalone/videos/embed.ts
View file

@ -6,7 +6,7 @@ import { peertubeTranslate } from '../../../../shared/core-utils/i18n'
import { HTMLServerConfig, LiveVideo, ResultList, VideoDetails, VideoPlaylist, VideoPlaylistElement } from '../../../../shared/models'
import { PeertubePlayerManager } from '../../assets/player'
import { TranslationsManager } from '../../assets/player/translations-manager'
import { getParamString, logger } from '../../root-helpers'
import { getParamString, logger, videoRequiresAuth } from '../../root-helpers'
import { PeerTubeEmbedApi } from './embed-api'
import { AuthHTTP, LiveManager, PeerTubePlugin, PlayerManagerOptions, PlaylistFetcher, PlaylistTracker, VideoFetcher } from './shared'
import { PlayerHTML } from './shared/player-html'
@ -167,22 +167,25 @@ export class PeerTubeEmbed {
private async buildVideoPlayer (videoResponse: Response, captionsPromise: Promise<Response>) {
const alreadyHadPlayer = this.resetPlayerElement()
const videoInfoPromise: Promise<{ video: VideoDetails, live?: LiveVideo }> = videoResponse.json()
.then((videoInfo: VideoDetails) => {
const videoInfoPromise = videoResponse.json()
.then(async (videoInfo: VideoDetails) => {
this.playerManagerOptions.loadParams(this.config, videoInfo)
if (!alreadyHadPlayer && !this.playerManagerOptions.hasAutoplay()) {
this.playerHTML.buildPlaceholder(videoInfo)
}
const live = videoInfo.isLive
? await this.videoFetcher.loadLive(videoInfo)
: undefined
if (!videoInfo.isLive) {
return { video: videoInfo }
}
const videoFileToken = videoRequiresAuth(videoInfo)
? await this.videoFetcher.loadVideoToken(videoInfo)
: undefined
return this.videoFetcher.loadVideoWithLive(videoInfo)
return { live, video: videoInfo, videoFileToken }
})
const [ { video, live }, translations, captionsResponse, PeertubePlayerManagerModule ] = await Promise.all([
const [ { video, live, videoFileToken }, translations, captionsResponse, PeertubePlayerManagerModule ] = await Promise.all([
videoInfoPromise,
this.translationsPromise,
captionsPromise,
@ -200,6 +203,9 @@ export class PeerTubeEmbed {
translations,
serverConfig: this.config,
authorizationHeader: () => this.http.getHeaderTokenValue(),
videoFileToken: () => videoFileToken,
onVideoUpdate: (uuid: string) => this.loadVideoAndBuildPlayer(uuid),
playlistTracker: this.playlistTracker,

28
client/src/standalone/videos/shared/auth-http.ts
View file

@ -1,5 +1,5 @@
import { HttpStatusCode, OAuth2ErrorCode, UserRefreshToken } from '../../../../../shared/models'
import { objectToUrlEncoded, UserTokens } from '../../../root-helpers'
import { OAuthUserTokens, objectToUrlEncoded } from '../../../root-helpers'
import { peertubeLocalStorage } from '../../../root-helpers/peertube-web-storage'
export class AuthHTTP {
@ -8,30 +8,30 @@ export class AuthHTTP {
CLIENT_SECRET: 'client_secret'
}
private userTokens: UserTokens
private userOAuthTokens: OAuthUserTokens
private headers = new Headers()
constructor () {
this.userTokens = UserTokens.getUserTokens(peertubeLocalStorage)
this.userOAuthTokens = OAuthUserTokens.getUserTokens(peertubeLocalStorage)
if (this.userTokens) this.setHeadersFromTokens()
if (this.userOAuthTokens) this.setHeadersFromTokens()
}
fetch (url: string, { optionalAuth }: { optionalAuth: boolean }) {
fetch (url: string, { optionalAuth, method }: { optionalAuth: boolean, method?: string }) {
const refreshFetchOptions = optionalAuth
? { headers: this.headers }
: {}
return this.refreshFetch(url.toString(), refreshFetchOptions)
return this.refreshFetch(url.toString(), { ...refreshFetchOptions, method })
}
getHeaderTokenValue () {
return `${this.userTokens.tokenType} ${this.userTokens.accessToken}`
return `${this.userOAuthTokens.tokenType} ${this.userOAuthTokens.accessToken}`
}
isLoggedIn () {
return !!this.userTokens
return !!this.userOAuthTokens
}
private refreshFetch (url: string, options?: RequestInit) {
@ -47,7 +47,7 @@ export class AuthHTTP {
headers.set('Content-Type', 'application/x-www-form-urlencoded')
const data = {
refresh_token: this.userTokens.refreshToken,
refresh_token: this.userOAuthTokens.refreshToken,
client_id: clientId,
client_secret: clientSecret,
response_type: 'code',
@ -64,15 +64,15 @@ export class AuthHTTP {
return res.json()
}).then((obj: UserRefreshToken & { code?: OAuth2ErrorCode }) => {
if (!obj || obj.code === OAuth2ErrorCode.INVALID_GRANT) {
UserTokens.flushLocalStorage(peertubeLocalStorage)
OAuthUserTokens.flushLocalStorage(peertubeLocalStorage)
this.removeTokensFromHeaders()
return resolve()
}
this.userTokens.accessToken = obj.access_token
this.userTokens.refreshToken = obj.refresh_token
UserTokens.saveToLocalStorage(peertubeLocalStorage, this.userTokens)
this.userOAuthTokens.accessToken = obj.access_token
this.userOAuthTokens.refreshToken = obj.refresh_token
OAuthUserTokens.saveToLocalStorage(peertubeLocalStorage, this.userOAuthTokens)
this.setHeadersFromTokens()
@ -84,7 +84,7 @@ export class AuthHTTP {
return refreshingTokenPromise
.catch(() => {
UserTokens.flushLocalStorage(peertubeLocalStorage)
OAuthUserTokens.flushLocalStorage(peertubeLocalStorage)
this.removeTokensFromHeaders()
}).then(() => fetch(url, {

12
client/src/standalone/videos/shared/player-manager-options.ts
View file

@ -17,7 +17,8 @@ import {
isP2PEnabled,
logger,
peertubeLocalStorage,
UserLocalStorageKeys
UserLocalStorageKeys,
videoRequiresAuth
} from '../../../root-helpers'
import { PeerTubePlugin } from './peertube-plugin'
import { PlayerHTML } from './player-html'
@ -154,6 +155,9 @@ export class PlayerManagerOptions {
captionsResponse: Response
live?: LiveVideo
authorizationHeader: () => string
videoFileToken: () => string
serverConfig: HTMLServerConfig
alreadyHadPlayer: boolean
@ -169,9 +173,11 @@ export class PlayerManagerOptions {
video,
captionsResponse,
alreadyHadPlayer,
videoFileToken,
translations,
playlistTracker,
live,
authorizationHeader,
serverConfig
} = options
@ -227,6 +233,10 @@ export class PlayerManagerOptions {
embedUrl: window.location.origin + video.embedPath,
embedTitle: video.name,
requiresAuth: videoRequiresAuth(video),
authorizationHeader,
videoFileToken,
errorNotifier: () => {
// Empty, we don't have a notifier in the embed
},

17
client/src/standalone/videos/shared/video-fetcher.ts
View file

@ -1,4 +1,4 @@
import { HttpStatusCode, LiveVideo, VideoDetails } from '../../../../../shared/models'
import { HttpStatusCode, LiveVideo, VideoDetails, VideoToken } from '../../../../../shared/models'
import { logger } from '../../../root-helpers'
import { AuthHTTP } from './auth-http'
@ -36,10 +36,15 @@ export class VideoFetcher {
return { captionsPromise, videoResponse }
}
loadVideoWithLive (video: VideoDetails) {
loadLive (video: VideoDetails) {
return this.http.fetch(this.getLiveUrl(video.uuid), { optionalAuth: true })
.then(res => res.json())
.then((live: LiveVideo) => ({ video, live }))
.then(res => res.json() as Promise<LiveVideo>)
}
loadVideoToken (video: VideoDetails) {
return this.http.fetch(this.getVideoTokenUrl(video.uuid), { optionalAuth: true, method: 'POST' })
.then(res => res.json() as Promise<VideoToken>)
.then(token => token.files.token)
}
getVideoViewsUrl (videoUUID: string) {
@ -61,4 +66,8 @@ export class VideoFetcher {
private getLiveUrl (videoId: string) {
return window.location.origin + '/api/v1/videos/live/' + videoId
}
private getVideoTokenUrl (id: string) {
return this.getVideoUrl(id) + '/token'
}
}

2
package.json
View file

@ -103,6 +103,7 @@
"@peertube/http-signature": "^1.7.0",
"@uploadx/core": "^6.0.0",
"async-lru": "^1.1.1",
"async-mutex": "^0.4.0",
"bcrypt": "5.0.1",
"bencode": "^2.0.2",
"bittorrent-tracker": "^9.0.0",
@ -177,7 +178,6 @@
},
"devDependencies": {
"@peertube/maildev": "^1.2.0",
"@types/async-lock": "^1.1.0",
"@types/bcrypt": "^5.0.0",
"@types/bencode": "^2.0.0",
"@types/bluebird": "^3.5.33",

74
scripts/migrations/peertube-2.1.ts
View file

@ -1,74 +0,0 @@
import { pathExists, stat, writeFile } from 'fs-extra'
import parseTorrent from 'parse-torrent'
import { join } from 'path'
import * as Sequelize from 'sequelize'
import { logger } from '@server/helpers/logger'
import { createTorrentPromise } from '@server/helpers/webtorrent'
import { CONFIG } from '@server/initializers/config'
import { HLS_STREAMING_PLAYLIST_DIRECTORY, STATIC_PATHS, WEBSERVER } from '@server/initializers/constants'
import { initDatabaseModels, sequelizeTypescript } from '../../server/initializers/database'
run()
.then(() => process.exit(0))
.catch(err => {
console.error(err)
process.exit(-1)
})
async function run () {
logger.info('Creating torrents and updating database for HSL files.')
await initDatabaseModels(true)
const query = 'select "videoFile".id as id, "videoFile".resolution as resolution, "video".uuid as uuid from "videoFile" ' +
'inner join "videoStreamingPlaylist" ON "videoStreamingPlaylist".id = "videoFile"."videoStreamingPlaylistId" ' +
'inner join video ON video.id = "videoStreamingPlaylist"."videoId" ' +
'WHERE video.remote IS FALSE'
const options = {
type: Sequelize.QueryTypes.SELECT
}
const res = await sequelizeTypescript.query(query, options)
for (const row of res) {
const videoFilename = `${row['uuid']}-${row['resolution']}-fragmented.mp4`
const videoFilePath = join(HLS_STREAMING_PLAYLIST_DIRECTORY, row['uuid'], videoFilename)
logger.info('Processing %s.', videoFilePath)
if (!await pathExists(videoFilePath)) {
console.warn('Cannot generate torrent of %s: file does not exist.', videoFilePath)
continue
}
const createTorrentOptions = {
// Keep the extname, it's used by the client to stream the file inside a web browser
name: `video ${row['uuid']}`,
createdBy: 'PeerTube',
announceList: [
[ WEBSERVER.WS + '://' + WEBSERVER.HOSTNAME + ':' + WEBSERVER.PORT + '/tracker/socket' ],
[ WEBSERVER.URL + '/tracker/announce' ]
],
urlList: [ WEBSERVER.URL + join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, row['uuid'], videoFilename) ]
}
const torrent = await createTorrentPromise(videoFilePath, createTorrentOptions)
const torrentName = `${row['uuid']}-${row['resolution']}-hls.torrent`
const filePath = join(CONFIG.STORAGE.TORRENTS_DIR, torrentName)
await writeFile(filePath, torrent)
const parsedTorrent = parseTorrent(torrent)
const infoHash = parsedTorrent.infoHash
const stats = await stat(videoFilePath)
const size = stats.size
const queryUpdate = 'UPDATE "videoFile" SET "infoHash" = ?, "size" = ? WHERE id = ?'
const options = {
type: Sequelize.QueryTypes.UPDATE,
replacements: [ infoHash, size, row['id'] ]
}
await sequelizeTypescript.query(queryUpdate, options)
}
}

28
scripts/prune-storage.ts
View file

@ -2,7 +2,7 @@ import { map } from 'bluebird'
import { readdir, remove, stat } from 'fs-extra'
import { basename, join } from 'path'
import { get, start } from 'prompt'
import { HLS_REDUNDANCY_DIRECTORY, HLS_STREAMING_PLAYLIST_DIRECTORY } from '@server/initializers/constants'
import { DIRECTORIES } from '@server/initializers/constants'
import { VideoFileModel } from '@server/models/video/video-file'
import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist'
import { uniqify } from '@shared/core-utils'
@ -37,9 +37,11 @@ async function run () {
console.log('Detecting files to remove, it could take a while...')
toDelete = toDelete.concat(
await pruneDirectory(CONFIG.STORAGE.VIDEOS_DIR, doesWebTorrentFileExist()),
await pruneDirectory(DIRECTORIES.VIDEOS.PUBLIC, doesWebTorrentFileExist()),
await pruneDirectory(DIRECTORIES.VIDEOS.PRIVATE, doesWebTorrentFileExist()),
await pruneDirectory(HLS_STREAMING_PLAYLIST_DIRECTORY, doesHLSPlaylistExist()),
await pruneDirectory(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, doesHLSPlaylistExist()),
await pruneDirectory(DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, doesHLSPlaylistExist()),
await pruneDirectory(CONFIG.STORAGE.TORRENTS_DIR, doesTorrentFileExist()),
@ -75,7 +77,7 @@ async function run () {
}
}
type ExistFun = (file: string) => Promise<boolean>
type ExistFun = (file: string) => Promise<boolean> | boolean
async function pruneDirectory (directory: string, existFun: ExistFun) {
const files = await readdir(directory)
@ -92,11 +94,21 @@ async function pruneDirectory (directory: string, existFun: ExistFun) {
}
function doesWebTorrentFileExist () {
return (filePath: string) => VideoFileModel.doesOwnedWebTorrentVideoFileExist(basename(filePath))
return (filePath: string) => {
// Don't delete private directory
if (filePath === DIRECTORIES.VIDEOS.PRIVATE) return true
return VideoFileModel.doesOwnedWebTorrentVideoFileExist(basename(filePath))
}
}
function doesHLSPlaylistExist () {
return (hlsPath: string) => VideoStreamingPlaylistModel.doesOwnedHLSPlaylistExist(basename(hlsPath))
return (hlsPath: string) => {
// Don't delete private directory
if (hlsPath === DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE) return true
return VideoStreamingPlaylistModel.doesOwnedHLSPlaylistExist(basename(hlsPath))
}
}
function doesTorrentFileExist () {
@ -127,8 +139,8 @@ async function doesRedundancyExist (filePath: string) {
const isPlaylist = (await stat(filePath)).isDirectory()
if (isPlaylist) {
// Don't delete HLS directory
if (filePath === HLS_REDUNDANCY_DIRECTORY) return true
// Don't delete HLS redundancy directory
if (filePath === DIRECTORIES.HLS_REDUNDANCY) return true
const uuid = getUUIDFromFilename(filePath)
const video = await VideoModel.loadWithFiles(uuid)

2
server/controllers/api/server/debug.ts
View file

@ -8,6 +8,7 @@ import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes'
import { UserRight } from '../../../../shared/models/users'
import { authenticate, ensureUserHasRight } from '../../../middlewares'
import { VideoChannelSyncLatestScheduler } from '@server/lib/schedulers/video-channel-sync-latest-scheduler'
import { UpdateVideosScheduler } from '@server/lib/schedulers/update-videos-scheduler'
const debugRouter = express.Router()
@ -45,6 +46,7 @@ async function runCommand (req: express.Request, res: express.Response) {
'remove-dandling-resumable-uploads': () => RemoveDanglingResumableUploadsScheduler.Instance.execute(),
'process-video-views-buffer': () => VideoViewsBufferScheduler.Instance.execute(),
'process-video-viewers': () => VideoViewsManager.Instance.processViewerStats(),
'process-update-videos-scheduler': () => UpdateVideosScheduler.Instance.execute(),
'process-video-channel-sync-latest': () => VideoChannelSyncLatestScheduler.Instance.execute()
}

2
server/controllers/api/videos/index.ts
View file

@ -41,6 +41,7 @@ import { ownershipVideoRouter } from './ownership'
import { rateVideoRouter } from './rate'
import { statsRouter } from './stats'
import { studioRouter } from './studio'
import { tokenRouter } from './token'
import { transcodingRouter } from './transcoding'
import { updateRouter } from './update'
import { uploadRouter } from './upload'
@ -63,6 +64,7 @@ videosRouter.use('/', uploadRouter)
videosRouter.use('/', updateRouter)
videosRouter.use('/', filesRouter)
videosRouter.use('/', transcodingRouter)
videosRouter.use('/', tokenRouter)
videosRouter.get('/categories',
openapiOperationDoc({ operationId: 'getCategories' }),

33
server/controllers/api/videos/token.ts Normal file
View file

@ -0,0 +1,33 @@
import express from 'express'
import { VideoTokensManager } from '@server/lib/video-tokens-manager'
import { VideoToken } from '@shared/models'
import { asyncMiddleware, authenticate, videosCustomGetValidator } from '../../../middlewares'
const tokenRouter = express.Router()
tokenRouter.post('/:id/token',
authenticate,
asyncMiddleware(videosCustomGetValidator('only-video')),
generateToken
)
// ---------------------------------------------------------------------------
export {
tokenRouter
}
// ---------------------------------------------------------------------------
function generateToken (req: express.Request, res: express.Response) {
const video = res.locals.onlyVideo
const { token, expires } = VideoTokensManager.Instance.create(video.uuid)
return res.json({
files: {
token,
expires
}
} as VideoToken)
}

76
server/controllers/api/videos/update.ts
View file

@ -1,12 +1,12 @@
import express from 'express'
import { Transaction } from 'sequelize/types'
import { changeVideoChannelShare } from '@server/lib/activitypub/share'
import { CreateJobArgument, JobQueue } from '@server/lib/job-queue'
import { buildVideoThumbnailsFromReq, setVideoTags } from '@server/lib/video'
import { addVideoJobsAfterUpdate, buildVideoThumbnailsFromReq, setVideoTags } from '@server/lib/video'
import { setVideoPrivacy } from '@server/lib/video-privacy'
import { openapiOperationDoc } from '@server/middlewares/doc'
import { FilteredModelAttributes } from '@server/types'
import { MVideoFullLight } from '@server/types/models'
import { HttpStatusCode, ManageVideoTorrentPayload, VideoUpdate } from '@shared/models'
import { HttpStatusCode, VideoUpdate } from '@shared/models'
import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger'
import { resetSequelizeInstance } from '../../../helpers/database-utils'
import { createReqFiles } from '../../../helpers/express-utils'
@ -18,6 +18,7 @@ import { autoBlacklistVideoIfNeeded } from '../../../lib/video-blacklist'
import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videosUpdateValidator } from '../../../middlewares'
import { ScheduleVideoUpdateModel } from '../../../models/video/schedule-video-update'
import { VideoModel } from '../../../models/video/video'
import { VideoPathManager } from '@server/lib/video-path-manager'
const lTags = loggerTagsFactory('api', 'video')
const auditLogger = auditLoggerFactory('videos')
@ -47,8 +48,8 @@ async function updateVideo (req: express.Request, res: express.Response) {
const oldVideoAuditView = new VideoAuditView(videoFromReq.toFormattedDetailsJSON())
const videoInfoToUpdate: VideoUpdate = req.body
const wasConfidentialVideo = videoFromReq.isConfidential()
const hadPrivacyForFederation = videoFromReq.hasPrivacyForFederation()
const oldPrivacy = videoFromReq.privacy
const [ thumbnailModel, previewModel ] = await buildVideoThumbnailsFromReq({
video: videoFromReq,
@ -57,12 +58,13 @@ async function updateVideo (req: express.Request, res: express.Response) {
automaticallyGenerated: false
})
const videoFileLockReleaser = await VideoPathManager.Instance.lockFiles(videoFromReq.uuid)
try {
const { videoInstanceUpdated, isNewVideo } = await sequelizeTypescript.transaction(async t => {
// Refresh video since thumbnails to prevent concurrent updates
const video = await VideoModel.loadFull(videoFromReq.id, t)
const sequelizeOptions = { transaction: t }
const oldVideoChannel = video.VideoChannel
const keysToUpdate: (keyof VideoUpdate & FilteredModelAttributes<VideoModel>)[] = [
@ -97,7 +99,7 @@ async function updateVideo (req: express.Request, res: express.Response) {
await video.setAsRefreshed(t)
}
const videoInstanceUpdated = await video.save(sequelizeOptions) as MVideoFullLight
const videoInstanceUpdated = await video.save({ transaction: t }) as MVideoFullLight
// Thumbnail & preview updates?
if (thumbnailModel) await videoInstanceUpdated.addAndSaveThumbnail(thumbnailModel, t)
@ -113,7 +115,9 @@ async function updateVideo (req: express.Request, res: express.Response) {
await videoInstanceUpdated.$set('VideoChannel', res.locals.videoChannel, { transaction: t })
videoInstanceUpdated.VideoChannel = res.locals.videoChannel
if (hadPrivacyForFederation === true) await changeVideoChannelShare(videoInstanceUpdated, oldVideoChannel, t)
if (hadPrivacyForFederation === true) {
await changeVideoChannelShare(videoInstanceUpdated, oldVideoChannel, t)
}
}
// Schedule an update in the future?
@ -139,7 +143,12 @@ async function updateVideo (req: express.Request, res: express.Response) {
Hooks.runAction('action:api.video.updated', { video: videoInstanceUpdated, body: req.body, req, res })
await addVideoJobsAfterUpdate({ video: videoInstanceUpdated, videoInfoToUpdate, wasConfidentialVideo, isNewVideo })
await addVideoJobsAfterUpdate({
video: videoInstanceUpdated,
nameChanged: !!videoInfoToUpdate.name,
oldPrivacy,
isNewVideo
})
} catch (err) {
// Force fields we want to update
// If the transaction is retried, sequelize will think the object has not changed
@ -147,6 +156,8 @@ async function updateVideo (req: express.Request, res: express.Response) {
resetSequelizeInstance(videoFromReq, videoFieldsSave)
throw err
} finally {
videoFileLockReleaser()
}
return res.type('json')
@ -164,7 +175,7 @@ async function updateVideoPrivacy (options: {
const isNewVideo = videoInstance.isNewVideo(videoInfoToUpdate.privacy)
const newPrivacy = parseInt(videoInfoToUpdate.privacy.toString(), 10)
videoInstance.setPrivacy(newPrivacy)
setVideoPrivacy(videoInstance, newPrivacy)
// Unfederate the video if the new privacy is not compatible with federation
if (hadPrivacyForFederation && !videoInstance.hasPrivacyForFederation()) {
@ -185,50 +196,3 @@ function updateSchedule (videoInstance: MVideoFullLight, videoInfoToUpdate: Vide
return ScheduleVideoUpdateModel.deleteByVideoId(videoInstance.id, transaction)
}
}
async function addVideoJobsAfterUpdate (options: {
video: MVideoFullLight
videoInfoToUpdate: VideoUpdate
wasConfidentialVideo: boolean
isNewVideo: boolean
}) {
const { video, videoInfoToUpdate, wasConfidentialVideo, isNewVideo } = options
const jobs: CreateJobArgument[] = []
if (!video.isLive && videoInfoToUpdate.name) {
for (const file of (video.VideoFiles || [])) {
const payload: ManageVideoTorrentPayload = { action: 'update-metadata', videoId: video.id, videoFileId: file.id }
jobs.push({ type: 'manage-video-torrent', payload })
}
const hls = video.getHLSPlaylist()
for (const file of (hls?.VideoFiles || [])) {
const payload: ManageVideoTorrentPayload = { action: 'update-metadata', streamingPlaylistId: hls.id, videoFileId: file.id }
jobs.push({ type: 'manage-video-torrent', payload })
}
}
jobs.push({
type: 'federate-video',
payload: {
videoUUID: video.uuid,
isNewVideo
}
})
if (wasConfidentialVideo) {
jobs.push({
type: 'notify',
payload: {
action: 'new-video',
videoUUID: video.uuid
}
})
}
return JobQueue.Instance.createSequentialJobFlow(...jobs)
}

4
server/controllers/download.ts
View file

@ -7,7 +7,7 @@ import { VideoPathManager } from '@server/lib/video-path-manager'
import { MStreamingPlaylist, MVideo, MVideoFile, MVideoFullLight } from '@server/types/models'
import { HttpStatusCode, VideoStorage, VideoStreamingPlaylistType } from '@shared/models'
import { STATIC_DOWNLOAD_PATHS } from '../initializers/constants'
import { asyncMiddleware, videosDownloadValidator } from '../middlewares'
import { asyncMiddleware, optionalAuthenticate, videosDownloadValidator } from '../middlewares'
const downloadRouter = express.Router()
@ -20,12 +20,14 @@ downloadRouter.use(
downloadRouter.use(
STATIC_DOWNLOAD_PATHS.VIDEOS + ':id-:resolution([0-9]+).:extension',
optionalAuthenticate,
asyncMiddleware(videosDownloadValidator),
asyncMiddleware(downloadVideoFile)
)
downloadRouter.use(
STATIC_DOWNLOAD_PATHS.HLS_VIDEOS + ':id-:resolution([0-9]+)-fragmented.:extension',
optionalAuthenticate,
asyncMiddleware(videosDownloadValidator),
asyncMiddleware(downloadHLSVideoFile)
)

33
server/controllers/static.ts
View file

@ -1,20 +1,34 @@
import cors from 'cors'
import express from 'express'
import { handleStaticError } from '@server/middlewares'
import {
asyncMiddleware,
ensureCanAccessPrivateVideoHLSFiles,
ensureCanAccessVideoPrivateWebTorrentFiles,
handleStaticError,
optionalAuthenticate
} from '@server/middlewares'
import { CONFIG } from '../initializers/config'
import { HLS_STREAMING_PLAYLIST_DIRECTORY, STATIC_MAX_AGE, STATIC_PATHS } from '../initializers/constants'
import { DIRECTORIES, STATIC_MAX_AGE, STATIC_PATHS } from '../initializers/constants'
const staticRouter = express.Router()
// Cors is very important to let other servers access torrent and video files
staticRouter.use(cors())
// Videos path for webseed
// WebTorrent/Classic videos
staticRouter.use(
STATIC_PATHS.WEBSEED,
express.static(CONFIG.STORAGE.VIDEOS_DIR, { fallthrough: false }),
STATIC_PATHS.PRIVATE_WEBSEED,
optionalAuthenticate,
asyncMiddleware(ensureCanAccessVideoPrivateWebTorrentFiles),
express.static(DIRECTORIES.VIDEOS.PRIVATE, { fallthrough: false }),
handleStaticError
)
staticRouter.use(
STATIC_PATHS.WEBSEED,
express.static(DIRECTORIES.VIDEOS.PUBLIC, { fallthrough: false }),
handleStaticError
)
staticRouter.use(
STATIC_PATHS.REDUNDANCY,
express.static(CONFIG.STORAGE.REDUNDANCY_DIR, { fallthrough: false }),
@ -22,9 +36,16 @@ staticRouter.use(
)
// HLS
staticRouter.use(
STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS,
optionalAuthenticate,
asyncMiddleware(ensureCanAccessPrivateVideoHLSFiles),
express.static(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, { fallthrough: false }),
handleStaticError
)
staticRouter.use(
STATIC_PATHS.STREAMING_PLAYLISTS.HLS,
express.static(HLS_STREAMING_PLAYLIST_DIRECTORY, { fallthrough: false }),
express.static(DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, { fallthrough: false }),
handleStaticError
)

13
server/helpers/ffmpeg/ffmpeg-vod.ts
View file

@ -1,14 +1,15 @@
import { MutexInterface } from 'async-mutex'
import { Job } from 'bullmq'
import { FfmpegCommand } from 'fluent-ffmpeg'
import { readFile, writeFile } from 'fs-extra'
import { dirname } from 'path'
import { VIDEO_TRANSCODING_FPS } from '@server/initializers/constants'
import { pick } from '@shared/core-utils'
import { AvailableEncoders, VideoResolution } from '@shared/models'
import { logger, loggerTagsFactory } from '../logger'
import { getFFmpeg, runCommand } from './ffmpeg-commons'
import { presetCopy, presetOnlyAudio, presetVOD } from './ffmpeg-presets'
import { computeFPS, ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS } from './ffprobe-utils'
import { VIDEO_TRANSCODING_FPS } from '@server/initializers/constants'
const lTags = loggerTagsFactory('ffmpeg')
@ -22,6 +23,10 @@ interface BaseTranscodeVODOptions {
inputPath: string
outputPath: string
// Will be released after the ffmpeg started
// To prevent a bug where the input file does not exist anymore when running ffmpeg
inputFileMutexReleaser: MutexInterface.Releaser
availableEncoders: AvailableEncoders
profile: string
@ -94,6 +99,12 @@ async function transcodeVOD (options: TranscodeVODOptions) {
command = await builders[options.type](command, options)
command.on('start', () => {
setTimeout(() => {
options.inputFileMutexReleaser()
}, 1000)
})
await runCommand({ command, job: options.job })
await fixHLSPlaylistIfNeeded(options)

6
server/helpers/upload.ts
View file

@ -1,10 +1,10 @@
import { join } from 'path'
import { RESUMABLE_UPLOAD_DIRECTORY } from '../initializers/constants'
import { DIRECTORIES } from '@server/initializers/constants'
function getResumableUploadPath (filename?: string) {
if (filename) return join(RESUMABLE_UPLOAD_DIRECTORY, filename)
if (filename) return join(DIRECTORIES.RESUMABLE_UPLOAD, filename)
return RESUMABLE_UPLOAD_DIRECTORY
return DIRECTORIES.RESUMABLE_UPLOAD
}
// ---------------------------------------------------------------------------

7
server/helpers/webtorrent.ts
View file

@ -164,7 +164,10 @@ function generateMagnetUri (
) {
const xs = videoFile.getTorrentUrl()
const announce = trackerUrls
let urlList = [ videoFile.getFileUrl(video) ]
let urlList = video.requiresAuth(video.uuid)
? []
: [ videoFile.getFileUrl(video) ]
const redundancies = videoFile.RedundancyVideos
if (isArray(redundancies)) urlList = urlList.concat(redundancies.map(r => r.fileUrl))
@ -240,6 +243,8 @@ function buildAnnounceList () {
}
function buildUrlList (video: MVideo, videoFile: MVideoFile) {
if (video.requiresAuth(video.uuid)) return []
return [ videoFile.getFileUrl(video) ]
}

37
server/initializers/constants.ts
View file

@ -662,10 +662,15 @@ const NSFW_POLICY_TYPES: { [ id: string ]: NSFWPolicyType } = {
// Express static paths (router)
const STATIC_PATHS = {
THUMBNAILS: '/static/thumbnails/',
WEBSEED: '/static/webseed/',
PRIVATE_WEBSEED: '/static/webseed/private/',
REDUNDANCY: '/static/redundancy/',
STREAMING_PLAYLISTS: {
HLS: '/static/streaming-playlists/hls'
HLS: '/static/streaming-playlists/hls',
PRIVATE_HLS: '/static/streaming-playlists/hls/private/'
}
}
const STATIC_DOWNLOAD_PATHS = {
@ -745,12 +750,32 @@ const LRU_CACHE = {
},
ACTOR_IMAGE_STATIC: {
MAX_SIZE: 500
},
STATIC_VIDEO_FILES_RIGHTS_CHECK: {
MAX_SIZE: 5000,
TTL: parseDurationToMs('10 seconds')
},
VIDEO_TOKENS: {
MAX_SIZE: 100_000,
TTL: parseDurationToMs('8 hours')
}
}
const RESUMABLE_UPLOAD_DIRECTORY = join(CONFIG.STORAGE.TMP_DIR, 'resumable-uploads')
const HLS_STREAMING_PLAYLIST_DIRECTORY = join(CONFIG.STORAGE.STREAMING_PLAYLISTS_DIR, 'hls')
const HLS_REDUNDANCY_DIRECTORY = join(CONFIG.STORAGE.REDUNDANCY_DIR, 'hls')
const DIRECTORIES = {
RESUMABLE_UPLOAD: join(CONFIG.STORAGE.TMP_DIR, 'resumable-uploads'),
HLS_STREAMING_PLAYLIST: {
PUBLIC: join(CONFIG.STORAGE.STREAMING_PLAYLISTS_DIR, 'hls'),
PRIVATE: join(CONFIG.STORAGE.STREAMING_PLAYLISTS_DIR, 'hls', 'private')
},
VIDEOS: {
PUBLIC: CONFIG.STORAGE.VIDEOS_DIR,
PRIVATE: join(CONFIG.STORAGE.VIDEOS_DIR, 'private')
},
HLS_REDUNDANCY: join(CONFIG.STORAGE.REDUNDANCY_DIR, 'hls')
}
const RESUMABLE_UPLOAD_SESSION_LIFETIME = SCHEDULER_INTERVALS_MS.REMOVE_DANGLING_RESUMABLE_UPLOADS
@ -971,9 +996,8 @@ export {
PEERTUBE_VERSION,
LAZY_STATIC_PATHS,
SEARCH_INDEX,
RESUMABLE_UPLOAD_DIRECTORY,
DIRECTORIES,
RESUMABLE_UPLOAD_SESSION_LIFETIME,
HLS_REDUNDANCY_DIRECTORY,
P2P_MEDIA_LOADER_PEER_VERSION,
ACTOR_IMAGES_SIZE,
ACCEPT_HEADERS,
@ -1007,7 +1031,6 @@ export {
VIDEO_FILTERS,
ROUTE_CACHE_LIFETIME,
SORTABLE_COLUMNS,
HLS_STREAMING_PLAYLIST_DIRECTORY,
JOB_TTL,
DEFAULT_THEME_NAME,
NSFW_POLICY_TYPES,

10
server/initializers/installer.ts
View file

@ -10,7 +10,7 @@ import { ApplicationModel } from '../models/application/application'
import { OAuthClientModel } from '../models/oauth/oauth-client'
import { applicationExist, clientsExist, usersExist } from './checker-after-init'
import { CONFIG } from './config'
import { FILES_CACHE, HLS_STREAMING_PLAYLIST_DIRECTORY, LAST_MIGRATION_VERSION, RESUMABLE_UPLOAD_DIRECTORY } from './constants'
import { DIRECTORIES, FILES_CACHE, LAST_MIGRATION_VERSION } from './constants'
import { sequelizeTypescript } from './database'
async function installApplication () {
@ -92,11 +92,13 @@ function createDirectoriesIfNotExist () {
tasks.push(ensureDir(dir))
}
// Playlist directories
tasks.push(ensureDir(HLS_STREAMING_PLAYLIST_DIRECTORY))
tasks.push(ensureDir(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE))
tasks.push(ensureDir(DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC))
tasks.push(ensureDir(DIRECTORIES.VIDEOS.PUBLIC))
tasks.push(ensureDir(DIRECTORIES.VIDEOS.PRIVATE))
// Resumable upload directory
tasks.push(ensureDir(RESUMABLE_UPLOAD_DIRECTORY))
tasks.push(ensureDir(DIRECTORIES.RESUMABLE_UPLOAD))
return Promise.all(tasks)
}

9
server/lib/auth/oauth.ts
View file

@ -95,14 +95,9 @@ async function handleOAuthToken (req: express.Request, options: { refreshTokenAu
function handleOAuthAuthenticate (
req: express.Request,
res: express.Response,
authenticateInQuery = false
res: express.Response
) {
const options = authenticateInQuery
? { allowBearerTokensInQueryString: true }
: {}
return oAuthServer.authenticate(new Request(req), new Response(res), options)
return oAuthServer.authenticate(new Request(req), new Response(res))
}
export {

2
server/lib/job-queue/handlers/manage-video-torrent.ts
View file

@ -82,7 +82,7 @@ async function loadStreamingPlaylistOrLog (streamingPlaylistId: number) {
async function loadFileOrLog (videoFileId: number) {
if (!videoFileId) return undefined
const file = await VideoFileModel.loadWithVideo(videoFileId)
const file = await VideoFileModel.load(videoFileId)
if (!file) {
logger.debug('Do not process torrent for file %d: does not exist anymore.', videoFileId)

6
server/lib/job-queue/handlers/move-to-object-storage.ts
View file

@ -3,10 +3,10 @@ import { remove } from 'fs-extra'
import { join } from 'path'
import { logger, loggerTagsFactory } from '@server/helpers/logger'
import { updateTorrentMetadata } from '@server/helpers/webtorrent'
import { CONFIG } from '@server/initializers/config'
import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants'
import { storeHLSFileFromFilename, storeWebTorrentFile } from '@server/lib/object-storage'
import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths'
import { VideoPathManager } from '@server/lib/video-path-manager'
import { moveToFailedMoveToObjectStorageState, moveToNextState } from '@server/lib/video-state'
import { VideoModel } from '@server/models/video/video'
import { VideoJobInfoModel } from '@server/models/video/video-job-info'
@ -72,9 +72,9 @@ async function moveWebTorrentFiles (video: MVideoWithAllFiles) {
for (const file of video.VideoFiles) {
if (file.storage !== VideoStorage.FILE_SYSTEM) continue
const fileUrl = await storeWebTorrentFile(file.filename)
const fileUrl = await storeWebTorrentFile(video, file)
const oldPath = join(CONFIG.STORAGE.VIDEOS_DIR, file.filename)
const oldPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, file)
await onFileMoved({ videoOrPlaylist: video, file, fileUrl, oldPath })
}
}

22
server/lib/job-queue/handlers/video-live-ending.ts
View file

@ -18,6 +18,7 @@ import { VideoStreamingPlaylistModel } from '@server/models/video/video-streamin
import { MVideo, MVideoLive, MVideoLiveSession, MVideoWithAllFiles } from '@server/types/models'
import { ThumbnailType, VideoLiveEndingPayload, VideoState } from '@shared/models'
import { logger, loggerTagsFactory } from '../../../helpers/logger'
import { VideoPathManager } from '@server/lib/video-path-manager'
const lTags = loggerTagsFactory('live', 'job')
@ -205,18 +206,27 @@ async function assignReplayFilesToVideo (options: {
const concatenatedTsFiles = await readdir(replayDirectory)
for (const concatenatedTsFile of concatenatedTsFiles) {
const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
const concatenatedTsFilePath = join(replayDirectory, concatenatedTsFile)
const probe = await ffprobePromise(concatenatedTsFilePath)
const { audioStream } = await getAudioStream(concatenatedTsFilePath, probe)
const { resolution } = await getVideoStreamDimensionsInfo(concatenatedTsFilePath, probe)
await generateHlsPlaylistResolutionFromTS({
video,
concatenatedTsFilePath,
resolution,
isAAC: audioStream?.codec_name === 'aac'
})
try {
await generateHlsPlaylistResolutionFromTS({
video,
inputFileMutexReleaser,
concatenatedTsFilePath,
resolution,
isAAC: audioStream?.codec_name === 'aac'
})
} catch (err) {
logger.error('Cannot generate HLS playlist resolution from TS files.', { err })
}
inputFileMutexReleaser()
}
return video

85
server/lib/job-queue/handlers/video-transcoding.ts
View file

@ -94,15 +94,24 @@ async function handleHLSJob (job: Job, payload: HLSTranscodingPayload, video: MV
const videoOrStreamingPlaylist = videoFileInput.getVideoOrStreamingPlaylist()
await VideoPathManager.Instance.makeAvailableVideoFile(videoFileInput.withVideoOrPlaylist(videoOrStreamingPlaylist), videoInputPath => {
return generateHlsPlaylistResolution({
video,
videoInputPath,
resolution: payload.resolution,
copyCodecs: payload.copyCodecs,
job
const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
try {
await videoFileInput.getVideo().reload()
await VideoPathManager.Instance.makeAvailableVideoFile(videoFileInput.withVideoOrPlaylist(videoOrStreamingPlaylist), videoInputPath => {
return generateHlsPlaylistResolution({
video,
videoInputPath,
inputFileMutexReleaser,
resolution: payload.resolution,
copyCodecs: payload.copyCodecs,
job
})
})
})
} finally {
inputFileMutexReleaser()
}
logger.info('HLS transcoding job for %s ended.', video.uuid, lTags(video.uuid))
@ -177,38 +186,44 @@ async function onVideoFirstWebTorrentTranscoding (
transcodeType: TranscodeVODOptionsType,
user: MUserId
) {
const { resolution, audioStream } = await videoArg.probeMaxQualityFile()
const mutexReleaser = await VideoPathManager.Instance.lockFiles(videoArg.uuid)
// Maybe the video changed in database, refresh it
const videoDatabase = await VideoModel.loadFull(videoArg.uuid)
// Video does not exist anymore
if (!videoDatabase) return undefined
try {
// Maybe the video changed in database, refresh it
const videoDatabase = await VideoModel.loadFull(videoArg.uuid)
// Video does not exist anymore
if (!videoDatabase) return undefined
// Generate HLS version of the original file
const originalFileHLSPayload = {
...payload,
const { resolution, audioStream } = await videoDatabase.probeMaxQualityFile()
hasAudio: !!audioStream,
resolution: videoDatabase.getMaxQualityFile().resolution,
// If we quick transcoded original file, force transcoding for HLS to avoid some weird playback issues
copyCodecs: transcodeType !== 'quick-transcode',
isMaxQuality: true
}
const hasHls = await createHlsJobIfEnabled(user, originalFileHLSPayload)
const hasNewResolutions = await createLowerResolutionsJobs({
video: videoDatabase,
user,
videoFileResolution: resolution,
hasAudio: !!audioStream,
type: 'webtorrent',
isNewVideo: payload.isNewVideo ?? true
})
// Generate HLS version of the original file
const originalFileHLSPayload = {
...payload,
await VideoJobInfoModel.decrease(videoDatabase.uuid, 'pendingTranscode')
hasAudio: !!audioStream,
resolution: videoDatabase.getMaxQualityFile().resolution,
// If we quick transcoded original file, force transcoding for HLS to avoid some weird playback issues
copyCodecs: transcodeType !== 'quick-transcode',
isMaxQuality: true
}
const hasHls = await createHlsJobIfEnabled(user, originalFileHLSPayload)
const hasNewResolutions = await createLowerResolutionsJobs({
video: videoDatabase,
user,
videoFileResolution: resolution,
hasAudio: !!audioStream,
type: 'webtorrent',
isNewVideo: payload.isNewVideo ?? true
})
// Move to next state if there are no other resolutions to generate
if (!hasHls && !hasNewResolutions) {
await retryTransactionWrapper(moveToNextState, { video: videoDatabase, isNewVideo: payload.isNewVideo })
await VideoJobInfoModel.decrease(videoDatabase.uuid, 'pendingTranscode')
// Move to next state if there are no other resolutions to generate
if (!hasHls && !hasNewResolutions) {
await retryTransactionWrapper(moveToNextState, { video: videoDatabase, isNewVideo: payload.isNewVideo })
}
} finally {
mutexReleaser()
}
}

9
server/lib/object-storage/videos.ts
View file

@ -1,8 +1,9 @@
import { basename, join } from 'path'
import { logger } from '@server/helpers/logger'
import { CONFIG } from '@server/initializers/config'
import { MStreamingPlaylistVideo, MVideoFile } from '@server/types/models'
import { MStreamingPlaylistVideo, MVideo, MVideoFile } from '@server/types/models'
import { getHLSDirectory } from '../paths'
import { VideoPathManager } from '../video-path-manager'
import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebTorrentObjectStorageKey } from './keys'
import { listKeysOfPrefix, lTags, makeAvailable, removeObject, removePrefix, storeObject } from './shared'
@ -30,10 +31,10 @@ function storeHLSFileFromPath (playlist: MStreamingPlaylistVideo, path: string)
// ---------------------------------------------------------------------------
function storeWebTorrentFile (filename: string) {
function storeWebTorrentFile (video: MVideo, file: MVideoFile) {
return storeObject({
inputPath: join(CONFIG.STORAGE.VIDEOS_DIR, filename),
objectStorageKey: generateWebTorrentObjectStorageKey(filename),
inputPath: VideoPathManager.Instance.getFSVideoFileOutputPath(video, file),
objectStorageKey: generateWebTorrentObjectStorageKey(file.filename),
bucketInfo: CONFIG.OBJECT_STORAGE.VIDEOS
})
}

17
server/lib/paths.ts
View file

@ -1,9 +1,10 @@
import { join } from 'path'
import { CONFIG } from '@server/initializers/config'
import { HLS_REDUNDANCY_DIRECTORY, HLS_STREAMING_PLAYLIST_DIRECTORY, VIDEO_LIVE } from '@server/initializers/constants'
import { DIRECTORIES, VIDEO_LIVE } from '@server/initializers/constants'
import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/types/models'
import { removeFragmentedMP4Ext } from '@shared/core-utils'
import { buildUUID } from '@shared/extra-utils'
import { isVideoInPrivateDirectory } from './video-privacy'
// ################## Video file name ##################
@ -17,20 +18,24 @@ function generateHLSVideoFilename (resolution: number) {
// ################## Streaming playlist ##################
function getLiveDirectory (video: MVideoUUID) {
function getLiveDirectory (video: MVideo) {
return getHLSDirectory(video)
}
function getLiveReplayBaseDirectory (video: MVideoUUID) {
function getLiveReplayBaseDirectory (video: MVideo) {
return join(getLiveDirectory(video), VIDEO_LIVE.REPLAY_DIRECTORY)
}
function getHLSDirectory (video: MVideoUUID) {
return join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid)
function getHLSDirectory (video: MVideo) {
if (isVideoInPrivateDirectory(video.privacy)) {
return join(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, video.uuid)
}
return join(DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, video.uuid)
}
function getHLSRedundancyDirectory (video: MVideoUUID) {
return join(HLS_REDUNDANCY_DIRECTORY, video.uuid)
return join(DIRECTORIES.HLS_REDUNDANCY, video.uuid)
}
function getHlsResolutionPlaylistFilename (videoFilename: string) {

66
server/lib/schedulers/update-videos-scheduler.ts
View file

@ -1,11 +1,14 @@
import { VideoModel } from '@server/models/video/video'
import { MVideoFullLight } from '@server/types/models'
import { MScheduleVideoUpdate } from '@server/types/models'
import { VideoPrivacy, VideoState } from '@shared/models'
import { logger } from '../../helpers/logger'
import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants'
import { sequelizeTypescript } from '../../initializers/database'
import { ScheduleVideoUpdateModel } from '../../models/video/schedule-video-update'
import { federateVideoIfNeeded } from '../activitypub/videos'
import { Notifier } from '../notifier'
import { addVideoJobsAfterUpdate } from '../video'
import { VideoPathManager } from '../video-path-manager'
import { setVideoPrivacy } from '../video-privacy'
import { AbstractScheduler } from './abstract-scheduler'
export class UpdateVideosScheduler extends AbstractScheduler {
@ -26,35 +29,54 @@ export class UpdateVideosScheduler extends AbstractScheduler {
if (!await ScheduleVideoUpdateModel.areVideosToUpdate()) return undefined
const schedules = await ScheduleVideoUpdateModel.listVideosToUpdate()
const publishedVideos: MVideoFullLight[] = []
for (const schedule of schedules) {
await sequelizeTypescript.transaction(async t => {
const video = await VideoModel.loadFull(schedule.videoId, t)
const videoOnly = await VideoModel.load(schedule.videoId)
const mutexReleaser = await VideoPathManager.Instance.lockFiles(videoOnly.uuid)
logger.info('Executing scheduled video update on %s.', video.uuid)
try {
const { video, published } = await this.updateAVideo(schedule)
if (schedule.privacy) {
const wasConfidentialVideo = video.isConfidential()
const isNewVideo = video.isNewVideo(schedule.privacy)
if (published) Notifier.Instance.notifyOnVideoPublishedAfterScheduledUpdate(video)
} catch (err) {
logger.error('Cannot update video', { err })
}
video.setPrivacy(schedule.privacy)
await video.save({ transaction: t })
await federateVideoIfNeeded(video, isNewVideo, t)
mutexReleaser()
}
}
if (wasConfidentialVideo) {
publishedVideos.push(video)
}
private async updateAVideo (schedule: MScheduleVideoUpdate) {
let oldPrivacy: VideoPrivacy
let isNewVideo: boolean
let published = false
const video = await sequelizeTypescript.transaction(async t => {
const video = await VideoModel.loadFull(schedule.videoId, t)
if (video.state === VideoState.TO_TRANSCODE) return
logger.info('Executing scheduled video update on %s.', video.uuid)
if (schedule.privacy) {
isNewVideo = video.isNewVideo(schedule.privacy)
oldPrivacy = video.privacy
setVideoPrivacy(video, schedule.privacy)
await video.save({ transaction: t })
if (oldPrivacy === VideoPrivacy.PRIVATE) {
published = true
}
}
await schedule.destroy({ transaction: t })
})
}
await schedule.destroy({ transaction: t })
for (const v of publishedVideos) {
Notifier.Instance.notifyOnNewVideoIfNeeded(v)
Notifier.Instance.notifyOnVideoPublishedAfterScheduledUpdate(v)
}
return video
})
await addVideoJobsAfterUpdate({ video, oldPrivacy, isNewVideo, nameChanged: false })
return { video, published }
}
static get Instance () {

4
server/lib/schedulers/videos-redundancy-scheduler.ts
View file

@ -16,7 +16,7 @@ import { VideosRedundancyStrategy } from '../../../shared/models/redundancy'
import { logger, loggerTagsFactory } from '../../helpers/logger'
import { downloadWebTorrentVideo } from '../../helpers/webtorrent'
import { CONFIG } from '../../initializers/config'
import { HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers/constants'
import { DIRECTORIES, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers/constants'
import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send'
import { getLocalVideoCacheFileActivityPubUrl, getLocalVideoCacheStreamingPlaylistActivityPubUrl } from '../activitypub/url'
@ -262,7 +262,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, strategy, lTags(video.uuid))
const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid)
const destDirectory = join(DIRECTORIES.HLS_REDUNDANCY, video.uuid)
const masterPlaylistUrl = playlist.getMasterPlaylistUrl(video)
const maxSizeKB = this.getTotalFileSizes([], [ playlist ]) / 1000

397
server/lib/transcoding/transcoding.ts
View file

@ -1,3 +1,4 @@
import { MutexInterface } from 'async-mutex'
import { Job } from 'bullmq'
import { copyFile, ensureDir, move, remove, stat } from 'fs-extra'
import { basename, extname as extnameUtil, join } from 'path'
@ -6,11 +7,13 @@ import { retryTransactionWrapper } from '@server/helpers/database-utils'
import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
import { sequelizeTypescript } from '@server/initializers/database'
import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models'
import { pick } from '@shared/core-utils'
import { VideoResolution, VideoStorage } from '../../../shared/models/videos'
import {
buildFileMetadata,
canDoQuickTranscode,
computeResolutionsToTranscode,
ffprobePromise,
getVideoStreamDuration,
getVideoStreamFPS,
transcodeVOD,
@ -33,7 +36,7 @@ import { VideoTranscodingProfilesManager } from './default-transcoding-profiles'
*/
// Optimize the original video file and replace it. The resolution is not changed.
function optimizeOriginalVideofile (options: {
async function optimizeOriginalVideofile (options: {
video: MVideoFullLight
inputVideoFile: MVideoFile
job: Job
@ -43,49 +46,61 @@ function optimizeOriginalVideofile (options: {
const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
const newExtname = '.mp4'
return VideoPathManager.Instance.makeAvailableVideoFile(inputVideoFile.withVideoOrPlaylist(video), async videoInputPath => {
const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname)
const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
const transcodeType: TranscodeVODOptionsType = await canDoQuickTranscode(videoInputPath)
? 'quick-transcode'
: 'video'
try {
await video.reload()
const resolution = buildOriginalFileResolution(inputVideoFile.resolution)
const fileWithVideoOrPlaylist = inputVideoFile.withVideoOrPlaylist(video)
const transcodeOptions: TranscodeVODOptions = {
type: transcodeType,
const result = await VideoPathManager.Instance.makeAvailableVideoFile(fileWithVideoOrPlaylist, async videoInputPath => {
const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname)
inputPath: videoInputPath,
outputPath: videoTranscodedPath,
const transcodeType: TranscodeVODOptionsType = await canDoQuickTranscode(videoInputPath)
? 'quick-transcode'
: 'video'
availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
profile: CONFIG.TRANSCODING.PROFILE,
const resolution = buildOriginalFileResolution(inputVideoFile.resolution)
resolution,
const transcodeOptions: TranscodeVODOptions = {
type: transcodeType,
job
}
inputPath: videoInputPath,
outputPath: videoTranscodedPath,
// Could be very long!
await transcodeVOD(transcodeOptions)
inputFileMutexReleaser,
// Important to do this before getVideoFilename() to take in account the new filename
inputVideoFile.resolution = resolution
inputVideoFile.extname = newExtname
inputVideoFile.filename = generateWebTorrentVideoFilename(resolution, newExtname)
inputVideoFile.storage = VideoStorage.FILE_SYSTEM
availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
profile: CONFIG.TRANSCODING.PROFILE,
const videoOutputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, inputVideoFile)
resolution,
const { videoFile } = await onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, videoOutputPath)
await remove(videoInputPath)
job
}
return { transcodeType, videoFile }
})
// Could be very long!
await transcodeVOD(transcodeOptions)
// Important to do this before getVideoFilename() to take in account the new filename
inputVideoFile.resolution = resolution
inputVideoFile.extname = newExtname
inputVideoFile.filename = generateWebTorrentVideoFilename(resolution, newExtname)
inputVideoFile.storage = VideoStorage.FILE_SYSTEM
const { videoFile } = await onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, inputVideoFile)
await remove(videoInputPath)
return { transcodeType, videoFile }
})
return result
} finally {
inputFileMutexReleaser()
}
}
// Transcode the original video file to a lower resolution compatible with WebTorrent
function transcodeNewWebTorrentResolution (options: {
async function transcodeNewWebTorrentResolution (options: {
video: MVideoFullLight
resolution: VideoResolution
job: Job
@ -95,53 +110,68 @@ function transcodeNewWebTorrentResolution (options: {
const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
const newExtname = '.mp4'
return VideoPathManager.Instance.makeAvailableVideoFile(video.getMaxQualityFile().withVideoOrPlaylist(video), async videoInputPath => {
const newVideoFile = new VideoFileModel({
resolution,
extname: newExtname,
filename: generateWebTorrentVideoFilename(resolution, newExtname),
size: 0,
videoId: video.id
const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
try {
await video.reload()
const file = video.getMaxQualityFile().withVideoOrPlaylist(video)
const result = await VideoPathManager.Instance.makeAvailableVideoFile(file, async videoInputPath => {
const newVideoFile = new VideoFileModel({
resolution,
extname: newExtname,
filename: generateWebTorrentVideoFilename(resolution, newExtname),
size: 0,
videoId: video.id
})
const videoTranscodedPath = join(transcodeDirectory, newVideoFile.filename)
const transcodeOptions = resolution === VideoResolution.H_NOVIDEO
? {
type: 'only-audio' as 'only-audio',
inputPath: videoInputPath,
outputPath: videoTranscodedPath,
inputFileMutexReleaser,
availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
profile: CONFIG.TRANSCODING.PROFILE,
resolution,
job
}
: {
type: 'video' as 'video',
inputPath: videoInputPath,
outputPath: videoTranscodedPath,
inputFileMutexReleaser,
availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
profile: CONFIG.TRANSCODING.PROFILE,
resolution,
job
}
await transcodeVOD(transcodeOptions)
return onWebTorrentVideoFileTranscoding(video, newVideoFile, videoTranscodedPath, newVideoFile)
})
const videoOutputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newVideoFile)
const videoTranscodedPath = join(transcodeDirectory, newVideoFile.filename)
const transcodeOptions = resolution === VideoResolution.H_NOVIDEO
? {
type: 'only-audio' as 'only-audio',
inputPath: videoInputPath,
outputPath: videoTranscodedPath,
availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
profile: CONFIG.TRANSCODING.PROFILE,
resolution,
job
}
: {
type: 'video' as 'video',
inputPath: videoInputPath,
outputPath: videoTranscodedPath,
availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
profile: CONFIG.TRANSCODING.PROFILE,
resolution,
job
}
await transcodeVOD(transcodeOptions)
return onWebTorrentVideoFileTranscoding(video, newVideoFile, videoTranscodedPath, videoOutputPath)
})
return result
} finally {
inputFileMutexReleaser()
}
}
// Merge an image with an audio file to create a video
function mergeAudioVideofile (options: {
async function mergeAudioVideofile (options: {
video: MVideoFullLight
resolution: VideoResolution
job: Job
@ -151,54 +181,67 @@ function mergeAudioVideofile (options: {
const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
const newExtname = '.mp4'
const inputVideoFile = video.getMinQualityFile()
const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
return VideoPathManager.Instance.makeAvailableVideoFile(inputVideoFile.withVideoOrPlaylist(video), async audioInputPath => {
const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname)
try {
await video.reload()
// If the user updates the video preview during transcoding
const previewPath = video.getPreview().getPath()
const tmpPreviewPath = join(CONFIG.STORAGE.TMP_DIR, basename(previewPath))
await copyFile(previewPath, tmpPreviewPath)
const inputVideoFile = video.getMinQualityFile()
const transcodeOptions = {
type: 'merge-audio' as 'merge-audio',
const fileWithVideoOrPlaylist = inputVideoFile.withVideoOrPlaylist(video)
inputPath: tmpPreviewPath,
outputPath: videoTranscodedPath,
const result = await VideoPathManager.Instance.makeAvailableVideoFile(fileWithVideoOrPlaylist, async audioInputPath => {
const videoTranscodedPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname)
availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
profile: CONFIG.TRANSCODING.PROFILE,
// If the user updates the video preview during transcoding
const previewPath = video.getPreview().getPath()
const tmpPreviewPath = join(CONFIG.STORAGE.TMP_DIR, basename(previewPath))
await copyFile(previewPath, tmpPreviewPath)
audioPath: audioInputPath,
resolution,
const transcodeOptions = {
type: 'merge-audio' as 'merge-audio',
job
}
inputPath: tmpPreviewPath,
outputPath: videoTranscodedPath,
try {
await transcodeVOD(transcodeOptions)
inputFileMutexReleaser,
await remove(audioInputPath)
await remove(tmpPreviewPath)
} catch (err) {
await remove(tmpPreviewPath)
throw err
}
availableEncoders: VideoTranscodingProfilesManager.Instance.getAvailableEncoders(),
profile: CONFIG.TRANSCODING.PROFILE,
// Important to do this before getVideoFilename() to take in account the new file extension
inputVideoFile.extname = newExtname
inputVideoFile.resolution = resolution
inputVideoFile.filename = generateWebTorrentVideoFilename(inputVideoFile.resolution, newExtname)
audioPath: audioInputPath,
resolution,
const videoOutputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, inputVideoFile)
// ffmpeg generated a new video file, so update the video duration
// See https://trac.ffmpeg.org/ticket/5456
video.duration = await getVideoStreamDuration(videoTranscodedPath)
await video.save()
job
}
return onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, videoOutputPath)
})
try {
await transcodeVOD(transcodeOptions)
await remove(audioInputPath)
await remove(tmpPreviewPath)
} catch (err) {
await remove(tmpPreviewPath)
throw err
}
// Important to do this before getVideoFilename() to take in account the new file extension
inputVideoFile.extname = newExtname
inputVideoFile.resolution = resolution
inputVideoFile.filename = generateWebTorrentVideoFilename(inputVideoFile.resolution, newExtname)
// ffmpeg generated a new video file, so update the video duration
// See https://trac.ffmpeg.org/ticket/5456
video.duration = await getVideoStreamDuration(videoTranscodedPath)
await video.save()
return onWebTorrentVideoFileTranscoding(video, inputVideoFile, videoTranscodedPath, inputVideoFile)
})
return result
} finally {
inputFileMutexReleaser()
}
}
// Concat TS segments from a live video to a fragmented mp4 HLS playlist
@ -207,13 +250,13 @@ async function generateHlsPlaylistResolutionFromTS (options: {
concatenatedTsFilePath: string
resolution: VideoResolution
isAAC: boolean
inputFileMutexReleaser: MutexInterface.Releaser
}) {
return generateHlsPlaylistCommon({
video: options.video,
resolution: options.resolution,
inputPath: options.concatenatedTsFilePath,
type: 'hls-from-ts' as 'hls-from-ts',
isAAC: options.isAAC
inputPath: options.concatenatedTsFilePath,
...pick(options, [ 'video', 'resolution', 'inputFileMutexReleaser', 'isAAC' ])
})
}
@ -223,15 +266,14 @@ function generateHlsPlaylistResolution (options: {
videoInputPath: string
resolution: VideoResolution
copyCodecs: boolean
inputFileMutexReleaser: MutexInterface.Releaser
job?: Job
}) {
return generateHlsPlaylistCommon({
video: options.video,
resolution: options.resolution,
copyCodecs: options.copyCodecs,
inputPath: options.videoInputPath,
type: 'hls' as 'hls',
job: options.job
inputPath: options.videoInputPath,
...pick(options, [ 'video', 'resolution', 'copyCodecs', 'inputFileMutexReleaser', 'job' ])
})
}
@ -251,27 +293,39 @@ async function onWebTorrentVideoFileTranscoding (
video: MVideoFullLight,
videoFile: MVideoFile,
transcodingPath: string,
outputPath: string
newVideoFile: MVideoFile
) {
const stats = await stat(transcodingPath)
const fps = await getVideoStreamFPS(transcodingPath)
const metadata = await buildFileMetadata(transcodingPath)
const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
await move(transcodingPath, outputPath, { overwrite: true })
try {
await video.reload()
videoFile.size = stats.size
videoFile.fps = fps
videoFile.metadata = metadata
const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newVideoFile)
await createTorrentAndSetInfoHash(video, videoFile)
const stats = await stat(transcodingPath)
const oldFile = await VideoFileModel.loadWebTorrentFile({ videoId: video.id, fps: videoFile.fps, resolution: videoFile.resolution })
if (oldFile) await video.removeWebTorrentFile(oldFile)
const probe = await ffprobePromise(transcodingPath)
const fps = await getVideoStreamFPS(transcodingPath, probe)
const metadata = await buildFileMetadata(transcodingPath, probe)
await VideoFileModel.customUpsert(videoFile, 'video', undefined)
video.VideoFiles = await video.$get('VideoFiles')
await move(transcodingPath, outputPath, { overwrite: true })
return { video, videoFile }
videoFile.size = stats.size
videoFile.fps = fps
videoFile.metadata = metadata
await createTorrentAndSetInfoHash(video, videoFile)
const oldFile = await VideoFileModel.loadWebTorrentFile({ videoId: video.id, fps: videoFile.fps, resolution: videoFile.resolution })
if (oldFile) await video.removeWebTorrentFile(oldFile)
await VideoFileModel.customUpsert(videoFile, 'video', undefined)
video.VideoFiles = await video.$get('VideoFiles')
return { video, videoFile }
} finally {
mutexReleaser()
}
}
async function generateHlsPlaylistCommon (options: {
@ -279,12 +333,15 @@ async function generateHlsPlaylistCommon (options: {
video: MVideo
inputPath: string
resolution: VideoResolution
inputFileMutexReleaser: MutexInterface.Releaser
copyCodecs?: boolean
isAAC?: boolean
job?: Job
}) {
const { type, video, inputPath, resolution, copyCodecs, isAAC, job } = options
const { type, video, inputPath, resolution, copyCodecs, isAAC, job, inputFileMutexReleaser } = options
const transcodeDirectory = CONFIG.STORAGE.TMP_DIR
const videoTranscodedBasePath = join(transcodeDirectory, type)
@ -308,6 +365,8 @@ async function generateHlsPlaylistCommon (options: {
isAAC,
inputFileMutexReleaser,
hlsPlaylist: {
videoFilename
},
@ -333,40 +392,54 @@ async function generateHlsPlaylistCommon (options: {
videoStreamingPlaylistId: playlist.id
})
const videoFilePath = VideoPathManager.Instance.getFSVideoFileOutputPath(playlist, newVideoFile)
await ensureDir(VideoPathManager.Instance.getFSHLSOutputPath(video))
const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
// Move playlist file
const resolutionPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, resolutionPlaylistFilename)
await move(resolutionPlaylistFileTranscodePath, resolutionPlaylistPath, { overwrite: true })
// Move video file
await move(join(videoTranscodedBasePath, videoFilename), videoFilePath, { overwrite: true })
try {
// VOD transcoding is a long task, refresh video attributes
await video.reload()
// Update video duration if it was not set (in case of a live for example)
if (!video.duration) {
video.duration = await getVideoStreamDuration(videoFilePath)
await video.save()
const videoFilePath = VideoPathManager.Instance.getFSVideoFileOutputPath(playlist, newVideoFile)
await ensureDir(VideoPathManager.Instance.getFSHLSOutputPath(video))
// Move playlist file
const resolutionPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, resolutionPlaylistFilename)
await move(resolutionPlaylistFileTranscodePath, resolutionPlaylistPath, { overwrite: true })
// Move video file
await move(join(videoTranscodedBasePath, videoFilename), videoFilePath, { overwrite: true })
// Update video duration if it was not set (in case of a live for example)
if (!video.duration) {
video.duration = await getVideoStreamDuration(videoFilePath)
await video.save()
}
const stats = await stat(videoFilePath)
newVideoFile.size = stats.size
newVideoFile.fps = await getVideoStreamFPS(videoFilePath)
newVideoFile.metadata = await buildFileMetadata(videoFilePath)
await createTorrentAndSetInfoHash(playlist, newVideoFile)
const oldFile = await VideoFileModel.loadHLSFile({
playlistId: playlist.id,
fps: newVideoFile.fps,
resolution: newVideoFile.resolution
})
if (oldFile) {
await video.removeStreamingPlaylistVideoFile(playlist, oldFile)
await oldFile.destroy()
}
const savedVideoFile = await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', undefined)
await updatePlaylistAfterFileChange(video, playlist)
return { resolutionPlaylistPath, videoFile: savedVideoFile }
} finally {
mutexReleaser()
}
const stats = await stat(videoFilePath)
newVideoFile.size = stats.size
newVideoFile.fps = await getVideoStreamFPS(videoFilePath)
newVideoFile.metadata = await buildFileMetadata(videoFilePath)
await createTorrentAndSetInfoHash(playlist, newVideoFile)
const oldFile = await VideoFileModel.loadHLSFile({ playlistId: playlist.id, fps: newVideoFile.fps, resolution: newVideoFile.resolution })
if (oldFile) {
await video.removeStreamingPlaylistVideoFile(playlist, oldFile)
await oldFile.destroy()
}
const savedVideoFile = await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', undefined)
await updatePlaylistAfterFileChange(video, playlist)
return { resolutionPlaylistPath, videoFile: savedVideoFile }
}
function buildOriginalFileResolution (inputResolution: number) {

51
server/lib/video-path-manager.ts
View file

@ -1,29 +1,31 @@
import { Mutex } from 'async-mutex'
import { remove } from 'fs-extra'
import { extname, join } from 'path'
import { logger, loggerTagsFactory } from '@server/helpers/logger'
import { extractVideo } from '@server/helpers/video'
import { CONFIG } from '@server/initializers/config'
import {
MStreamingPlaylistVideo,
MVideo,
MVideoFile,
MVideoFileStreamingPlaylistVideo,
MVideoFileVideo,
MVideoUUID
} from '@server/types/models'
import { DIRECTORIES } from '@server/initializers/constants'
import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoFileStreamingPlaylistVideo, MVideoFileVideo } from '@server/types/models'
import { buildUUID } from '@shared/extra-utils'
import { VideoStorage } from '@shared/models'
import { makeHLSFileAvailable, makeWebTorrentFileAvailable } from './object-storage'
import { getHLSDirectory, getHLSRedundancyDirectory, getHlsResolutionPlaylistFilename } from './paths'
import { isVideoInPrivateDirectory } from './video-privacy'
type MakeAvailableCB <T> = (path: string) => Promise<T> | T
const lTags = loggerTagsFactory('video-path-manager')
class VideoPathManager {
private static instance: VideoPathManager
// Key is a video UUID
private readonly videoFileMutexStore = new Map<string, Mutex>()
private constructor () {}
getFSHLSOutputPath (video: MVideoUUID, filename?: string) {
getFSHLSOutputPath (video: MVideo, filename?: string) {
const base = getHLSDirectory(video)
if (!filename) return base
@ -41,13 +43,17 @@ class VideoPathManager {
}
getFSVideoFileOutputPath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) {
if (videoFile.isHLS()) {
const video = extractVideo(videoOrPlaylist)
const video = extractVideo(videoOrPlaylist)
if (videoFile.isHLS()) {
return join(getHLSDirectory(video), videoFile.filename)
}
return join(CONFIG.STORAGE.VIDEOS_DIR, videoFile.filename)
if (isVideoInPrivateDirectory(video.privacy)) {
return join(DIRECTORIES.VIDEOS.PRIVATE, videoFile.filename)
}
return join(DIRECTORIES.VIDEOS.PUBLIC, videoFile.filename)
}
async makeAvailableVideoFile <T> (videoFile: MVideoFileVideo | MVideoFileStreamingPlaylistVideo, cb: MakeAvailableCB<T>) {
@ -113,6 +119,27 @@ class VideoPathManager {
)
}
async lockFiles (videoUUID: string) {
if (!this.videoFileMutexStore.has(videoUUID)) {
this.videoFileMutexStore.set(videoUUID, new Mutex())
}
const mutex = this.videoFileMutexStore.get(videoUUID)
const releaser = await mutex.acquire()
logger.debug('Locked files of %s.', videoUUID, lTags(videoUUID))
return releaser
}
unlockFiles (videoUUID: string) {
const mutex = this.videoFileMutexStore.get(videoUUID)
mutex.release()
logger.debug('Released lockfiles of %s.', videoUUID, lTags(videoUUID))
}
private async makeAvailableFactory <T> (method: () => Promise<string> | string, clean: boolean, cb: MakeAvailableCB<T>) {
let result: T

96
server/lib/video-privacy.ts Normal file
View file

@ -0,0 +1,96 @@
import { move } from 'fs-extra'
import { join } from 'path'
import { logger } from '@server/helpers/logger'
import { DIRECTORIES } from '@server/initializers/constants'
import { MVideo, MVideoFullLight } from '@server/types/models'
import { VideoPrivacy } from '@shared/models'
function setVideoPrivacy (video: MVideo, newPrivacy: VideoPrivacy) {
if (video.privacy === VideoPrivacy.PRIVATE && newPrivacy !== VideoPrivacy.PRIVATE) {
video.publishedAt = new Date()
}
video.privacy = newPrivacy
}
function isVideoInPrivateDirectory (privacy: VideoPrivacy) {
return privacy === VideoPrivacy.PRIVATE || privacy === VideoPrivacy.INTERNAL
}
function isVideoInPublicDirectory (privacy: VideoPrivacy) {
return !isVideoInPrivateDirectory(privacy)
}
async function moveFilesIfPrivacyChanged (video: MVideoFullLight, oldPrivacy: VideoPrivacy) {
// Now public, previously private
if (isVideoInPublicDirectory(video.privacy) && isVideoInPrivateDirectory(oldPrivacy)) {
await moveFiles({ type: 'private-to-public', video })
return true
}
// Now private, previously public
if (isVideoInPrivateDirectory(video.privacy) && isVideoInPublicDirectory(oldPrivacy)) {
await moveFiles({ type: 'public-to-private', video })
return true
}
return false
}
export {
setVideoPrivacy,
isVideoInPrivateDirectory,
isVideoInPublicDirectory,
moveFilesIfPrivacyChanged
}
// ---------------------------------------------------------------------------
async function moveFiles (options: {
type: 'private-to-public' | 'public-to-private'
video: MVideoFullLight
}) {
const { type, video } = options
const directories = type === 'private-to-public'
? {
webtorrent: { old: DIRECTORIES.VIDEOS.PRIVATE, new: DIRECTORIES.VIDEOS.PUBLIC },
hls: { old: DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, new: DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC }
}
: {
webtorrent: { old: DIRECTORIES.VIDEOS.PUBLIC, new: DIRECTORIES.VIDEOS.PRIVATE },
hls: { old: DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, new: DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE }
}
for (const file of video.VideoFiles) {
const source = join(directories.webtorrent.old, file.filename)
const destination = join(directories.webtorrent.new, file.filename)
try {
logger.info('Moving WebTorrent files of %s after privacy change (%s -> %s).', video.uuid, source, destination)
await move(source, destination)
} catch (err) {
logger.error('Cannot move webtorrent file %s to %s after privacy change', source, destination, { err })
}
}
const hls = video.getHLSPlaylist()
if (hls) {
const source = join(directories.hls.old, video.uuid)
const destination = join(directories.hls.new, video.uuid)
try {
logger.info('Moving HLS files of %s after privacy change (%s -> %s).', video.uuid, source, destination)
await move(source, destination)
} catch (err) {
logger.error('Cannot move HLS file %s to %s after privacy change', source, destination, { err })
}
}
}

49
server/lib/video-tokens-manager.ts Normal file
View file

@ -0,0 +1,49 @@
import LRUCache from 'lru-cache'
import { LRU_CACHE } from '@server/initializers/constants'
import { buildUUID } from '@shared/extra-utils'
// ---------------------------------------------------------------------------
// Create temporary tokens that can be used as URL query parameters to access video static files
// ---------------------------------------------------------------------------
class VideoTokensManager {
private static instance: VideoTokensManager
private readonly lruCache = new LRUCache<string, string>({
max: LRU_CACHE.VIDEO_TOKENS.MAX_SIZE,
ttl: LRU_CACHE.VIDEO_TOKENS.TTL
})
private constructor () {}
create (videoUUID: string) {
const token = buildUUID()
const expires = new Date(new Date().getTime() + LRU_CACHE.VIDEO_TOKENS.TTL)
this.lruCache.set(token, videoUUID)
return { token, expires }
}
hasToken (options: {
token: string
videoUUID: string
}) {
const value = this.lruCache.get(options.token)
if (!value) return false
return value === options.videoUUID
}
static get Instance () {
return this.instance || (this.instance = new this())
}
}
// ---------------------------------------------------------------------------
export {
VideoTokensManager
}

61
server/lib/video.ts
View file

@ -7,10 +7,11 @@ import { TagModel } from '@server/models/video/tag'
import { VideoModel } from '@server/models/video/video'
import { VideoJobInfoModel } from '@server/models/video/video-job-info'
import { FilteredModelAttributes } from '@server/types'
import { MThumbnail, MUserId, MVideoFile, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models'
import { ThumbnailType, VideoCreate, VideoPrivacy, VideoState, VideoTranscodingPayload } from '@shared/models'
import { CreateJobOptions } from './job-queue/job-queue'
import { MThumbnail, MUserId, MVideoFile, MVideoFullLight, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models'
import { ManageVideoTorrentPayload, ThumbnailType, VideoCreate, VideoPrivacy, VideoState, VideoTranscodingPayload } from '@shared/models'
import { CreateJobArgument, CreateJobOptions, JobQueue } from './job-queue/job-queue'
import { updateVideoMiniatureFromExisting } from './thumbnail'
import { moveFilesIfPrivacyChanged } from './video-privacy'
function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): FilteredModelAttributes<VideoModel> {
return {
@ -177,6 +178,59 @@ const getCachedVideoDuration = memoizee(getVideoDuration, {
// ---------------------------------------------------------------------------
async function addVideoJobsAfterUpdate (options: {
video: MVideoFullLight
isNewVideo: boolean
nameChanged: boolean
oldPrivacy: VideoPrivacy
}) {
const { video, nameChanged, oldPrivacy, isNewVideo } = options
const jobs: CreateJobArgument[] = []
const filePathChanged = await moveFilesIfPrivacyChanged(video, oldPrivacy)
if (!video.isLive && (nameChanged || filePathChanged)) {
for (const file of (video.VideoFiles || [])) {
const payload: ManageVideoTorrentPayload = { action: 'update-metadata', videoId: video.id, videoFileId: file.id }
jobs.push({ type: 'manage-video-torrent', payload })
}
const hls = video.getHLSPlaylist()
for (const file of (hls?.VideoFiles || [])) {
const payload: ManageVideoTorrentPayload = { action: 'update-metadata', streamingPlaylistId: hls.id, videoFileId: file.id }
jobs.push({ type: 'manage-video-torrent', payload })
}
}
jobs.push({
type: 'federate-video',
payload: {
videoUUID: video.uuid,
isNewVideo
}
})
const wasConfidentialVideo = new Set([ VideoPrivacy.PRIVATE, VideoPrivacy.UNLISTED, VideoPrivacy.INTERNAL ]).has(oldPrivacy)
if (wasConfidentialVideo) {
jobs.push({
type: 'notify',
payload: {
action: 'new-video',
videoUUID: video.uuid
}
})
}
return JobQueue.Instance.createSequentialJobFlow(...jobs)
}
// ---------------------------------------------------------------------------
export {
buildLocalVideoFromReq,
buildVideoThumbnailsFromReq,
@ -185,5 +239,6 @@ export {
buildTranscodingJob,
buildMoveToObjectStorageJob,
getTranscodingJobPriority,
addVideoJobsAfterUpdate,
getCachedVideoDuration
}

8
server/middlewares/auth.ts
View file

@ -5,8 +5,8 @@ import { HttpStatusCode } from '../../shared/models/http/http-error-codes'
import { logger } from '../helpers/logger'
import { handleOAuthAuthenticate } from '../lib/auth/oauth'
function authenticate (req: express.Request, res: express.Response, next: express.NextFunction, authenticateInQuery = false) {
handleOAuthAuthenticate(req, res, authenticateInQuery)
function authenticate (req: express.Request, res: express.Response, next: express.NextFunction) {
handleOAuthAuthenticate(req, res)
.then((token: any) => {
res.locals.oauth = { token }
res.locals.authenticated = true
@ -47,7 +47,7 @@ function authenticateSocket (socket: Socket, next: (err?: any) => void) {
.catch(err => logger.error('Cannot get access token.', { err }))
}
function authenticatePromise (req: express.Request, res: express.Response, authenticateInQuery = false) {
function authenticatePromise (req: express.Request, res: express.Response) {
return new Promise<void>(resolve => {
// Already authenticated? (or tried to)
if (res.locals.oauth?.token.User) return resolve()
@ -59,7 +59,7 @@ function authenticatePromise (req: express.Request, res: express.Response, authe
})
}
authenticate(req, res, () => resolve(), authenticateInQuery)
authenticate(req, res, () => resolve())
})
}

7
server/middlewares/validators/index.ts
View file

@ -1,7 +1,6 @@
export * from './activitypub'
export * from './videos'
export * from './abuse'
export * from './account'
export * from './activitypub'
export * from './actor-image'
export * from './blocklist'
export * from './bulk'
@ -10,8 +9,8 @@ export * from './express'
export * from './feeds'
export * from './follows'
export * from './jobs'
export * from './metrics'
export * from './logs'
export * from './metrics'
export * from './oembed'
export * from './pagination'
export * from './plugins'
@ -19,9 +18,11 @@ export * from './redundancy'
export * from './search'
export * from './server'
export * from './sort'
export * from './static'
export * from './themes'
export * from './user-history'
export * from './user-notifications'
export * from './user-subscriptions'
export * from './users'
export * from './videos'
export * from './webfinger'

54
server/middlewares/validators/shared/videos.ts
View file

@ -1,7 +1,7 @@
import { Request, Response } from 'express'
import { isUUIDValid } from '@server/helpers/custom-validators/misc'
import { loadVideo, VideoLoadType } from '@server/lib/model-loaders'
import { isAbleToUploadVideo } from '@server/lib/user'
import { VideoTokensManager } from '@server/lib/video-tokens-manager'
import { authenticatePromise } from '@server/middlewares/auth'
import { VideoModel } from '@server/models/video/video'
import { VideoChannelModel } from '@server/models/video/video-channel'
@ -108,26 +108,21 @@ async function checkCanSeeVideo (options: {
res: Response
paramId: string
video: MVideo
authenticateInQuery?: boolean // default false
}) {
const { req, res, video, paramId, authenticateInQuery = false } = options
const { req, res, video, paramId } = options
if (video.requiresAuth()) {
return checkCanSeeAuthVideo(req, res, video, authenticateInQuery)
if (video.requiresAuth(paramId)) {
return checkCanSeeAuthVideo(req, res, video)
}
if (video.privacy === VideoPrivacy.UNLISTED) {
if (isUUIDValid(paramId)) return true
return checkCanSeeAuthVideo(req, res, video, authenticateInQuery)
if (video.privacy === VideoPrivacy.UNLISTED || video.privacy === VideoPrivacy.PUBLIC) {
return true
}
if (video.privacy === VideoPrivacy.PUBLIC) return true
throw new Error('Fatal error when checking video right ' + video.url)
throw new Error('Unknown video privacy when checking video right ' + video.url)
}
async function checkCanSeeAuthVideo (req: Request, res: Response, video: MVideoId | MVideoWithRights, authenticateInQuery = false) {
async function checkCanSeeAuthVideo (req: Request, res: Response, video: MVideoId | MVideoWithRights) {
const fail = () => {
res.fail({
status: HttpStatusCode.FORBIDDEN_403,
@ -137,7 +132,7 @@ async function checkCanSeeAuthVideo (req: Request, res: Response, video: MVideoI
return false
}
await authenticatePromise(req, res, authenticateInQuery)
await authenticatePromise(req, res)
const user = res.locals.oauth?.token.User
if (!user) return fail()
@ -173,6 +168,36 @@ async function checkCanSeeAuthVideo (req: Request, res: Response, video: MVideoI
// ---------------------------------------------------------------------------
async function checkCanAccessVideoStaticFiles (options: {
video: MVideo
req: Request
res: Response
paramId: string
}) {
const { video, req, res, paramId } = options
if (res.locals.oauth?.token.User) {
return checkCanSeeVideo(options)
}
if (!video.requiresAuth(paramId)) return true
const videoFileToken = req.query.videoFileToken
if (!videoFileToken) {
res.sendStatus(HttpStatusCode.FORBIDDEN_403)
return false
}
if (VideoTokensManager.Instance.hasToken({ token: videoFileToken, videoUUID: video.uuid })) {
return true
}
res.sendStatus(HttpStatusCode.FORBIDDEN_403)
return false
}
// ---------------------------------------------------------------------------
function checkUserCanManageVideo (user: MUser, video: MVideoAccountLight, right: UserRight, res: Response, onlyOwned = true) {
// Retrieve the user who did the request
if (onlyOwned && video.isOwned() === false) {
@ -220,6 +245,7 @@ export {
doesVideoExist,
doesVideoFileOfVideoExist,
checkCanAccessVideoStaticFiles,
checkUserCanManageVideo,
checkCanSeeVideo,
checkUserQuota

131
server/middlewares/validators/static.ts Normal file
View file

@ -0,0 +1,131 @@
import express from 'express'
import { query } from 'express-validator'
import LRUCache from 'lru-cache'
import { basename, dirname } from 'path'
import { exists, isUUIDValid } from '@server/helpers/custom-validators/misc'
import { logger } from '@server/helpers/logger'
import { LRU_CACHE } from '@server/initializers/constants'
import { VideoModel } from '@server/models/video/video'
import { VideoFileModel } from '@server/models/video/video-file'
import { HttpStatusCode } from '@shared/models'
import { areValidationErrors, checkCanAccessVideoStaticFiles } from './shared'
const staticFileTokenBypass = new LRUCache<string, boolean>({
max: LRU_CACHE.STATIC_VIDEO_FILES_RIGHTS_CHECK.MAX_SIZE,
ttl: LRU_CACHE.STATIC_VIDEO_FILES_RIGHTS_CHECK.TTL
})
const ensureCanAccessVideoPrivateWebTorrentFiles = [
query('videoFileToken').optional().custom(exists),
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
if (areValidationErrors(req, res)) return
const token = extractTokenOrDie(req, res)
if (!token) return
const cacheKey = token + '-' + req.originalUrl
if (staticFileTokenBypass.has(cacheKey)) {
const allowedFromCache = staticFileTokenBypass.get(cacheKey)
if (allowedFromCache === true) return next()
return res.sendStatus(HttpStatusCode.FORBIDDEN_403)
}
const allowed = await isWebTorrentAllowed(req, res)
staticFileTokenBypass.set(cacheKey, allowed)
if (allowed !== true) return
return next()
}
]
const ensureCanAccessPrivateVideoHLSFiles = [
query('videoFileToken').optional().custom(exists),
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
if (areValidationErrors(req, res)) return
const videoUUID = basename(dirname(req.originalUrl))
if (!isUUIDValid(videoUUID)) {
logger.debug('Path does not contain valid video UUID to serve static file %s', req.originalUrl)
return res.sendStatus(HttpStatusCode.FORBIDDEN_403)
}
const token = extractTokenOrDie(req, res)
if (!token) return
const cacheKey = token + '-' + videoUUID
if (staticFileTokenBypass.has(cacheKey)) {
const allowedFromCache = staticFileTokenBypass.get(cacheKey)
if (allowedFromCache === true) return next()
return res.sendStatus(HttpStatusCode.FORBIDDEN_403)
}
const allowed = await isHLSAllowed(req, res, videoUUID)
staticFileTokenBypass.set(cacheKey, allowed)
if (allowed !== true) return
return next()
}
]
export {
ensureCanAccessVideoPrivateWebTorrentFiles,
ensureCanAccessPrivateVideoHLSFiles
}
// ---------------------------------------------------------------------------
async function isWebTorrentAllowed (req: express.Request, res: express.Response) {
const filename = basename(req.path)
const file = await VideoFileModel.loadWithVideoByFilename(filename)
if (!file) {
logger.debug('Unknown static file %s to serve', req.originalUrl, { filename })
res.sendStatus(HttpStatusCode.FORBIDDEN_403)
return false
}
const video = file.getVideo()
return checkCanAccessVideoStaticFiles({ req, res, video, paramId: video.uuid })
}
async function isHLSAllowed (req: express.Request, res: express.Response, videoUUID: string) {
const video = await VideoModel.load(videoUUID)
if (!video) {
logger.debug('Unknown static file %s to serve', req.originalUrl, { videoUUID })
res.sendStatus(HttpStatusCode.FORBIDDEN_403)
return false
}
return checkCanAccessVideoStaticFiles({ req, res, video, paramId: video.uuid })
}
function extractTokenOrDie (req: express.Request, res: express.Response) {
const token = res.locals.oauth?.token.accessToken || req.query.videoFileToken
if (!token) {
return res.fail({
message: 'Bearer token is missing in headers or video file token is missing in URL query parameters',
status: HttpStatusCode.FORBIDDEN_403
})
}
return token
}

33
server/middlewares/validators/videos/videos.ts
View file

@ -7,7 +7,7 @@ import { getServerActor } from '@server/models/application/application'
import { ExpressPromiseHandler } from '@server/types/express-handler'
import { MUserAccountId, MVideoFullLight } from '@server/types/models'
import { arrayify, getAllPrivacies } from '@shared/core-utils'
import { HttpStatusCode, ServerErrorCode, UserRight, VideoInclude } from '@shared/models'
import { HttpStatusCode, ServerErrorCode, UserRight, VideoInclude, VideoState } from '@shared/models'
import {
exists,
isBooleanValid,
@ -48,6 +48,7 @@ import { Hooks } from '../../../lib/plugins/hooks'
import { VideoModel } from '../../../models/video/video'
import {
areValidationErrors,
checkCanAccessVideoStaticFiles,
checkCanSeeVideo,
checkUserCanManageVideo,
checkUserQuota,
@ -232,6 +233,11 @@ const videosUpdateValidator = getCommonVideoEditAttributes().concat([
if (areErrorsInScheduleUpdate(req, res)) return cleanUpReqFiles(req)
if (!await doesVideoExist(req.params.id, res)) return cleanUpReqFiles(req)
const video = getVideoWithAttributes(res)
if (req.body.privacy && video.isLive && video.state !== VideoState.WAITING_FOR_LIVE) {
return res.fail({ message: 'Cannot update privacy of a live that has already started' })
}
// Check if the user who did the request is able to update the video
const user = res.locals.oauth.token.User
if (!checkUserCanManageVideo(user, res.locals.videoAll, UserRight.UPDATE_ANY_VIDEO, res)) return cleanUpReqFiles(req)
@ -271,10 +277,7 @@ async function checkVideoFollowConstraints (req: express.Request, res: express.R
})
}
const videosCustomGetValidator = (
fetchType: 'for-api' | 'all' | 'only-video' | 'only-immutable-attributes',
authenticateInQuery = false
) => {
const videosCustomGetValidator = (fetchType: 'for-api' | 'all' | 'only-video' | 'only-immutable-attributes') => {
return [
isValidVideoIdParam('id'),
@ -287,7 +290,7 @@ const videosCustomGetValidator = (
const video = getVideoWithAttributes(res) as MVideoFullLight
if (!await checkCanSeeVideo({ req, res, video, paramId: req.params.id, authenticateInQuery })) return
if (!await checkCanSeeVideo({ req, res, video, paramId: req.params.id })) return
return next()
}
@ -295,7 +298,6 @@ const videosCustomGetValidator = (
}
const videosGetValidator = videosCustomGetValidator('all')
const videosDownloadValidator = videosCustomGetValidator('all', true)
const videoFileMetadataGetValidator = getCommonVideoEditAttributes().concat([
isValidVideoIdParam('id'),
@ -311,6 +313,21 @@ const videoFileMetadataGetValidator = getCommonVideoEditAttributes().concat([
}
])
const videosDownloadValidator = [
isValidVideoIdParam('id'),
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
if (areValidationErrors(req, res)) return
if (!await doesVideoExist(req.params.id, res, 'all')) return
const video = getVideoWithAttributes(res)
if (!await checkCanAccessVideoStaticFiles({ req, res, video, paramId: req.params.id })) return
return next()
}
]
const videosRemoveValidator = [
isValidVideoIdParam('id'),
@ -372,7 +389,7 @@ function getCommonVideoEditAttributes () {
.custom(isBooleanValid).withMessage('Should have a valid waitTranscoding boolean'),
body('privacy')
.optional()
.customSanitizer(toValueOrNull)
.customSanitizer(toIntOrNull)
.custom(isVideoPrivacyValid),
body('description')
.optional()

22
server/models/video/formatter/video-format-utils.ts
View file

@ -34,6 +34,7 @@ import {
import {
MServer,
MStreamingPlaylistRedundanciesOpt,
MUserId,
MVideo,
MVideoAP,
MVideoFile,
@ -245,8 +246,12 @@ function sortByResolutionDesc (fileA: MVideoFile, fileB: MVideoFile) {
function videoFilesModelToFormattedJSON (
video: MVideoFormattable,
videoFiles: MVideoFileRedundanciesOpt[],
includeMagnet = true
options: {
includeMagnet?: boolean // default true
} = {}
): VideoFile[] {
const { includeMagnet = true } = options
const trackerUrls = includeMagnet
? video.getTrackerUrls()
: []
@ -281,11 +286,14 @@ function videoFilesModelToFormattedJSON (
})
}
function addVideoFilesInAPAcc (
acc: ActivityUrlObject[] | ActivityTagObject[],
video: MVideo,
function addVideoFilesInAPAcc (options: {
acc: ActivityUrlObject[] | ActivityTagObject[]
video: MVideo
files: MVideoFile[]
) {
user?: MUserId
}) {
const { acc, video, files } = options
const trackerUrls = video.getTrackerUrls()
const sortedFiles = (files || [])
@ -370,7 +378,7 @@ function videoModelToActivityPubObject (video: MVideoAP): VideoObject {
}
]
addVideoFilesInAPAcc(url, video, video.VideoFiles || [])
addVideoFilesInAPAcc({ acc: url, video, files: video.VideoFiles || [] })
for (const playlist of (video.VideoStreamingPlaylists || [])) {
const tag = playlist.p2pMediaLoaderInfohashes
@ -382,7 +390,7 @@ function videoModelToActivityPubObject (video: MVideoAP): VideoObject {
href: playlist.getSha256SegmentsUrl(video)
})
addVideoFilesInAPAcc(tag, video, playlist.VideoFiles || [])
addVideoFilesInAPAcc({ acc: tag, video, files: playlist.VideoFiles || [] })
url.push({
type: 'Link',

29
server/models/video/video-file.ts
View file

@ -24,6 +24,7 @@ import { extractVideo } from '@server/helpers/video'
import { buildRemoteVideoBaseUrl } from '@server/lib/activitypub/url'
import { getHLSPublicFileUrl, getWebTorrentPublicFileUrl } from '@server/lib/object-storage'
import { getFSTorrentFilePath } from '@server/lib/paths'
import { isVideoInPrivateDirectory } from '@server/lib/video-privacy'
import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoWithHost } from '@server/types/models'
import { VideoResolution, VideoStorage } from '@shared/models'
import { AttributesOnly } from '@shared/typescript-utils'
@ -295,6 +296,16 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
return VideoFileModel.findOne(query)
}
static loadWithVideoByFilename (filename: string): Promise<MVideoFileVideo | MVideoFileStreamingPlaylistVideo> {
const query = {
where: {
filename
}
}
return VideoFileModel.scope(ScopeNames.WITH_VIDEO_OR_PLAYLIST).findOne(query)
}
static loadWithVideoOrPlaylistByTorrentFilename (filename: string) {
const query = {
where: {
@ -305,6 +316,10 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
return VideoFileModel.scope(ScopeNames.WITH_VIDEO_OR_PLAYLIST).findOne(query)
}
static load (id: number): Promise<MVideoFile> {
return VideoFileModel.findByPk(id)
}
static loadWithMetadata (id: number) {
return VideoFileModel.scope(ScopeNames.WITH_METADATA).findByPk(id)
}
@ -467,7 +482,7 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
}
getVideoOrStreamingPlaylist (this: MVideoFileVideo | MVideoFileStreamingPlaylistVideo): MVideo | MStreamingPlaylistVideo {
if (this.videoId) return (this as MVideoFileVideo).Video
if (this.videoId || (this as MVideoFileVideo).Video) return (this as MVideoFileVideo).Video
return (this as MVideoFileStreamingPlaylistVideo).VideoStreamingPlaylist
}
@ -508,7 +523,17 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
}
getFileStaticPath (video: MVideo) {
if (this.isHLS()) return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, video.uuid, this.filename)
if (this.isHLS()) {
if (isVideoInPrivateDirectory(video.privacy)) {
return join(STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS, video.uuid, this.filename)
}
return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, video.uuid, this.filename)
}
if (isVideoInPrivateDirectory(video.privacy)) {
return join(STATIC_PATHS.PRIVATE_WEBSEED, this.filename)
}
return join(STATIC_PATHS.WEBSEED, this.filename)
}

21
server/models/video/video-streaming-playlist.ts
View file

@ -17,6 +17,7 @@ import {
} from 'sequelize-typescript'
import { getHLSPublicFileUrl } from '@server/lib/object-storage'
import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename } from '@server/lib/paths'
import { isVideoInPrivateDirectory } from '@server/lib/video-privacy'
import { VideoFileModel } from '@server/models/video/video-file'
import { MStreamingPlaylist, MStreamingPlaylistFilesVideo, MVideo } from '@server/types/models'
import { sha1 } from '@shared/extra-utils'
@ -250,7 +251,7 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
return getHLSPublicFileUrl(this.playlistUrl)
}
return WEBSERVER.URL + this.getMasterPlaylistStaticPath(video.uuid)
return WEBSERVER.URL + this.getMasterPlaylistStaticPath(video)
}
return this.playlistUrl
@ -262,7 +263,7 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
return getHLSPublicFileUrl(this.segmentsSha256Url)
}
return WEBSERVER.URL + this.getSha256SegmentsStaticPath(video.uuid)
return WEBSERVER.URL + this.getSha256SegmentsStaticPath(video)
}
return this.segmentsSha256Url
@ -287,11 +288,19 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
return Object.assign(this, { Video: video })
}
private getMasterPlaylistStaticPath (videoUUID: string) {
return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, videoUUID, this.playlistFilename)
private getMasterPlaylistStaticPath (video: MVideo) {
if (isVideoInPrivateDirectory(video.privacy)) {
return join(STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS, video.uuid, this.playlistFilename)
}
return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, video.uuid, this.playlistFilename)
}
private getSha256SegmentsStaticPath (videoUUID: string) {
return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, videoUUID, this.segmentsSha256Filename)
private getSha256SegmentsStaticPath (video: MVideo) {
if (isVideoInPrivateDirectory(video.privacy)) {
return join(STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS, video.uuid, this.segmentsSha256Filename)
}
return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, video.uuid, this.segmentsSha256Filename)
}
}

24
server/models/video/video.ts
View file

@ -52,7 +52,7 @@ import {
import { AttributesOnly } from '@shared/typescript-utils'
import { peertubeTruncate } from '../../helpers/core-utils'
import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
import { exists, isBooleanValid } from '../../helpers/custom-validators/misc'
import { exists, isBooleanValid, isUUIDValid } from '../../helpers/custom-validators/misc'
import {
isVideoDescriptionValid,
isVideoDurationValid,
@ -1696,12 +1696,12 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
let files: VideoFile[] = []
if (Array.isArray(this.VideoFiles)) {
const result = videoFilesModelToFormattedJSON(this, this.VideoFiles, includeMagnet)
const result = videoFilesModelToFormattedJSON(this, this.VideoFiles, { includeMagnet })
files = files.concat(result)
}
for (const p of (this.VideoStreamingPlaylists || [])) {
const result = videoFilesModelToFormattedJSON(this, p.VideoFiles, includeMagnet)
const result = videoFilesModelToFormattedJSON(this, p.VideoFiles, { includeMagnet })
files = files.concat(result)
}
@ -1868,22 +1868,14 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
return setAsUpdated('video', this.id, transaction)
}
requiresAuth () {
return this.privacy === VideoPrivacy.PRIVATE || this.privacy === VideoPrivacy.INTERNAL || !!this.VideoBlacklist
}
requiresAuth (paramId: string) {
if (this.privacy === VideoPrivacy.UNLISTED) {
if (!isUUIDValid(paramId)) return true
setPrivacy (newPrivacy: VideoPrivacy) {
if (this.privacy === VideoPrivacy.PRIVATE && newPrivacy !== VideoPrivacy.PRIVATE) {
this.publishedAt = new Date()
return false
}
this.privacy = newPrivacy
}
isConfidential () {
return this.privacy === VideoPrivacy.PRIVATE ||
this.privacy === VideoPrivacy.UNLISTED ||
this.privacy === VideoPrivacy.INTERNAL
return this.privacy === VideoPrivacy.PRIVATE || this.privacy === VideoPrivacy.INTERNAL || !!this.VideoBlacklist
}
async setNewState (newState: VideoState, isNewVideo: boolean, transaction: Transaction) {

1
server/tests/api/check-params/index.ts
View file

@ -34,6 +34,7 @@ import './video-imports'
import './video-playlists'
import './video-source'
import './video-studio'
import './video-token'
import './videos-common-filters'
import './videos-history'
import './videos-overviews'

17
server/tests/api/check-params/live.ts
View file

@ -502,6 +502,23 @@ describe('Test video lives API validator', function () {
await stopFfmpeg(ffmpegCommand)
})
it('Should fail to change live privacy if it has already started', async function () {
this.timeout(40000)
const live = await command.get({ videoId: video.id })
const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: live.rtmpUrl, streamKey: live.streamKey })
await command.waitUntilPublished({ videoId: video.id })
await server.videos.update({
id: video.id,
attributes: { privacy: VideoPrivacy.PUBLIC },
expectedStatus: HttpStatusCode.BAD_REQUEST_400
})
await stopFfmpeg(ffmpegCommand)
})
it('Should fail to stream twice in the save live', async function () {
this.timeout(40000)

217
server/tests/api/check-params/video-files.ts
View file

@ -1,10 +1,12 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import { HttpStatusCode, UserRole } from '@shared/models'
import { getAllFiles } from '@shared/core-utils'
import { HttpStatusCode, UserRole, VideoDetails, VideoPrivacy } from '@shared/models'
import {
cleanupTests,
createMultipleServers,
doubleFollow,
makeRawRequest,
PeerTubeServer,
setAccessTokensToServers,
waitJobs
@ -13,22 +15,9 @@ import {
describe('Test videos files', function () {
let servers: PeerTubeServer[]
let webtorrentId: string
let hlsId: string
let remoteId: string
let userToken: string
let moderatorToken: string
let validId1: string
let validId2: string
let hlsFileId: number
let webtorrentFileId: number
let remoteHLSFileId: number
let remoteWebtorrentFileId: number
// ---------------------------------------------------------------
before(async function () {
@ -41,117 +30,163 @@ describe('Test videos files', function () {
userToken = await servers[0].users.generateUserAndToken('user', UserRole.USER)
moderatorToken = await servers[0].users.generateUserAndToken('moderator', UserRole.MODERATOR)
})
{
const { uuid } = await servers[1].videos.quickUpload({ name: 'remote video' })
await waitJobs(servers)
describe('Getting metadata', function () {
let video: VideoDetails
const video = await servers[1].videos.get({ id: uuid })
remoteId = video.uuid
remoteHLSFileId = video.streamingPlaylists[0].files[0].id
remoteWebtorrentFileId = video.files[0].id
}
before(async function () {
const { uuid } = await servers[0].videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE })
video = await servers[0].videos.getWithToken({ id: uuid })
})
{
await servers[0].config.enableTranscoding(true, true)
it('Should not get metadata of private video without token', async function () {
for (const file of getAllFiles(video)) {
await makeRawRequest({ url: file.metadataUrl, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 })
}
})
it('Should not get metadata of private video without the appropriate token', async function () {
for (const file of getAllFiles(video)) {
await makeRawRequest({ url: file.metadataUrl, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
}
})
it('Should get metadata of private video with the appropriate token', async function () {
for (const file of getAllFiles(video)) {
await makeRawRequest({ url: file.metadataUrl, token: servers[0].accessToken, expectedStatus: HttpStatusCode.OK_200 })
}
})
})
describe('Deleting files', function () {
let webtorrentId: string
let hlsId: string
let remoteId: string
let validId1: string
let validId2: string
let hlsFileId: number
let webtorrentFileId: number
let remoteHLSFileId: number
let remoteWebtorrentFileId: number
before(async function () {
this.timeout(300_000)
{
const { uuid } = await servers[0].videos.quickUpload({ name: 'both 1' })
const { uuid } = await servers[1].videos.quickUpload({ name: 'remote video' })
await waitJobs(servers)
const video = await servers[0].videos.get({ id: uuid })
validId1 = video.uuid
hlsFileId = video.streamingPlaylists[0].files[0].id
webtorrentFileId = video.files[0].id
const video = await servers[1].videos.get({ id: uuid })
remoteId = video.uuid
remoteHLSFileId = video.streamingPlaylists[0].files[0].id
remoteWebtorrentFileId = video.files[0].id
}
{
const { uuid } = await servers[0].videos.quickUpload({ name: 'both 2' })
validId2 = uuid
await servers[0].config.enableTranscoding(true, true)
{
const { uuid } = await servers[0].videos.quickUpload({ name: 'both 1' })
await waitJobs(servers)
const video = await servers[0].videos.get({ id: uuid })
validId1 = video.uuid
hlsFileId = video.streamingPlaylists[0].files[0].id
webtorrentFileId = video.files[0].id
}
{
const { uuid } = await servers[0].videos.quickUpload({ name: 'both 2' })
validId2 = uuid
}
}
}
await waitJobs(servers)
await waitJobs(servers)
{
await servers[0].config.enableTranscoding(false, true)
const { uuid } = await servers[0].videos.quickUpload({ name: 'hls' })
hlsId = uuid
}
{
await servers[0].config.enableTranscoding(false, true)
const { uuid } = await servers[0].videos.quickUpload({ name: 'hls' })
hlsId = uuid
}
await waitJobs(servers)
await waitJobs(servers)
{
await servers[0].config.enableTranscoding(false, true)
const { uuid } = await servers[0].videos.quickUpload({ name: 'webtorrent' })
webtorrentId = uuid
}
{
await servers[0].config.enableTranscoding(false, true)
const { uuid } = await servers[0].videos.quickUpload({ name: 'webtorrent' })
webtorrentId = uuid
}
await waitJobs(servers)
})
await waitJobs(servers)
})
it('Should not delete files of a unknown video', async function () {
const expectedStatus = HttpStatusCode.NOT_FOUND_404
it('Should not delete files of a unknown video', async function () {
const expectedStatus = HttpStatusCode.NOT_FOUND_404
await servers[0].videos.removeHLSPlaylist({ videoId: 404, expectedStatus })
await servers[0].videos.removeAllWebTorrentFiles({ videoId: 404, expectedStatus })
await servers[0].videos.removeHLSPlaylist({ videoId: 404, expectedStatus })
await servers[0].videos.removeAllWebTorrentFiles({ videoId: 404, expectedStatus })
await servers[0].videos.removeHLSFile({ videoId: 404, fileId: hlsFileId, expectedStatus })
await servers[0].videos.removeWebTorrentFile({ videoId: 404, fileId: webtorrentFileId, expectedStatus })
})
await servers[0].videos.removeHLSFile({ videoId: 404, fileId: hlsFileId, expectedStatus })
await servers[0].videos.removeWebTorrentFile({ videoId: 404, fileId: webtorrentFileId, expectedStatus })
})
it('Should not delete unknown files', async function () {
const expectedStatus = HttpStatusCode.NOT_FOUND_404
it('Should not delete unknown files', async function () {
const expectedStatus = HttpStatusCode.NOT_FOUND_404
await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: webtorrentFileId, expectedStatus })
await servers[0].videos.removeWebTorrentFile({ videoId: validId1, fileId: hlsFileId, expectedStatus })
})
await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: webtorrentFileId, expectedStatus })
await servers[0].videos.removeWebTorrentFile({ videoId: validId1, fileId: hlsFileId, expectedStatus })
})
it('Should not delete files of a remote video', async function () {
const expectedStatus = HttpStatusCode.BAD_REQUEST_400
it('Should not delete files of a remote video', async function () {
const expectedStatus = HttpStatusCode.BAD_REQUEST_400
await servers[0].videos.removeHLSPlaylist({ videoId: remoteId, expectedStatus })
await servers[0].videos.removeAllWebTorrentFiles({ videoId: remoteId, expectedStatus })
await servers[0].videos.removeHLSPlaylist({ videoId: remoteId, expectedStatus })
await servers[0].videos.removeAllWebTorrentFiles({ videoId: remoteId, expectedStatus })
await servers[0].videos.removeHLSFile({ videoId: remoteId, fileId: remoteHLSFileId, expectedStatus })
await servers[0].videos.removeWebTorrentFile({ videoId: remoteId, fileId: remoteWebtorrentFileId, expectedStatus })
})
await servers[0].videos.removeHLSFile({ videoId: remoteId, fileId: remoteHLSFileId, expectedStatus })
await servers[0].videos.removeWebTorrentFile({ videoId: remoteId, fileId: remoteWebtorrentFileId, expectedStatus })
})
it('Should not delete files by a non admin user', async function () {
const expectedStatus = HttpStatusCode.FORBIDDEN_403
it('Should not delete files by a non admin user', async function () {
const expectedStatus = HttpStatusCode.FORBIDDEN_403
await servers[0].videos.removeHLSPlaylist({ videoId: validId1, token: userToken, expectedStatus })
await servers[0].videos.removeHLSPlaylist({ videoId: validId1, token: moderatorToken, expectedStatus })
await servers[0].videos.removeHLSPlaylist({ videoId: validId1, token: userToken, expectedStatus })
await servers[0].videos.removeHLSPlaylist({ videoId: validId1, token: moderatorToken, expectedStatus })
await servers[0].videos.removeAllWebTorrentFiles({ videoId: validId1, token: userToken, expectedStatus })
await servers[0].videos.removeAllWebTorrentFiles({ videoId: validId1, token: moderatorToken, expectedStatus })
await servers[0].videos.removeAllWebTorrentFiles({ videoId: validId1, token: userToken, expectedStatus })
await servers[0].videos.removeAllWebTorrentFiles({ videoId: validId1, token: moderatorToken, expectedStatus })
await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: hlsFileId, token: userToken, expectedStatus })
await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: hlsFileId, token: moderatorToken, expectedStatus })
await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: hlsFileId, token: userToken, expectedStatus })
await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: hlsFileId, token: moderatorToken, expectedStatus })
await servers[0].videos.removeWebTorrentFile({ videoId: validId1, fileId: webtorrentFileId, token: userToken, expectedStatus })
await servers[0].videos.removeWebTorrentFile({ videoId: validId1, fileId: webtorrentFileId, token: moderatorToken, expectedStatus })
})
await servers[0].videos.removeWebTorrentFile({ videoId: validId1, fileId: webtorrentFileId, token: userToken, expectedStatus })
await servers[0].videos.removeWebTorrentFile({ videoId: validId1, fileId: webtorrentFileId, token: moderatorToken, expectedStatus })
})
it('Should not delete files if the files are not available', async function () {
await servers[0].videos.removeHLSPlaylist({ videoId: hlsId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 })
await servers[0].videos.removeAllWebTorrentFiles({ videoId: webtorrentId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 })
it('Should not delete files if the files are not available', async function () {
await servers[0].videos.removeHLSPlaylist({ videoId: hlsId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 })
await servers[0].videos.removeAllWebTorrentFiles({ videoId: webtorrentId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 })
await servers[0].videos.removeHLSFile({ videoId: hlsId, fileId: 404, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
await servers[0].videos.removeWebTorrentFile({ videoId: webtorrentId, fileId: 404, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
})
await servers[0].videos.removeHLSFile({ videoId: hlsId, fileId: 404, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
await servers[0].videos.removeWebTorrentFile({ videoId: webtorrentId, fileId: 404, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
})
it('Should not delete files if no both versions are available', async function () {
await servers[0].videos.removeHLSPlaylist({ videoId: hlsId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 })
await servers[0].videos.removeAllWebTorrentFiles({ videoId: webtorrentId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 })
})
it('Should not delete files if no both versions are available', async function () {
await servers[0].videos.removeHLSPlaylist({ videoId: hlsId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 })
await servers[0].videos.removeAllWebTorrentFiles({ videoId: webtorrentId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 })
})
it('Should delete files if both versions are available', async function () {
await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: hlsFileId })
await servers[0].videos.removeWebTorrentFile({ videoId: validId1, fileId: webtorrentFileId })
it('Should delete files if both versions are available', async function () {
await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: hlsFileId })
await servers[0].videos.removeWebTorrentFile({ videoId: validId1, fileId: webtorrentFileId })
await servers[0].videos.removeHLSPlaylist({ videoId: validId1 })
await servers[0].videos.removeAllWebTorrentFiles({ videoId: validId2 })
await servers[0].videos.removeHLSPlaylist({ videoId: validId1 })
await servers[0].videos.removeAllWebTorrentFiles({ videoId: validId2 })
})
})
after(async function () {

44
server/tests/api/check-params/video-token.ts Normal file
View file

@ -0,0 +1,44 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import { HttpStatusCode, VideoPrivacy } from '@shared/models'
import { cleanupTests, createSingleServer, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands'
describe('Test video tokens', function () {
let server: PeerTubeServer
let videoId: string
let userToken: string
// ---------------------------------------------------------------
before(async function () {
this.timeout(300_000)
server = await createSingleServer(1)
await setAccessTokensToServers([ server ])
const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE })
videoId = uuid
userToken = await server.users.generateUserAndToken('user1')
})
it('Should not generate tokens for unauthenticated user', async function () {
await server.videoToken.create({ videoId, token: null, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 })
})
it('Should not generate tokens of unknown video', async function () {
await server.videoToken.create({ videoId: 404, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
})
it('Should not generate tokens of a non owned video', async function () {
await server.videoToken.create({ videoId, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
})
it('Should generate token', async function () {
await server.videoToken.create({ videoId })
})
after(async function () {
await cleanupTests([ server ])
})
})

4
server/tests/api/live/live-fast-restream.ts
View file

@ -79,8 +79,8 @@ describe('Fast restream in live', function () {
expect(video.streamingPlaylists).to.have.lengthOf(1)
await server.live.getSegmentFile({ videoUUID: liveId, segment: 0, playlistNumber: 0 })
await makeRawRequest(video.streamingPlaylists[0].playlistUrl, HttpStatusCode.OK_200)
await makeRawRequest(video.streamingPlaylists[0].segmentsSha256Url, HttpStatusCode.OK_200)
await makeRawRequest({ url: video.streamingPlaylists[0].playlistUrl, expectedStatus: HttpStatusCode.OK_200 })
await makeRawRequest({ url: video.streamingPlaylists[0].segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 })
await wait(100)
}

13
server/tests/api/live/live.ts
View file

@ -21,6 +21,7 @@ import {
doubleFollow,
killallServers,
LiveCommand,
makeGetRequest,
makeRawRequest,
PeerTubeServer,
sendRTMPStream,
@ -157,8 +158,8 @@ describe('Test live', function () {
expect(video.privacy.id).to.equal(VideoPrivacy.UNLISTED)
expect(video.nsfw).to.be.true
await makeRawRequest(server.url + video.thumbnailPath, HttpStatusCode.OK_200)
await makeRawRequest(server.url + video.previewPath, HttpStatusCode.OK_200)
await makeGetRequest({ url: server.url, path: video.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 })
await makeGetRequest({ url: server.url, path: video.previewPath, expectedStatus: HttpStatusCode.OK_200 })
}
})
@ -532,8 +533,8 @@ describe('Test live', function () {
expect(video.files).to.have.lengthOf(0)
const hlsPlaylist = video.streamingPlaylists.find(s => s.type === VideoStreamingPlaylistType.HLS)
await makeRawRequest(hlsPlaylist.playlistUrl, HttpStatusCode.OK_200)
await makeRawRequest(hlsPlaylist.segmentsSha256Url, HttpStatusCode.OK_200)
await makeRawRequest({ url: hlsPlaylist.playlistUrl, expectedStatus: HttpStatusCode.OK_200 })
await makeRawRequest({ url: hlsPlaylist.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 })
// We should have generated random filenames
expect(basename(hlsPlaylist.playlistUrl)).to.not.equal('master.m3u8')
@ -564,8 +565,8 @@ describe('Test live', function () {
expect(probe.format.bit_rate).to.be.below(maxBitrateLimits[videoStream.height])
expect(probe.format.bit_rate).to.be.at.least(minBitrateLimits[videoStream.height])
await makeRawRequest(file.torrentUrl, HttpStatusCode.OK_200)
await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
await makeRawRequest({ url: file.torrentUrl, expectedStatus: HttpStatusCode.OK_200 })
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
}
}
})

2
server/tests/api/object-storage/live.ts
View file

@ -48,7 +48,7 @@ async function checkFilesExist (servers: PeerTubeServer[], videoUUID: string, nu
for (const file of files) {
expectStartWith(file.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl())
await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
}
}
}

6
server/tests/api/object-storage/video-imports.ts
View file

@ -66,7 +66,7 @@ describe('Object storage for video import', function () {
const fileUrl = video.files[0].fileUrl
expectStartWith(fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl())
await makeRawRequest(fileUrl, HttpStatusCode.OK_200)
await makeRawRequest({ url: fileUrl, expectedStatus: HttpStatusCode.OK_200 })
})
})
@ -91,13 +91,13 @@ describe('Object storage for video import', function () {
for (const file of video.files) {
expectStartWith(file.fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl())
await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
}
for (const file of video.streamingPlaylists[0].files) {
expectStartWith(file.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl())
await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
}
})
})

20
server/tests/api/object-storage/videos.ts
View file

@ -59,11 +59,11 @@ async function checkFiles (options: {
expectStartWith(file.fileUrl, start)
const res = await makeRawRequest(file.fileDownloadUrl, HttpStatusCode.FOUND_302)
const res = await makeRawRequest({ url: file.fileDownloadUrl, expectedStatus: HttpStatusCode.FOUND_302 })
const location = res.headers['location']
expectStartWith(location, start)
await makeRawRequest(location, HttpStatusCode.OK_200)
await makeRawRequest({ url: location, expectedStatus: HttpStatusCode.OK_200 })
}
const hls = video.streamingPlaylists[0]
@ -81,19 +81,19 @@ async function checkFiles (options: {
expectStartWith(hls.playlistUrl, start)
expectStartWith(hls.segmentsSha256Url, start)
await makeRawRequest(hls.playlistUrl, HttpStatusCode.OK_200)
await makeRawRequest({ url: hls.playlistUrl, expectedStatus: HttpStatusCode.OK_200 })
const resSha = await makeRawRequest(hls.segmentsSha256Url, HttpStatusCode.OK_200)
const resSha = await makeRawRequest({ url: hls.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 })
expect(JSON.stringify(resSha.body)).to.not.throw
for (const file of hls.files) {
expectStartWith(file.fileUrl, start)
const res = await makeRawRequest(file.fileDownloadUrl, HttpStatusCode.FOUND_302)
const res = await makeRawRequest({ url: file.fileDownloadUrl, expectedStatus: HttpStatusCode.FOUND_302 })
const location = res.headers['location']
expectStartWith(location, start)
await makeRawRequest(location, HttpStatusCode.OK_200)
await makeRawRequest({ url: location, expectedStatus: HttpStatusCode.OK_200 })
}
}
@ -104,7 +104,7 @@ async function checkFiles (options: {
expect(torrent.files.length).to.equal(1)
expect(torrent.files[0].path).to.exist.and.to.not.equal('')
const res = await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
const res = await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
expect(res.body).to.have.length.above(100)
}
@ -220,7 +220,7 @@ function runTestSuite (options: {
it('Should fetch correctly all the files', async function () {
for (const url of deletedUrls.concat(keptUrls)) {
await makeRawRequest(url, HttpStatusCode.OK_200)
await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 })
}
})
@ -231,13 +231,13 @@ function runTestSuite (options: {
await waitJobs(servers)
for (const url of deletedUrls) {
await makeRawRequest(url, HttpStatusCode.NOT_FOUND_404)
await makeRawRequest({ url, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
}
})
it('Should have kept other files', async function () {
for (const url of keptUrls) {
await makeRawRequest(url, HttpStatusCode.OK_200)
await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 })
}
})

2
server/tests/api/redundancy/redundancy.ts
View file

@ -39,7 +39,7 @@ async function checkMagnetWebseeds (file: VideoFile, baseWebseeds: string[], ser
expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length)
for (const url of parsed.urlList) {
await makeRawRequest(url, HttpStatusCode.OK_200)
await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 })
}
}

6
server/tests/api/server/open-telemetry.ts
View file

@ -18,7 +18,7 @@ describe('Open Telemetry', function () {
let hasError = false
try {
await makeRawRequest(metricsUrl, HttpStatusCode.NOT_FOUND_404)
await makeRawRequest({ url: metricsUrl, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
} catch (err) {
hasError = err.message.includes('ECONNREFUSED')
}
@ -37,7 +37,7 @@ describe('Open Telemetry', function () {
}
})
const res = await makeRawRequest(metricsUrl, HttpStatusCode.OK_200)
const res = await makeRawRequest({ url: metricsUrl, expectedStatus: HttpStatusCode.OK_200 })
expect(res.text).to.contain('peertube_job_queue_total{')
})
@ -60,7 +60,7 @@ describe('Open Telemetry', function () {
}
})
const res = await makeRawRequest(metricsUrl, HttpStatusCode.OK_200)
const res = await makeRawRequest({ url: metricsUrl, expectedStatus: HttpStatusCode.OK_200 })
expect(res.text).to.contain('peertube_playback_http_downloaded_bytes_total{')
})

10
server/tests/api/transcoding/create-transcoding.ts
View file

@ -20,7 +20,7 @@ import {
async function checkFilesInObjectStorage (video: VideoDetails) {
for (const file of video.files) {
expectStartWith(file.fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl())
await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
}
if (video.streamingPlaylists.length === 0) return
@ -28,14 +28,14 @@ async function checkFilesInObjectStorage (video: VideoDetails) {
const hlsPlaylist = video.streamingPlaylists[0]
for (const file of hlsPlaylist.files) {
expectStartWith(file.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl())
await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
}
expectStartWith(hlsPlaylist.playlistUrl, ObjectStorageCommand.getPlaylistBaseUrl())
await makeRawRequest(hlsPlaylist.playlistUrl, HttpStatusCode.OK_200)
await makeRawRequest({ url: hlsPlaylist.playlistUrl, expectedStatus: HttpStatusCode.OK_200 })
expectStartWith(hlsPlaylist.segmentsSha256Url, ObjectStorageCommand.getPlaylistBaseUrl())
await makeRawRequest(hlsPlaylist.segmentsSha256Url, HttpStatusCode.OK_200)
await makeRawRequest({ url: hlsPlaylist.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 })
}
function runTests (objectStorage: boolean) {
@ -234,7 +234,7 @@ function runTests (objectStorage: boolean) {
it('Should have correctly deleted previous files', async function () {
for (const fileUrl of shouldBeDeleted) {
await makeRawRequest(fileUrl, HttpStatusCode.NOT_FOUND_404)
await makeRawRequest({ url: fileUrl, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
}
})

163
server/tests/api/transcoding/hls.ts
View file

@ -1,168 +1,48 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import { expect } from 'chai'
import { basename, join } from 'path'
import {
checkDirectoryIsEmpty,
checkResolutionsInMasterPlaylist,
checkSegmentHash,
checkTmpIsEmpty,
expectStartWith,
hlsInfohashExist
} from '@server/tests/shared'
import { areObjectStorageTestsDisabled, removeFragmentedMP4Ext, uuidRegex } from '@shared/core-utils'
import { HttpStatusCode, VideoStreamingPlaylistType } from '@shared/models'
import { join } from 'path'
import { checkDirectoryIsEmpty, checkTmpIsEmpty, completeCheckHlsPlaylist } from '@server/tests/shared'
import { areObjectStorageTestsDisabled } from '@shared/core-utils'
import { HttpStatusCode } from '@shared/models'
import {
cleanupTests,
createMultipleServers,
doubleFollow,
makeRawRequest,
ObjectStorageCommand,
PeerTubeServer,
setAccessTokensToServers,
waitJobs,
webtorrentAdd
waitJobs
} from '@shared/server-commands'
import { DEFAULT_AUDIO_RESOLUTION } from '../../../initializers/constants'
async function checkHlsPlaylist (options: {
servers: PeerTubeServer[]
videoUUID: string
hlsOnly: boolean
resolutions?: number[]
objectStorageBaseUrl: string
}) {
const { videoUUID, hlsOnly, objectStorageBaseUrl } = options
const resolutions = options.resolutions ?? [ 240, 360, 480, 720 ]
for (const server of options.servers) {
const videoDetails = await server.videos.get({ id: videoUUID })
const baseUrl = `http://${videoDetails.account.host}`
expect(videoDetails.streamingPlaylists).to.have.lengthOf(1)
const hlsPlaylist = videoDetails.streamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS)
expect(hlsPlaylist).to.not.be.undefined
const hlsFiles = hlsPlaylist.files
expect(hlsFiles).to.have.lengthOf(resolutions.length)
if (hlsOnly) expect(videoDetails.files).to.have.lengthOf(0)
else expect(videoDetails.files).to.have.lengthOf(resolutions.length)
// Check JSON files
for (const resolution of resolutions) {
const file = hlsFiles.find(f => f.resolution.id === resolution)
expect(file).to.not.be.undefined
expect(file.magnetUri).to.have.lengthOf.above(2)
expect(file.torrentUrl).to.match(
new RegExp(`http://${server.host}/lazy-static/torrents/${uuidRegex}-${file.resolution.id}-hls.torrent`)
)
if (objectStorageBaseUrl) {
expectStartWith(file.fileUrl, objectStorageBaseUrl)
} else {
expect(file.fileUrl).to.match(
new RegExp(`${baseUrl}/static/streaming-playlists/hls/${videoDetails.uuid}/${uuidRegex}-${file.resolution.id}-fragmented.mp4`)
)
}
expect(file.resolution.label).to.equal(resolution + 'p')
await makeRawRequest(file.torrentUrl, HttpStatusCode.OK_200)
await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
const torrent = await webtorrentAdd(file.magnetUri, true)
expect(torrent.files).to.be.an('array')
expect(torrent.files.length).to.equal(1)
expect(torrent.files[0].path).to.exist.and.to.not.equal('')
}
// Check master playlist
{
await checkResolutionsInMasterPlaylist({ server, playlistUrl: hlsPlaylist.playlistUrl, resolutions })
const masterPlaylist = await server.streamingPlaylists.get({ url: hlsPlaylist.playlistUrl })
let i = 0
for (const resolution of resolutions) {
expect(masterPlaylist).to.contain(`${resolution}.m3u8`)
expect(masterPlaylist).to.contain(`${resolution}.m3u8`)
const url = 'http://' + videoDetails.account.host
await hlsInfohashExist(url, hlsPlaylist.playlistUrl, i)
i++
}
}
// Check resolution playlists
{
for (const resolution of resolutions) {
const file = hlsFiles.find(f => f.resolution.id === resolution)
const playlistName = removeFragmentedMP4Ext(basename(file.fileUrl)) + '.m3u8'
const url = objectStorageBaseUrl
? `${objectStorageBaseUrl}hls/${videoUUID}/${playlistName}`
: `${baseUrl}/static/streaming-playlists/hls/${videoUUID}/${playlistName}`
const subPlaylist = await server.streamingPlaylists.get({ url })
expect(subPlaylist).to.match(new RegExp(`${uuidRegex}-${resolution}-fragmented.mp4`))
expect(subPlaylist).to.contain(basename(file.fileUrl))
}
}
{
const baseUrlAndPath = objectStorageBaseUrl
? objectStorageBaseUrl + 'hls/' + videoUUID
: baseUrl + '/static/streaming-playlists/hls/' + videoUUID
for (const resolution of resolutions) {
await checkSegmentHash({
server,
baseUrlPlaylist: baseUrlAndPath,
baseUrlSegment: baseUrlAndPath,
resolution,
hlsPlaylist
})
}
}
}
}
describe('Test HLS videos', function () {
let servers: PeerTubeServer[] = []
let videoUUID = ''
let videoAudioUUID = ''
function runTestSuite (hlsOnly: boolean, objectStorageBaseUrl?: string) {
const videoUUIDs: string[] = []
it('Should upload a video and transcode it to HLS', async function () {
this.timeout(120000)
const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video 1', fixture: 'video_short.webm' } })
videoUUID = uuid
videoUUIDs.push(uuid)
await waitJobs(servers)
await checkHlsPlaylist({ servers, videoUUID, hlsOnly, objectStorageBaseUrl })
await completeCheckHlsPlaylist({ servers, videoUUID: uuid, hlsOnly, objectStorageBaseUrl })
})
it('Should upload an audio file and transcode it to HLS', async function () {
this.timeout(120000)
const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video audio', fixture: 'sample.ogg' } })
videoAudioUUID = uuid
videoUUIDs.push(uuid)
await waitJobs(servers)
await checkHlsPlaylist({
await completeCheckHlsPlaylist({
servers,
videoUUID: videoAudioUUID,
videoUUID: uuid,
hlsOnly,
resolutions: [ DEFAULT_AUDIO_RESOLUTION, 360, 240 ],
objectStorageBaseUrl
@ -172,31 +52,36 @@ describe('Test HLS videos', function () {
it('Should update the video', async function () {
this.timeout(30000)
await servers[0].videos.update({ id: videoUUID, attributes: { name: 'video 1 updated' } })
await servers[0].videos.update({ id: videoUUIDs[0], attributes: { name: 'video 1 updated' } })
await waitJobs(servers)
await checkHlsPlaylist({ servers, videoUUID, hlsOnly, objectStorageBaseUrl })
await completeCheckHlsPlaylist({ servers, videoUUID: videoUUIDs[0], hlsOnly, objectStorageBaseUrl })
})
it('Should delete videos', async function () {
this.timeout(10000)
await servers[0].videos.remove({ id: videoUUID })
await servers[0].videos.remove({ id: videoAudioUUID })
for (const uuid of videoUUIDs) {
await servers[0].videos.remove({ id: uuid })
}
await waitJobs(servers)
for (const server of servers) {
await server.videos.get({ id: videoUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
await server.videos.get({ id: videoAudioUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
for (const uuid of videoUUIDs) {
await server.videos.get({ id: uuid, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
}
}
})
it('Should have the playlists/segment deleted from the disk', async function () {
for (const server of servers) {
await checkDirectoryIsEmpty(server, 'videos')
await checkDirectoryIsEmpty(server, join('streaming-playlists', 'hls'))
await checkDirectoryIsEmpty(server, 'videos', [ 'private' ])
await checkDirectoryIsEmpty(server, join('videos', 'private'))
await checkDirectoryIsEmpty(server, join('streaming-playlists', 'hls'), [ 'private' ])
await checkDirectoryIsEmpty(server, join('streaming-playlists', 'hls', 'private'))
}
})

1
server/tests/api/transcoding/index.ts
View file

@ -2,4 +2,5 @@ export * from './audio-only'
export * from './create-transcoding'
export * from './hls'
export * from './transcoder'
export * from './update-while-transcoding'
export * from './video-studio'

151
server/tests/api/transcoding/update-while-transcoding.ts Normal file
View file

@ -0,0 +1,151 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import { completeCheckHlsPlaylist } from '@server/tests/shared'
import { areObjectStorageTestsDisabled, wait } from '@shared/core-utils'
import { VideoPrivacy } from '@shared/models'
import {
cleanupTests,
createMultipleServers,
doubleFollow,
ObjectStorageCommand,
PeerTubeServer,
setAccessTokensToServers,
waitJobs
} from '@shared/server-commands'
describe('Test update video privacy while transcoding', function () {
let servers: PeerTubeServer[] = []
const videoUUIDs: string[] = []
function runTestSuite (hlsOnly: boolean, objectStorageBaseUrl?: string) {
it('Should not have an error while quickly updating a private video to public after upload #1', async function () {
this.timeout(360_000)
const attributes = {
name: 'quick update',
privacy: VideoPrivacy.PRIVATE
}
const { uuid } = await servers[0].videos.upload({ attributes, waitTorrentGeneration: false })
await servers[0].videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.PUBLIC } })
videoUUIDs.push(uuid)
await waitJobs(servers)
await completeCheckHlsPlaylist({ servers, videoUUID: uuid, hlsOnly, objectStorageBaseUrl })
})
it('Should not have an error while quickly updating a private video to public after upload #2', async function () {
{
const attributes = {
name: 'quick update 2',
privacy: VideoPrivacy.PRIVATE
}
const { uuid } = await servers[0].videos.upload({ attributes, waitTorrentGeneration: true })
await servers[0].videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.PUBLIC } })
videoUUIDs.push(uuid)
await waitJobs(servers)
await completeCheckHlsPlaylist({ servers, videoUUID: uuid, hlsOnly, objectStorageBaseUrl })
}
})
it('Should not have an error while quickly updating a private video to public after upload #3', async function () {
const attributes = {
name: 'quick update 3',
privacy: VideoPrivacy.PRIVATE
}
const { uuid } = await servers[0].videos.upload({ attributes, waitTorrentGeneration: true })
await wait(1000)
await servers[0].videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.PUBLIC } })
videoUUIDs.push(uuid)
await waitJobs(servers)
await completeCheckHlsPlaylist({ servers, videoUUID: uuid, hlsOnly, objectStorageBaseUrl })
})
}
before(async function () {
this.timeout(120000)
const configOverride = {
transcoding: {
enabled: true,
allow_audio_files: true,
hls: {
enabled: true
}
}
}
servers = await createMultipleServers(2, configOverride)
// Get the access tokens
await setAccessTokensToServers(servers)
// Server 1 and server 2 follow each other
await doubleFollow(servers[0], servers[1])
})
describe('With WebTorrent & HLS enabled', function () {
runTestSuite(false)
})
describe('With only HLS enabled', function () {
before(async function () {
await servers[0].config.updateCustomSubConfig({
newConfig: {
transcoding: {
enabled: true,
allowAudioFiles: true,
resolutions: {
'144p': false,
'240p': true,
'360p': true,
'480p': true,
'720p': true,
'1080p': true,
'1440p': true,
'2160p': true
},
hls: {
enabled: true
},
webtorrent: {
enabled: false
}
}
}
})
})
runTestSuite(true)
})
describe('With object storage enabled', function () {
if (areObjectStorageTestsDisabled()) return
before(async function () {
this.timeout(120000)
const configOverride = ObjectStorageCommand.getDefaultConfig()
await ObjectStorageCommand.prepareDefaultBuckets()
await servers[0].kill()
await servers[0].run(configOverride)
})
runTestSuite(true, ObjectStorageCommand.getPlaylistBaseUrl())
})
after(async function () {
await cleanupTests(servers)
})
})

1
server/tests/api/videos/index.ts
View file

@ -19,3 +19,4 @@ import './videos-common-filters'
import './videos-history'
import './videos-overview'
import './video-source'
import './video-static-file-privacy'

2
server/tests/api/videos/video-files.ts
View file

@ -153,7 +153,7 @@ describe('Test videos files', function () {
expect(video.streamingPlaylists[0].files).to.have.lengthOf(files.length - 1)
expect(video.streamingPlaylists[0].files.find(f => f.id === toDelete.id)).to.not.exist
const { text } = await makeRawRequest(video.streamingPlaylists[0].playlistUrl)
const { text } = await makeRawRequest({ url: video.streamingPlaylists[0].playlistUrl, expectedStatus: HttpStatusCode.OK_200 })
expect(text.includes(`-${toDelete.resolution.id}.m3u8`)).to.be.false
expect(text.includes(`-${video.streamingPlaylists[0].files[0].resolution.id}.m3u8`)).to.be.true

389
server/tests/api/videos/video-static-file-privacy.ts Normal file
View file

@ -0,0 +1,389 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import { expect } from 'chai'
import { decode } from 'magnet-uri'
import { expectStartWith } from '@server/tests/shared'
import { getAllFiles, wait } from '@shared/core-utils'
import { HttpStatusCode, LiveVideo, VideoDetails, VideoPrivacy } from '@shared/models'
import {
cleanupTests,
createSingleServer,
findExternalSavedVideo,
makeRawRequest,
parseTorrentVideo,
PeerTubeServer,
sendRTMPStream,
setAccessTokensToServers,
setDefaultVideoChannel,
stopFfmpeg,
waitJobs
} from '@shared/server-commands'
describe('Test video static file privacy', function () {
let server: PeerTubeServer
let userToken: string
before(async function () {
this.timeout(50000)
server = await createSingleServer(1)
await setAccessTokensToServers([ server ])
await setDefaultVideoChannel([ server ])
userToken = await server.users.generateUserAndToken('user1')
})
describe('VOD static file path', function () {
function runSuite () {
async function checkPrivateWebTorrentFiles (uuid: string) {
const video = await server.videos.getWithToken({ id: uuid })
for (const file of video.files) {
expect(file.fileDownloadUrl).to.not.include('/private/')
expectStartWith(file.fileUrl, server.url + '/static/webseed/private/')
const torrent = await parseTorrentVideo(server, file)
expect(torrent.urlList).to.have.lengthOf(0)
const magnet = decode(file.magnetUri)
expect(magnet.urlList).to.have.lengthOf(0)
await makeRawRequest({ url: file.fileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
}
const hls = video.streamingPlaylists[0]
if (hls) {
expectStartWith(hls.playlistUrl, server.url + '/static/streaming-playlists/hls/private/')
expectStartWith(hls.segmentsSha256Url, server.url + '/static/streaming-playlists/hls/private/')
await makeRawRequest({ url: hls.playlistUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
await makeRawRequest({ url: hls.segmentsSha256Url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
}
}
async function checkPublicWebTorrentFiles (uuid: string) {
const video = await server.videos.get({ id: uuid })
for (const file of getAllFiles(video)) {
expect(file.fileDownloadUrl).to.not.include('/private/')
expect(file.fileUrl).to.not.include('/private/')
const torrent = await parseTorrentVideo(server, file)
expect(torrent.urlList[0]).to.not.include('private')
const magnet = decode(file.magnetUri)
expect(magnet.urlList[0]).to.not.include('private')
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
await makeRawRequest({ url: torrent.urlList[0], expectedStatus: HttpStatusCode.OK_200 })
await makeRawRequest({ url: magnet.urlList[0], expectedStatus: HttpStatusCode.OK_200 })
}
const hls = video.streamingPlaylists[0]
if (hls) {
expect(hls.playlistUrl).to.not.include('private')
expect(hls.segmentsSha256Url).to.not.include('private')
await makeRawRequest({ url: hls.playlistUrl, expectedStatus: HttpStatusCode.OK_200 })
await makeRawRequest({ url: hls.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 })
}
}
it('Should upload a private/internal video and have a private static path', async function () {
this.timeout(120000)
for (const privacy of [ VideoPrivacy.PRIVATE, VideoPrivacy.INTERNAL ]) {
const { uuid } = await server.videos.quickUpload({ name: 'video', privacy })
await waitJobs([ server ])
await checkPrivateWebTorrentFiles(uuid)
}
})
it('Should upload a public video and update it as private/internal to have a private static path', async function () {
this.timeout(120000)
for (const privacy of [ VideoPrivacy.PRIVATE, VideoPrivacy.INTERNAL ]) {
const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PUBLIC })
await waitJobs([ server ])
await server.videos.update({ id: uuid, attributes: { privacy } })
await waitJobs([ server ])
await checkPrivateWebTorrentFiles(uuid)
}
})
it('Should upload a private video and update it to unlisted to have a public static path', async function () {
this.timeout(120000)
const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE })
await waitJobs([ server ])
await server.videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.UNLISTED } })
await waitJobs([ server ])
await checkPublicWebTorrentFiles(uuid)
})
it('Should upload an internal video and update it to public to have a public static path', async function () {
this.timeout(120000)
const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.INTERNAL })
await waitJobs([ server ])
await server.videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.PUBLIC } })
await waitJobs([ server ])
await checkPublicWebTorrentFiles(uuid)
})
it('Should upload an internal video and schedule a public publish', async function () {
this.timeout(120000)
const attributes = {
name: 'video',
privacy: VideoPrivacy.PRIVATE,
scheduleUpdate: {
updateAt: new Date(Date.now() + 1000).toISOString(),
privacy: VideoPrivacy.PUBLIC as VideoPrivacy.PUBLIC
}
}
const { uuid } = await server.videos.upload({ attributes })
await waitJobs([ server ])
await wait(1000)
await server.debug.sendCommand({ body: { command: 'process-update-videos-scheduler' } })
await waitJobs([ server ])
await checkPublicWebTorrentFiles(uuid)
})
}
describe('Without transcoding', function () {
runSuite()
})
describe('With transcoding', function () {
before(async function () {
await server.config.enableMinimumTranscoding()
})
runSuite()
})
})
describe('VOD static file right check', function () {
let unrelatedFileToken: string
async function checkVideoFiles (options: {
id: string
expectedStatus: HttpStatusCode
token: string
videoFileToken: string
}) {
const { id, expectedStatus, token, videoFileToken } = options
const video = await server.videos.getWithToken({ id })
for (const file of getAllFiles(video)) {
await makeRawRequest({ url: file.fileUrl, token, expectedStatus })
await makeRawRequest({ url: file.fileDownloadUrl, token, expectedStatus })
await makeRawRequest({ url: file.fileUrl, query: { videoFileToken }, expectedStatus })
await makeRawRequest({ url: file.fileDownloadUrl, query: { videoFileToken }, expectedStatus })
}
const hls = video.streamingPlaylists[0]
await makeRawRequest({ url: hls.playlistUrl, token, expectedStatus })
await makeRawRequest({ url: hls.segmentsSha256Url, token, expectedStatus })
await makeRawRequest({ url: hls.playlistUrl, query: { videoFileToken }, expectedStatus })
await makeRawRequest({ url: hls.segmentsSha256Url, query: { videoFileToken }, expectedStatus })
}
before(async function () {
await server.config.enableMinimumTranscoding()
const { uuid } = await server.videos.quickUpload({ name: 'another video' })
unrelatedFileToken = await server.videoToken.getVideoFileToken({ videoId: uuid })
})
it('Should not be able to access a private video files without OAuth token and file token', async function () {
this.timeout(120000)
const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.INTERNAL })
await waitJobs([ server ])
await checkVideoFiles({ id: uuid, expectedStatus: HttpStatusCode.FORBIDDEN_403, token: null, videoFileToken: null })
})
it('Should not be able to access an internal video files without appropriate OAuth token and file token', async function () {
this.timeout(120000)
const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE })
await waitJobs([ server ])
await checkVideoFiles({
id: uuid,
expectedStatus: HttpStatusCode.FORBIDDEN_403,
token: userToken,
videoFileToken: unrelatedFileToken
})
})
it('Should be able to access a private video files with appropriate OAuth token or file token', async function () {
this.timeout(120000)
const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE })
const videoFileToken = await server.videoToken.getVideoFileToken({ videoId: uuid })
await waitJobs([ server ])
await checkVideoFiles({ id: uuid, expectedStatus: HttpStatusCode.OK_200, token: server.accessToken, videoFileToken })
})
it('Should be able to access a private video of another user with an admin OAuth token or file token', async function () {
this.timeout(120000)
const { uuid } = await server.videos.quickUpload({ name: 'video', token: userToken, privacy: VideoPrivacy.PRIVATE })
const videoFileToken = await server.videoToken.getVideoFileToken({ videoId: uuid })
await waitJobs([ server ])
await checkVideoFiles({ id: uuid, expectedStatus: HttpStatusCode.OK_200, token: server.accessToken, videoFileToken })
})
})
describe('Live static file path and check', function () {
let normalLiveId: string
let normalLive: LiveVideo
let permanentLiveId: string
let permanentLive: LiveVideo
let unrelatedFileToken: string
async function checkLiveFiles (live: LiveVideo, liveId: string) {
const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: live.rtmpUrl, streamKey: live.streamKey })
await server.live.waitUntilPublished({ videoId: liveId })
const video = await server.videos.getWithToken({ id: liveId })
const fileToken = await server.videoToken.getVideoFileToken({ videoId: video.uuid })
const hls = video.streamingPlaylists[0]
for (const url of [ hls.playlistUrl, hls.segmentsSha256Url ]) {
expectStartWith(url, server.url + '/static/streaming-playlists/hls/private/')
await makeRawRequest({ url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
await makeRawRequest({ url, query: { videoFileToken: fileToken }, expectedStatus: HttpStatusCode.OK_200 })
await makeRawRequest({ url, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
await makeRawRequest({ url, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
await makeRawRequest({ url, query: { videoFileToken: unrelatedFileToken }, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
}
await stopFfmpeg(ffmpegCommand)
}
async function checkReplay (replay: VideoDetails) {
const fileToken = await server.videoToken.getVideoFileToken({ videoId: replay.uuid })
const hls = replay.streamingPlaylists[0]
expect(hls.files).to.not.have.lengthOf(0)
for (const file of hls.files) {
await makeRawRequest({ url: file.fileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
await makeRawRequest({ url: file.fileUrl, query: { videoFileToken: fileToken }, expectedStatus: HttpStatusCode.OK_200 })
await makeRawRequest({ url: file.fileUrl, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
await makeRawRequest({
url: file.fileUrl,
query: { videoFileToken: unrelatedFileToken },
expectedStatus: HttpStatusCode.FORBIDDEN_403
})
}
for (const url of [ hls.playlistUrl, hls.segmentsSha256Url ]) {
expectStartWith(url, server.url + '/static/streaming-playlists/hls/private/')
await makeRawRequest({ url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
await makeRawRequest({ url, query: { videoFileToken: fileToken }, expectedStatus: HttpStatusCode.OK_200 })
await makeRawRequest({ url, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
await makeRawRequest({ url, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
await makeRawRequest({ url, query: { videoFileToken: unrelatedFileToken }, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
}
}
before(async function () {
await server.config.enableMinimumTranscoding()
const { uuid } = await server.videos.quickUpload({ name: 'another video' })
unrelatedFileToken = await server.videoToken.getVideoFileToken({ videoId: uuid })
await server.config.enableLive({
allowReplay: true,
transcoding: true,
resolutions: 'min'
})
{
const { video, live } = await server.live.quickCreate({ saveReplay: true, permanentLive: false, privacy: VideoPrivacy.PRIVATE })
normalLiveId = video.uuid
normalLive = live
}
{
const { video, live } = await server.live.quickCreate({ saveReplay: true, permanentLive: true, privacy: VideoPrivacy.PRIVATE })
permanentLiveId = video.uuid
permanentLive = live
}
})
it('Should create a private normal live and have a private static path', async function () {
this.timeout(240000)
await checkLiveFiles(normalLive, normalLiveId)
})
it('Should create a private permanent live and have a private static path', async function () {
this.timeout(240000)
await checkLiveFiles(permanentLive, permanentLiveId)
})
it('Should have created a replay of the normal live with a private static path', async function () {
this.timeout(240000)
await server.live.waitUntilReplacedByReplay({ videoId: normalLiveId })
const replay = await server.videos.getWithToken({ id: normalLiveId })
await checkReplay(replay)
})
it('Should have created a replay of the permanent live with a private static path', async function () {
this.timeout(240000)
await server.live.waitUntilWaiting({ videoId: permanentLiveId })
await waitJobs([ server ])
const live = await server.videos.getWithToken({ id: permanentLiveId })
const replayFromList = await findExternalSavedVideo(server, live)
const replay = await server.videos.getWithToken({ id: replayFromList.id })
await checkReplay(replay)
})
})
after(async function () {
await cleanupTests([ server ])
})
})

2
server/tests/cli/create-import-video-file-job.ts
View file

@ -29,7 +29,7 @@ async function checkFiles (video: VideoDetails, objectStorage: boolean) {
for (const file of video.files) {
if (objectStorage) expectStartWith(file.fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl())
await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
}
}

4
server/tests/cli/create-move-video-storage-job.ts
View file

@ -22,7 +22,7 @@ async function checkFiles (origin: PeerTubeServer, video: VideoDetails, inObject
expectStartWith(file.fileUrl, start)
await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
}
const start = inObjectStorage
@ -36,7 +36,7 @@ async function checkFiles (origin: PeerTubeServer, video: VideoDetails, inObject
for (const file of hls.files) {
expectStartWith(file.fileUrl, start)
await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
}
}

2
server/tests/cli/create-transcoding-job.ts
View file

@ -23,7 +23,7 @@ async function checkFilesInObjectStorage (files: VideoFile[], type: 'webtorrent'
expectStartWith(file.fileUrl, shouldStartWith)
await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
}
}

41
server/tests/cli/prune-storage.ts
View file

@ -5,7 +5,7 @@ import { createFile, readdir } from 'fs-extra'
import { join } from 'path'
import { wait } from '@shared/core-utils'
import { buildUUID } from '@shared/extra-utils'
import { HttpStatusCode, VideoPlaylistPrivacy } from '@shared/models'
import { HttpStatusCode, VideoPlaylistPrivacy, VideoPrivacy } from '@shared/models'
import {
cleanupTests,
CLICommand,
@ -36,22 +36,28 @@ async function assertNotExists (server: PeerTubeServer, directory: string, subst
async function assertCountAreOkay (servers: PeerTubeServer[]) {
for (const server of servers) {
const videosCount = await countFiles(server, 'videos')
expect(videosCount).to.equal(8)
expect(videosCount).to.equal(9) // 2 videos with 4 resolutions + private directory
const privateVideosCount = await countFiles(server, 'videos/private')
expect(privateVideosCount).to.equal(4)
const torrentsCount = await countFiles(server, 'torrents')
expect(torrentsCount).to.equal(16)
expect(torrentsCount).to.equal(24)
const previewsCount = await countFiles(server, 'previews')
expect(previewsCount).to.equal(2)
expect(previewsCount).to.equal(3)
const thumbnailsCount = await countFiles(server, 'thumbnails')
expect(thumbnailsCount).to.equal(6)
expect(thumbnailsCount).to.equal(7) // 3 local videos, 1 local playlist, 2 remotes videos and 1 remote playlist
const avatarsCount = await countFiles(server, 'avatars')
expect(avatarsCount).to.equal(4)
const hlsRootCount = await countFiles(server, 'streaming-playlists/hls')
expect(hlsRootCount).to.equal(2)
const hlsRootCount = await countFiles(server, join('streaming-playlists', 'hls'))
expect(hlsRootCount).to.equal(3) // 2 videos + private directory
const hlsPrivateRootCount = await countFiles(server, join('streaming-playlists', 'hls', 'private'))
expect(hlsPrivateRootCount).to.equal(1)
}
}
@ -67,8 +73,10 @@ describe('Test prune storage scripts', function () {
await setDefaultVideoChannel(servers)
for (const server of servers) {
await server.videos.upload({ attributes: { name: 'video 1' } })
await server.videos.upload({ attributes: { name: 'video 2' } })
await server.videos.upload({ attributes: { name: 'video 1', privacy: VideoPrivacy.PUBLIC } })
await server.videos.upload({ attributes: { name: 'video 2', privacy: VideoPrivacy.PUBLIC } })
await server.videos.upload({ attributes: { name: 'video 3', privacy: VideoPrivacy.PRIVATE } })
await server.users.updateMyAvatar({ fixture: 'avatar.png' })
@ -123,13 +131,16 @@ describe('Test prune storage scripts', function () {
it('Should create some dirty files', async function () {
for (let i = 0; i < 2; i++) {
{
const base = servers[0].servers.buildDirectory('videos')
const basePublic = servers[0].servers.buildDirectory('videos')
const basePrivate = servers[0].servers.buildDirectory(join('videos', 'private'))
const n1 = buildUUID() + '.mp4'
const n2 = buildUUID() + '.webm'
await createFile(join(base, n1))
await createFile(join(base, n2))
await createFile(join(basePublic, n1))
await createFile(join(basePublic, n2))
await createFile(join(basePrivate, n1))
await createFile(join(basePrivate, n2))
badNames['videos'] = [ n1, n2 ]
}
@ -184,10 +195,12 @@ describe('Test prune storage scripts', function () {
{
const directory = join('streaming-playlists', 'hls')
const base = servers[0].servers.buildDirectory(directory)
const basePublic = servers[0].servers.buildDirectory(directory)
const basePrivate = servers[0].servers.buildDirectory(join(directory, 'private'))
const n1 = buildUUID()
await createFile(join(base, n1))
await createFile(join(basePublic, n1))
await createFile(join(basePrivate, n1))
badNames[directory] = [ n1 ]
}
}

20
server/tests/cli/regenerate-thumbnails.ts
View file

@ -6,7 +6,7 @@ import {
cleanupTests,
createMultipleServers,
doubleFollow,
makeRawRequest,
makeGetRequest,
PeerTubeServer,
setAccessTokensToServers,
waitJobs
@ -16,8 +16,8 @@ async function testThumbnail (server: PeerTubeServer, videoId: number | string)
const video = await server.videos.get({ id: videoId })
const requests = [
makeRawRequest(join(server.url, video.thumbnailPath), HttpStatusCode.OK_200),
makeRawRequest(join(server.url, video.thumbnailPath), HttpStatusCode.OK_200)
makeGetRequest({ url: server.url, path: video.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 }),
makeGetRequest({ url: server.url, path: video.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 })
]
for (const req of requests) {
@ -69,17 +69,17 @@ describe('Test regenerate thumbnails script', function () {
it('Should have empty thumbnails', async function () {
{
const res = await makeRawRequest(join(servers[0].url, video1.thumbnailPath), HttpStatusCode.OK_200)
const res = await makeGetRequest({ url: servers[0].url, path: video1.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 })
expect(res.body).to.have.lengthOf(0)
}
{
const res = await makeRawRequest(join(servers[0].url, video2.thumbnailPath), HttpStatusCode.OK_200)
const res = await makeGetRequest({ url: servers[0].url, path: video2.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 })
expect(res.body).to.not.have.lengthOf(0)
}
{
const res = await makeRawRequest(join(servers[0].url, remoteVideo.thumbnailPath), HttpStatusCode.OK_200)
const res = await makeGetRequest({ url: servers[0].url, path: remoteVideo.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 })
expect(res.body).to.have.lengthOf(0)
}
})
@ -94,21 +94,21 @@ describe('Test regenerate thumbnails script', function () {
await testThumbnail(servers[0], video1.uuid)
await testThumbnail(servers[0], video2.uuid)
const res = await makeRawRequest(join(servers[0].url, remoteVideo.thumbnailPath), HttpStatusCode.OK_200)
const res = await makeGetRequest({ url: servers[0].url, path: remoteVideo.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 })
expect(res.body).to.have.lengthOf(0)
})
it('Should have deleted old thumbnail files', async function () {
{
await makeRawRequest(join(servers[0].url, video1.thumbnailPath), HttpStatusCode.NOT_FOUND_404)
await makeGetRequest({ url: servers[0].url, path: video1.thumbnailPath, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
}
{
await makeRawRequest(join(servers[0].url, video2.thumbnailPath), HttpStatusCode.NOT_FOUND_404)
await makeGetRequest({ url: servers[0].url, path: video2.thumbnailPath, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
}
{
const res = await makeRawRequest(join(servers[0].url, remoteVideo.thumbnailPath), HttpStatusCode.OK_200)
const res = await makeGetRequest({ url: servers[0].url, path: remoteVideo.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 })
expect(res.body).to.have.lengthOf(0)
}
})

2
server/tests/feeds/feeds.ts
View file

@ -314,7 +314,7 @@ describe('Test syndication feeds', () => {
const jsonObj = JSON.parse(json)
const imageUrl = jsonObj.icon
expect(imageUrl).to.include('/lazy-static/avatars/')
await makeRawRequest(imageUrl)
await makeRawRequest({ url: imageUrl })
})
})

36
server/tests/plugins/filter-hooks.ts
View file

@ -6,6 +6,7 @@ import {
cleanupTests,
createMultipleServers,
doubleFollow,
makeGetRequest,
makeRawRequest,
PeerTubeServer,
PluginsCommand,
@ -461,30 +462,41 @@ describe('Test plugin filter hooks', function () {
})
it('Should run filter:api.download.torrent.allowed.result', async function () {
const res = await makeRawRequest(downloadVideos[0].files[0].torrentDownloadUrl, 403)
const res = await makeRawRequest({ url: downloadVideos[0].files[0].torrentDownloadUrl, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
expect(res.body.error).to.equal('Liu Bei')
await makeRawRequest(downloadVideos[1].files[0].torrentDownloadUrl, 200)
await makeRawRequest(downloadVideos[2].files[0].torrentDownloadUrl, 200)
await makeRawRequest({ url: downloadVideos[1].files[0].torrentDownloadUrl, expectedStatus: HttpStatusCode.OK_200 })
await makeRawRequest({ url: downloadVideos[2].files[0].torrentDownloadUrl, expectedStatus: HttpStatusCode.OK_200 })
})
it('Should run filter:api.download.video.allowed.result', async function () {
{
const res = await makeRawRequest(downloadVideos[1].files[0].fileDownloadUrl, 403)
const res = await makeRawRequest({ url: downloadVideos[1].files[0].fileDownloadUrl, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
expect(res.body.error).to.equal('Cao Cao')
await makeRawRequest(downloadVideos[0].files[0].fileDownloadUrl, 200)
await makeRawRequest(downloadVideos[2].files[0].fileDownloadUrl, 200)
await makeRawRequest({ url: downloadVideos[0].files[0].fileDownloadUrl, expectedStatus: HttpStatusCode.OK_200 })
await makeRawRequest({ url: downloadVideos[2].files[0].fileDownloadUrl, expectedStatus: HttpStatusCode.OK_200 })
}
{
const res = await makeRawRequest(downloadVideos[2].streamingPlaylists[0].files[0].fileDownloadUrl, 403)
const res = await makeRawRequest({
url: downloadVideos[2].streamingPlaylists[0].files[0].fileDownloadUrl,
expectedStatus: HttpStatusCode.FORBIDDEN_403
})
expect(res.body.error).to.equal('Sun Jian')
await makeRawRequest(downloadVideos[2].files[0].fileDownloadUrl, 200)
await makeRawRequest({ url: downloadVideos[2].files[0].fileDownloadUrl, expectedStatus: HttpStatusCode.OK_200 })
await makeRawRequest(downloadVideos[0].streamingPlaylists[0].files[0].fileDownloadUrl, 200)
await makeRawRequest(downloadVideos[1].streamingPlaylists[0].files[0].fileDownloadUrl, 200)
await makeRawRequest({
url: downloadVideos[0].streamingPlaylists[0].files[0].fileDownloadUrl,
expectedStatus: HttpStatusCode.OK_200
})
await makeRawRequest({
url: downloadVideos[1].streamingPlaylists[0].files[0].fileDownloadUrl,
expectedStatus: HttpStatusCode.OK_200
})
}
})
})
@ -515,12 +527,12 @@ describe('Test plugin filter hooks', function () {
})
it('Should run filter:html.embed.video.allowed.result', async function () {
const res = await makeRawRequest(servers[0].url + embedVideos[0].embedPath, 200)
const res = await makeGetRequest({ url: servers[0].url, path: embedVideos[0].embedPath, expectedStatus: HttpStatusCode.OK_200 })
expect(res.text).to.equal('Lu Bu')
})
it('Should run filter:html.embed.video-playlist.allowed.result', async function () {
const res = await makeRawRequest(servers[0].url + embedPlaylists[0].embedPath, 200)
const res = await makeGetRequest({ url: servers[0].url, path: embedPlaylists[0].embedPath, expectedStatus: HttpStatusCode.OK_200 })
expect(res.text).to.equal('Diao Chan')
})
})

6
server/tests/plugins/plugin-helpers.ts
View file

@ -307,7 +307,7 @@ describe('Test plugin helpers', function () {
expect(file.fps).to.equal(25)
expect(await pathExists(file.path)).to.be.true
await makeRawRequest(file.url, HttpStatusCode.OK_200)
await makeRawRequest({ url: file.url, expectedStatus: HttpStatusCode.OK_200 })
}
}
@ -321,12 +321,12 @@ describe('Test plugin helpers', function () {
const miniature = body.thumbnails.find(t => t.type === ThumbnailType.MINIATURE)
expect(miniature).to.exist
expect(await pathExists(miniature.path)).to.be.true
await makeRawRequest(miniature.url, HttpStatusCode.OK_200)
await makeRawRequest({ url: miniature.url, expectedStatus: HttpStatusCode.OK_200 })
const preview = body.thumbnails.find(t => t.type === ThumbnailType.PREVIEW)
expect(preview).to.exist
expect(await pathExists(preview.path)).to.be.true
await makeRawRequest(preview.url, HttpStatusCode.OK_200)
await makeRawRequest({ url: preview.url, expectedStatus: HttpStatusCode.OK_200 })
}
})

122
server/tests/shared/streaming-playlists.ts
View file

@ -1,9 +1,13 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import { expect } from 'chai'
import { basename } from 'path'
import { removeFragmentedMP4Ext } from '@shared/core-utils'
import { removeFragmentedMP4Ext, uuidRegex } from '@shared/core-utils'
import { sha256 } from '@shared/extra-utils'
import { HttpStatusCode, VideoStreamingPlaylist } from '@shared/models'
import { PeerTubeServer } from '@shared/server-commands'
import { HttpStatusCode, VideoStreamingPlaylist, VideoStreamingPlaylistType } from '@shared/models'
import { makeRawRequest, PeerTubeServer, webtorrentAdd } from '@shared/server-commands'
import { expectStartWith } from './checks'
import { hlsInfohashExist } from './tracker'
async function checkSegmentHash (options: {
server: PeerTubeServer
@ -75,8 +79,118 @@ async function checkResolutionsInMasterPlaylist (options: {
expect(playlistsLength).to.have.lengthOf(resolutions.length)
}
async function completeCheckHlsPlaylist (options: {
servers: PeerTubeServer[]
videoUUID: string
hlsOnly: boolean
resolutions?: number[]
objectStorageBaseUrl: string
}) {
const { videoUUID, hlsOnly, objectStorageBaseUrl } = options
const resolutions = options.resolutions ?? [ 240, 360, 480, 720 ]
for (const server of options.servers) {
const videoDetails = await server.videos.get({ id: videoUUID })
const baseUrl = `http://${videoDetails.account.host}`
expect(videoDetails.streamingPlaylists).to.have.lengthOf(1)
const hlsPlaylist = videoDetails.streamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS)
expect(hlsPlaylist).to.not.be.undefined
const hlsFiles = hlsPlaylist.files
expect(hlsFiles).to.have.lengthOf(resolutions.length)
if (hlsOnly) expect(videoDetails.files).to.have.lengthOf(0)
else expect(videoDetails.files).to.have.lengthOf(resolutions.length)
// Check JSON files
for (const resolution of resolutions) {
const file = hlsFiles.find(f => f.resolution.id === resolution)
expect(file).to.not.be.undefined
expect(file.magnetUri).to.have.lengthOf.above(2)
expect(file.torrentUrl).to.match(
new RegExp(`http://${server.host}/lazy-static/torrents/${uuidRegex}-${file.resolution.id}-hls.torrent`)
)
if (objectStorageBaseUrl) {
expectStartWith(file.fileUrl, objectStorageBaseUrl)
} else {
expect(file.fileUrl).to.match(
new RegExp(`${baseUrl}/static/streaming-playlists/hls/${videoDetails.uuid}/${uuidRegex}-${file.resolution.id}-fragmented.mp4`)
)
}
expect(file.resolution.label).to.equal(resolution + 'p')
await makeRawRequest({ url: file.torrentUrl, expectedStatus: HttpStatusCode.OK_200 })
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
const torrent = await webtorrentAdd(file.magnetUri, true)
expect(torrent.files).to.be.an('array')
expect(torrent.files.length).to.equal(1)
expect(torrent.files[0].path).to.exist.and.to.not.equal('')
}
// Check master playlist
{
await checkResolutionsInMasterPlaylist({ server, playlistUrl: hlsPlaylist.playlistUrl, resolutions })
const masterPlaylist = await server.streamingPlaylists.get({ url: hlsPlaylist.playlistUrl })
let i = 0
for (const resolution of resolutions) {
expect(masterPlaylist).to.contain(`${resolution}.m3u8`)
expect(masterPlaylist).to.contain(`${resolution}.m3u8`)
const url = 'http://' + videoDetails.account.host
await hlsInfohashExist(url, hlsPlaylist.playlistUrl, i)
i++
}
}
// Check resolution playlists
{
for (const resolution of resolutions) {
const file = hlsFiles.find(f => f.resolution.id === resolution)
const playlistName = removeFragmentedMP4Ext(basename(file.fileUrl)) + '.m3u8'
const url = objectStorageBaseUrl
? `${objectStorageBaseUrl}hls/${videoUUID}/${playlistName}`
: `${baseUrl}/static/streaming-playlists/hls/${videoUUID}/${playlistName}`
const subPlaylist = await server.streamingPlaylists.get({ url })
expect(subPlaylist).to.match(new RegExp(`${uuidRegex}-${resolution}-fragmented.mp4`))
expect(subPlaylist).to.contain(basename(file.fileUrl))
}
}
{
const baseUrlAndPath = objectStorageBaseUrl
? objectStorageBaseUrl + 'hls/' + videoUUID
: baseUrl + '/static/streaming-playlists/hls/' + videoUUID
for (const resolution of resolutions) {
await checkSegmentHash({
server,
baseUrlPlaylist: baseUrlAndPath,
baseUrlSegment: baseUrlAndPath,
resolution,
hlsPlaylist
})
}
}
}
}
export {
checkSegmentHash,
checkLiveSegmentHash,
checkResolutionsInMasterPlaylist
checkResolutionsInMasterPlaylist,
completeCheckHlsPlaylist
}

6
server/tests/shared/videos.ts
View file

@ -125,9 +125,9 @@ async function completeVideoCheck (
expect(file.fileDownloadUrl).to.match(new RegExp(`http://${originHost}/download/videos/${uuidRegex}-${file.resolution.id}${extension}`))
await Promise.all([
makeRawRequest(file.torrentUrl, 200),
makeRawRequest(file.torrentDownloadUrl, 200),
makeRawRequest(file.metadataUrl, 200)
makeRawRequest({ url: file.torrentUrl, expectedStatus: HttpStatusCode.OK_200 }),
makeRawRequest({ url: file.torrentDownloadUrl, expectedStatus: HttpStatusCode.OK_200 }),
makeRawRequest({ url: file.metadataUrl, expectedStatus: HttpStatusCode.OK_200 })
])
expect(file.resolution.id).to.equal(attributeFile.resolution)

12
shared/core-utils/common/url.ts
View file

@ -1,6 +1,16 @@
import { Video, VideoPlaylist } from '../../models'
import { secondsToTime } from './date'
function addQueryParams (url: string, params: { [ id: string ]: string }) {
const objUrl = new URL(url)
for (const key of Object.keys(params)) {
objUrl.searchParams.append(key, params[key])
}
return objUrl.toString()
}
function buildPlaylistLink (playlist: Pick<VideoPlaylist, 'shortUUID'>, base?: string) {
return (base ?? window.location.origin) + buildPlaylistWatchPath(playlist)
}
@ -103,6 +113,8 @@ function decoratePlaylistLink (options: {
// ---------------------------------------------------------------------------
export {
addQueryParams,
buildPlaylistLink,
buildVideoLink,

1
shared/models/server/debug.model.ts
View file

@ -8,4 +8,5 @@ export interface SendDebugCommand {
| 'process-video-views-buffer'
| 'process-video-viewers'
| 'process-video-channel-sync-latest'
| 'process-update-videos-scheduler'
}

2
shared/models/videos/index.ts
View file

@ -33,6 +33,8 @@ export * from './video-storage.enum'
export * from './video-streaming-playlist.model'
export * from './video-streaming-playlist.type'
export * from './video-token.model'
export * from './video-update.model'
export * from './video-view.model'
export * from './video.model'

6
shared/models/videos/video-token.model.ts Normal file
View file

@ -0,0 +1,6 @@
export interface VideoToken {
files: {
token: string
expires: string | Date
}
}

19
shared/server-commands/requests/requests.ts
View file

@ -3,7 +3,7 @@
import { decode } from 'querystring'
import request from 'supertest'
import { URL } from 'url'
import { buildAbsoluteFixturePath } from '@shared/core-utils'
import { buildAbsoluteFixturePath, pick } from '@shared/core-utils'
import { HttpStatusCode } from '@shared/models'
export type CommonRequestParams = {
@ -21,10 +21,21 @@ export type CommonRequestParams = {
expectedStatus?: HttpStatusCode
}
function makeRawRequest (url: string, expectedStatus?: HttpStatusCode, range?: string) {
const { host, protocol, pathname } = new URL(url)
function makeRawRequest (options: {
url: string
token?: string
expectedStatus?: HttpStatusCode
range?: string
query?: { [ id: string ]: string }
}) {
const { host, protocol, pathname } = new URL(options.url)
return makeGetRequest({ url: `${protocol}//${host}`, path: pathname, expectedStatus, range })
return makeGetRequest({
url: `${protocol}//${host}`,
path: pathname,
...pick(options, [ 'expectedStatus', 'range', 'token', 'query' ])
})
}
function makeGetRequest (options: CommonRequestParams & {

3
shared/server-commands/server/server.ts
View file

@ -36,6 +36,7 @@ import {
StreamingPlaylistsCommand,
VideosCommand,
VideoStudioCommand,
VideoTokenCommand,
ViewsCommand
} from '../videos'
import { CommentsCommand } from '../videos/comments-command'
@ -145,6 +146,7 @@ export class PeerTubeServer {
videoStats?: VideoStatsCommand
views?: ViewsCommand
twoFactor?: TwoFactorCommand
videoToken?: VideoTokenCommand
constructor (options: { serverNumber: number } | { url: string }) {
if ((options as any).url) {
@ -427,5 +429,6 @@ export class PeerTubeServer {
this.videoStats = new VideoStatsCommand(this)
this.views = new ViewsCommand(this)
this.twoFactor = new TwoFactorCommand(this)
this.videoToken = new VideoTokenCommand(this)
}
}

1
shared/server-commands/videos/index.ts
View file

@ -14,5 +14,6 @@ export * from './services-command'
export * from './streaming-playlists-command'
export * from './comments-command'
export * from './video-studio-command'
export * from './video-token-command'
export * from './views-command'
export * from './videos-command'

26
shared/server-commands/videos/live-command.ts
View file

@ -12,6 +12,7 @@ import {
ResultList,
VideoCreateResult,
VideoDetails,
VideoPrivacy,
VideoState
} from '@shared/models'
import { unwrapBody } from '../requests'
@ -115,6 +116,31 @@ export class LiveCommand extends AbstractCommand {
return body.video
}
async quickCreate (options: OverrideCommandOptions & {
saveReplay: boolean
permanentLive: boolean
privacy?: VideoPrivacy
}) {
const { saveReplay, permanentLive, privacy } = options
const { uuid } = await this.create({
...options,
fields: {
name: 'live',
permanentLive,
saveReplay,
channelId: this.server.store.channel.id,
privacy
}
})
const video = await this.server.videos.getWithToken({ id: uuid })
const live = await this.get({ videoId: uuid })
return { video, live }
}
// ---------------------------------------------------------------------------
async sendRTMPStreamInVideo (options: OverrideCommandOptions & {

7
shared/server-commands/videos/live.ts
View file

@ -1,6 +1,6 @@
import ffmpeg, { FfmpegCommand } from 'fluent-ffmpeg'
import { buildAbsoluteFixturePath, wait } from '@shared/core-utils'
import { VideoDetails, VideoInclude } from '@shared/models'
import { VideoDetails, VideoInclude, VideoPrivacy } from '@shared/models'
import { PeerTubeServer } from '../server/server'
function sendRTMPStream (options: {
@ -98,7 +98,10 @@ async function waitUntilLiveReplacedByReplayOnAllServers (servers: PeerTubeServe
}
async function findExternalSavedVideo (server: PeerTubeServer, liveDetails: VideoDetails) {
const { data } = await server.videos.list({ token: server.accessToken, sort: '-publishedAt', include: VideoInclude.BLACKLISTED })
const include = VideoInclude.BLACKLISTED
const privacyOneOf = [ VideoPrivacy.INTERNAL, VideoPrivacy.PRIVATE, VideoPrivacy.PUBLIC, VideoPrivacy.UNLISTED ]
const { data } = await server.videos.list({ token: server.accessToken, sort: '-publishedAt', include, privacyOneOf })
return data.find(v => v.name === liveDetails.name + ' - ' + new Date(liveDetails.publishedAt).toLocaleString())
}

Some files were not shown because too many files have changed in this diff Show more