Implement replace file in server side

This commit is contained in:
Chocobozzz 2023-07-19 16:02:49 +02:00
parent c6867725fb
commit 12dc3a942a
No known key found for this signature in database
GPG key ID: 583A612D890159BE
55 changed files with 1547 additions and 325 deletions

View file

@ -595,6 +595,11 @@ video_studio:
remote_runners:
enabled: false
video_file:
update:
# Add ability for users to replace the video file of an existing video
enabled: false
import:
# Add ability for your users to import remote videos (from YouTube, torrent...)
videos:

View file

@ -605,6 +605,11 @@ video_studio:
remote_runners:
enabled: false
video_file:
update:
# Add ability for users to replace the video file of an existing video
enabled: false
import:
# Add ability for your users to import remote videos (from YouTube, torrent...)
videos:

View file

@ -284,6 +284,11 @@ function customConfig (): CustomConfig {
enabled: CONFIG.VIDEO_STUDIO.REMOTE_RUNNERS.ENABLED
}
},
videoFile: {
update: {
enabled: CONFIG.VIDEO_FILE.UPDATE.ENABLED
}
},
import: {
videos: {
concurrency: CONFIG.IMPORT.VIDEOS.CONCURRENCY,

View file

@ -26,7 +26,6 @@ import {
setDefaultVideosSort,
videosCustomGetValidator,
videosGetValidator,
videoSourceGetValidator,
videosRemoveValidator,
videosSortValidator
} from '../../../middlewares'
@ -39,7 +38,9 @@ import { filesRouter } from './files'
import { videoImportsRouter } from './import'
import { liveRouter } from './live'
import { ownershipVideoRouter } from './ownership'
import { videoPasswordRouter } from './passwords'
import { rateVideoRouter } from './rate'
import { videoSourceRouter } from './source'
import { statsRouter } from './stats'
import { storyboardRouter } from './storyboard'
import { studioRouter } from './studio'
@ -48,7 +49,6 @@ import { transcodingRouter } from './transcoding'
import { updateRouter } from './update'
import { uploadRouter } from './upload'
import { viewRouter } from './view'
import { videoPasswordRouter } from './passwords'
const auditLogger = auditLoggerFactory('videos')
const videosRouter = express.Router()
@ -72,6 +72,7 @@ videosRouter.use('/', transcodingRouter)
videosRouter.use('/', tokenRouter)
videosRouter.use('/', videoPasswordRouter)
videosRouter.use('/', storyboardRouter)
videosRouter.use('/', videoSourceRouter)
videosRouter.get('/categories',
openapiOperationDoc({ operationId: 'getCategories' }),
@ -108,13 +109,6 @@ videosRouter.get('/:id/description',
asyncMiddleware(getVideoDescription)
)
videosRouter.get('/:id/source',
openapiOperationDoc({ operationId: 'getVideoSource' }),
authenticate,
asyncMiddleware(videoSourceGetValidator),
getVideoSource
)
videosRouter.get('/:id',
openapiOperationDoc({ operationId: 'getVideo' }),
optionalAuthenticate,
@ -177,10 +171,6 @@ async function getVideoDescription (req: express.Request, res: express.Response)
return res.json({ description })
}
function getVideoSource (req: express.Request, res: express.Response) {
return res.json(res.locals.videoSource.toFormattedJSON())
}
async function listVideos (req: express.Request, res: express.Response) {
const serverActor = await getServerActor()

View file

@ -0,0 +1,206 @@
import express from 'express'
import { move } from 'fs-extra'
import { sequelizeTypescript } from '@server/initializers/database'
import { CreateJobArgument, CreateJobOptions, JobQueue } from '@server/lib/job-queue'
import { Hooks } from '@server/lib/plugins/hooks'
import { regenerateMiniaturesIfNeeded } from '@server/lib/thumbnail'
import { uploadx } from '@server/lib/uploadx'
import { buildMoveToObjectStorageJob } from '@server/lib/video'
import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist'
import { buildNewFile } from '@server/lib/video-file'
import { VideoPathManager } from '@server/lib/video-path-manager'
import { buildNextVideoState } from '@server/lib/video-state'
import { openapiOperationDoc } from '@server/middlewares/doc'
import { VideoModel } from '@server/models/video/video'
import { VideoSourceModel } from '@server/models/video/video-source'
import { MStreamingPlaylistFiles, MVideo, MVideoFile, MVideoFullLight } from '@server/types/models'
import { HttpStatusCode, VideoState } from '@shared/models'
import { logger, loggerTagsFactory } from '../../../helpers/logger'
import {
asyncMiddleware,
authenticate,
replaceVideoSourceResumableInitValidator,
replaceVideoSourceResumableValidator,
videoSourceGetLatestValidator
} from '../../../middlewares'
const lTags = loggerTagsFactory('api', 'video')
const videoSourceRouter = express.Router()
videoSourceRouter.get('/:id/source',
openapiOperationDoc({ operationId: 'getVideoSource' }),
authenticate,
asyncMiddleware(videoSourceGetLatestValidator),
getVideoLatestSource
)
videoSourceRouter.post('/:id/source/replace-resumable',
authenticate,
asyncMiddleware(replaceVideoSourceResumableInitValidator),
(req, res) => uploadx.upload(req, res) // Prevent next() call, explicitely tell to uploadx it's the end
)
videoSourceRouter.delete('/:id/source/replace-resumable',
authenticate,
(req, res) => uploadx.upload(req, res) // Prevent next() call, explicitely tell to uploadx it's the end
)
videoSourceRouter.put('/:id/source/replace-resumable',
authenticate,
uploadx.upload, // uploadx doesn't next() before the file upload completes
asyncMiddleware(replaceVideoSourceResumableValidator),
asyncMiddleware(replaceVideoSourceResumable)
)
// ---------------------------------------------------------------------------
export {
videoSourceRouter
}
// ---------------------------------------------------------------------------
function getVideoLatestSource (req: express.Request, res: express.Response) {
return res.json(res.locals.videoSource.toFormattedJSON())
}
async function replaceVideoSourceResumable (req: express.Request, res: express.Response) {
const videoPhysicalFile = res.locals.updateVideoFileResumable
const user = res.locals.oauth.token.User
const videoFile = await buildNewFile({ path: videoPhysicalFile.path, mode: 'web-video' })
const originalFilename = videoPhysicalFile.originalname
const videoFileMutexReleaser = await VideoPathManager.Instance.lockFiles(res.locals.videoAll.uuid)
try {
const destination = VideoPathManager.Instance.getFSVideoFileOutputPath(res.locals.videoAll, videoFile)
await move(videoPhysicalFile.path, destination)
let oldWebVideoFiles: MVideoFile[] = []
let oldStreamingPlaylists: MStreamingPlaylistFiles[] = []
const inputFileUpdatedAt = new Date()
const video = await sequelizeTypescript.transaction(async transaction => {
const video = await VideoModel.loadFull(res.locals.videoAll.id, transaction)
oldWebVideoFiles = video.VideoFiles
oldStreamingPlaylists = video.VideoStreamingPlaylists
for (const file of video.VideoFiles) {
await file.destroy({ transaction })
}
for (const playlist of oldStreamingPlaylists) {
await playlist.destroy({ transaction })
}
videoFile.videoId = video.id
await videoFile.save({ transaction })
video.VideoFiles = [ videoFile ]
video.VideoStreamingPlaylists = []
video.state = buildNextVideoState()
video.duration = videoPhysicalFile.duration
video.inputFileUpdatedAt = inputFileUpdatedAt
await video.save({ transaction })
await autoBlacklistVideoIfNeeded({
video,
user,
isRemote: false,
isNew: false,
isNewFile: true,
transaction
})
return video
})
await removeOldFiles({ video, files: oldWebVideoFiles, playlists: oldStreamingPlaylists })
await VideoSourceModel.create({
filename: originalFilename,
videoId: video.id,
createdAt: inputFileUpdatedAt
})
await regenerateMiniaturesIfNeeded(video)
await video.VideoChannel.setAsUpdated()
await addVideoJobsAfterUpload(video, video.getMaxQualityFile())
logger.info('Replaced video file of video %s with uuid %s.', video.name, video.uuid, lTags(video.uuid))
Hooks.runAction('action:api.video.file-updated', { video, req, res })
return res.sendStatus(HttpStatusCode.NO_CONTENT_204)
} finally {
videoFileMutexReleaser()
}
}
async function addVideoJobsAfterUpload (video: MVideoFullLight, videoFile: MVideoFile) {
const jobs: (CreateJobArgument & CreateJobOptions)[] = [
{
type: 'manage-video-torrent' as 'manage-video-torrent',
payload: {
videoId: video.id,
videoFileId: videoFile.id,
action: 'create'
}
},
{
type: 'generate-video-storyboard' as 'generate-video-storyboard',
payload: {
videoUUID: video.uuid,
// No need to federate, we process these jobs sequentially
federate: false
}
},
{
type: 'federate-video' as 'federate-video',
payload: {
videoUUID: video.uuid,
isNewVideo: false
}
}
]
if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) {
jobs.push(await buildMoveToObjectStorageJob({ video, isNewVideo: false, previousVideoState: undefined }))
}
if (video.state === VideoState.TO_TRANSCODE) {
jobs.push({
type: 'transcoding-job-builder' as 'transcoding-job-builder',
payload: {
videoUUID: video.uuid,
optimizeJob: {
isNewVideo: false
}
}
})
}
return JobQueue.Instance.createSequentialJobFlow(...jobs)
}
async function removeOldFiles (options: {
video: MVideo
files: MVideoFile[]
playlists: MStreamingPlaylistFiles[]
}) {
const { video, files, playlists } = options
for (const file of files) {
await video.removeWebVideoFile(file)
}
for (const playlist of playlists) {
await video.removeStreamingPlaylistFiles(playlist)
}
}

View file

@ -130,6 +130,7 @@ async function updateVideo (req: express.Request, res: express.Response) {
user: res.locals.oauth.token.User,
isRemote: false,
isNew: false,
isNewFile: false,
transaction: t
})

View file

@ -11,8 +11,9 @@ import { buildNewFile } from '@server/lib/video-file'
import { VideoPathManager } from '@server/lib/video-path-manager'
import { buildNextVideoState } from '@server/lib/video-state'
import { openapiOperationDoc } from '@server/middlewares/doc'
import { VideoPasswordModel } from '@server/models/video/video-password'
import { VideoSourceModel } from '@server/models/video/video-source'
import { MUserId, MVideoFile, MVideoFullLight } from '@server/types/models'
import { MVideoFile, MVideoFullLight } from '@server/types/models'
import { uuidToShort } from '@shared/extra-utils'
import { HttpStatusCode, VideoCreate, VideoPrivacy, VideoState } from '@shared/models'
import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger'
@ -33,7 +34,6 @@ import {
} from '../../../middlewares'
import { ScheduleVideoUpdateModel } from '../../../models/video/schedule-video-update'
import { VideoModel } from '../../../models/video/video'
import { VideoPasswordModel } from '@server/models/video/video-password'
const lTags = loggerTagsFactory('api', 'video')
const auditLogger = auditLoggerFactory('videos')
@ -109,7 +109,7 @@ async function addVideoLegacy (req: express.Request, res: express.Response) {
}
async function addVideoResumable (req: express.Request, res: express.Response) {
const videoPhysicalFile = res.locals.videoFileResumable
const videoPhysicalFile = res.locals.uploadVideoFileResumable
const videoInfo = videoPhysicalFile.metadata
const files = { previewfile: videoInfo.previewfile, thumbnailfile: videoInfo.thumbnailfile }
@ -193,6 +193,7 @@ async function addVideo (options: {
user,
isRemote: false,
isNew: true,
isNewFile: true,
transaction: t
})
@ -209,7 +210,7 @@ async function addVideo (options: {
// Channel has a new content, set as updated
await videoCreated.VideoChannel.setAsUpdated()
addVideoJobsAfterUpload(videoCreated, videoFile, user)
addVideoJobsAfterUpload(videoCreated, videoFile)
.catch(err => logger.error('Cannot build new video jobs of %s.', videoCreated.uuid, { err, ...lTags(videoCreated.uuid) }))
Hooks.runAction('action:api.video.uploaded', { video: videoCreated, req, res })
@ -223,7 +224,7 @@ async function addVideo (options: {
}
}
async function addVideoJobsAfterUpload (video: MVideoFullLight, videoFile: MVideoFile, user: MUserId) {
async function addVideoJobsAfterUpload (video: MVideoFullLight, videoFile: MVideoFile) {
const jobs: (CreateJobArgument & CreateJobOptions)[] = [
{
type: 'manage-video-torrent' as 'manage-video-torrent',

View file

@ -76,6 +76,7 @@ function sanitizeAndCheckVideoTorrentObject (video: any) {
isDateValid(video.published) &&
isDateValid(video.updated) &&
(!video.originallyPublishedAt || isDateValid(video.originallyPublishedAt)) &&
(!video.uploadDate || isDateValid(video.uploadDate)) &&
(!video.content || isRemoteVideoContentValid(video.mediaType, video.content)) &&
video.attributedTo.length !== 0
}

View file

@ -63,6 +63,8 @@ async function generateImageFromVideoFile (options: {
} catch (err) {
logger.debug('Cannot remove pending image path after generation error.', { err, ...lTags() })
}
throw err
}
}

View file

@ -40,6 +40,7 @@ function checkMissedConfig () {
'transcoding.resolutions.480p', 'transcoding.resolutions.720p', 'transcoding.resolutions.1080p', 'transcoding.resolutions.1440p',
'transcoding.resolutions.2160p', 'transcoding.always_transcode_original_resolution', 'transcoding.remote_runners.enabled',
'video_studio.enabled', 'video_studio.remote_runners.enabled',
'video_file.update.enabled',
'remote_runners.stalled_jobs.vod', 'remote_runners.stalled_jobs.live',
'import.videos.http.enabled', 'import.videos.torrent.enabled', 'import.videos.concurrency', 'import.videos.timeout',
'import.video_channel_synchronization.enabled', 'import.video_channel_synchronization.max_per_user',

View file

@ -435,6 +435,11 @@ const CONFIG = {
get ENABLED () { return config.get<boolean>('video_studio.remote_runners.enabled') }
}
},
VIDEO_FILE: {
UPDATE: {
get ENABLED () { return config.get<boolean>('video_file.update.enabled') }
}
},
IMPORT: {
VIDEOS: {
get CONCURRENCY () { return config.get<number>('import.videos.concurrency') },

View file

@ -27,7 +27,7 @@ import { CONFIG, registerConfigChangedHandler } from './config'
// ---------------------------------------------------------------------------
const LAST_MIGRATION_VERSION = 795
const LAST_MIGRATION_VERSION = 800
// ---------------------------------------------------------------------------

View file

@ -0,0 +1,38 @@
import * as Sequelize from 'sequelize'
async function up (utils: {
transaction: Sequelize.Transaction
queryInterface: Sequelize.QueryInterface
sequelize: Sequelize.Sequelize
}): Promise<void> {
const { transaction } = utils
{
const query = 'DELETE FROM "videoSource" WHERE "videoId" IS NULL'
await utils.sequelize.query(query, { transaction })
}
{
const query = 'ALTER TABLE "videoSource" ALTER COLUMN "videoId" SET NOT NULL'
await utils.sequelize.query(query, { transaction })
}
{
const data = {
type: Sequelize.DATE,
allowNull: true,
defaultValue: null
}
await utils.queryInterface.addColumn('video', 'inputFileUpdatedAt', data, { transaction })
}
}
function down (options) {
throw new Error('Not implemented.')
}
export {
up,
down
}

View file

@ -60,6 +60,9 @@ const contextStore: { [ id in ContextType ]: (string | { [ id: string ]: string
},
originallyPublishedAt: 'sc:datePublished',
uploadDate: 'sc:uploadDate',
views: {
'@type': 'sc:Number',
'@id': 'pt:views'

View file

@ -49,6 +49,7 @@ export class APVideoCreator extends APVideoAbstractBuilder {
user: undefined,
isRemote: true,
isNew: true,
isNewFile: true,
transaction: t
})

View file

@ -231,6 +231,10 @@ function getVideoAttributesFromObject (videoChannel: MChannelId, videoObject: Vi
? new Date(videoObject.originallyPublishedAt)
: null,
inputFileUpdatedAt: videoObject.uploadDate
? new Date(videoObject.uploadDate)
: null,
updatedAt: new Date(videoObject.updated),
views: videoObject.views,
remote: true,

View file

@ -38,6 +38,8 @@ export class APVideoUpdater extends APVideoAbstractBuilder {
{ videoObject: this.videoObject, ...this.lTags() }
)
const oldInputFileUpdatedAt = this.video.inputFileUpdatedAt
try {
const channelActor = await this.getOrCreateVideoChannelFromVideoObject()
@ -74,6 +76,7 @@ export class APVideoUpdater extends APVideoAbstractBuilder {
user: undefined,
isRemote: true,
isNew: false,
isNewFile: oldInputFileUpdatedAt !== videoUpdated.inputFileUpdatedAt,
transaction: undefined
})
@ -129,6 +132,7 @@ export class APVideoUpdater extends APVideoAbstractBuilder {
this.video.createdAt = videoData.createdAt
this.video.publishedAt = videoData.publishedAt
this.video.originallyPublishedAt = videoData.originallyPublishedAt
this.video.inputFileUpdatedAt = videoData.inputFileUpdatedAt
this.video.privacy = videoData.privacy
this.video.channelId = videoData.channelId
this.video.views = videoData.views

View file

@ -7,7 +7,7 @@ import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url'
import { federateVideoIfNeeded } from '@server/lib/activitypub/videos'
import { cleanupAndDestroyPermanentLive, cleanupTMPLiveFiles, cleanupUnsavedNormalLive } from '@server/lib/live'
import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getLiveReplayBaseDirectory } from '@server/lib/paths'
import { generateLocalVideoMiniature } from '@server/lib/thumbnail'
import { generateLocalVideoMiniature, regenerateMiniaturesIfNeeded } from '@server/lib/thumbnail'
import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/hls-transcoding'
import { VideoPathManager } from '@server/lib/video-path-manager'
import { moveToNextState } from '@server/lib/video-state'
@ -197,23 +197,7 @@ async function replaceLiveByReplay (options: {
}
// Regenerate the thumbnail & preview?
if (videoWithFiles.getMiniature().automaticallyGenerated === true) {
const miniature = await generateLocalVideoMiniature({
video: videoWithFiles,
videoFile: videoWithFiles.getMaxQualityFile(),
type: ThumbnailType.MINIATURE
})
await videoWithFiles.addAndSaveThumbnail(miniature)
}
if (videoWithFiles.getPreview().automaticallyGenerated === true) {
const preview = await generateLocalVideoMiniature({
video: videoWithFiles,
videoFile: videoWithFiles.getMaxQualityFile(),
type: ThumbnailType.PREVIEW
})
await videoWithFiles.addAndSaveThumbnail(preview)
}
await regenerateMiniaturesIfNeeded(videoWithFiles)
// We consider this is a new video
await moveToNextState({ video: videoWithFiles, isNewVideo: true })

View file

@ -36,7 +36,7 @@ export type AcceptResult = {
// ---------------------------------------------------------------------------
// Stub function that can be filtered by plugins
function isLocalVideoAccepted (object: {
function isLocalVideoFileAccepted (object: {
videoBody: VideoCreate
videoFile: VideoUploadFile
user: UserModel
@ -201,7 +201,7 @@ function createAccountAbuse (options: {
export {
isLocalLiveVideoAccepted,
isLocalVideoAccepted,
isLocalVideoFileAccepted,
isLocalVideoThreadAccepted,
isRemoteVideoCommentAccepted,
isLocalVideoCommentReplyAccepted,

View file

@ -171,6 +171,11 @@ class ServerConfigManager {
enabled: CONFIG.VIDEO_STUDIO.REMOTE_RUNNERS.ENABLED
}
},
videoFile: {
update: {
enabled: CONFIG.VIDEO_FILE.UPDATE.ENABLED
}
},
import: {
videos: {
http: {

View file

@ -4,7 +4,7 @@ import { generateImageFilename, generateImageFromVideoFile } from '../helpers/im
import { CONFIG } from '../initializers/config'
import { ASSETS_PATH, PREVIEWS_SIZE, THUMBNAILS_SIZE } from '../initializers/constants'
import { ThumbnailModel } from '../models/video/thumbnail'
import { MVideoFile, MVideoThumbnail, MVideoUUID } from '../types/models'
import { MVideoFile, MVideoThumbnail, MVideoUUID, MVideoWithAllFiles } from '../types/models'
import { MThumbnail } from '../types/models/video/thumbnail'
import { MVideoPlaylistThumbnail } from '../types/models/video/video-playlist'
import { VideoPathManager } from './video-path-manager'
@ -187,8 +187,31 @@ function updateRemoteVideoThumbnail (options: {
// ---------------------------------------------------------------------------
async function regenerateMiniaturesIfNeeded (video: MVideoWithAllFiles) {
if (video.getMiniature().automaticallyGenerated === true) {
const miniature = await generateLocalVideoMiniature({
video,
videoFile: video.getMaxQualityFile(),
type: ThumbnailType.MINIATURE
})
await video.addAndSaveThumbnail(miniature)
}
if (video.getPreview().automaticallyGenerated === true) {
const preview = await generateLocalVideoMiniature({
video,
videoFile: video.getMaxQualityFile(),
type: ThumbnailType.PREVIEW
})
await video.addAndSaveThumbnail(preview)
}
}
// ---------------------------------------------------------------------------
export {
generateLocalVideoMiniature,
regenerateMiniaturesIfNeeded,
updateLocalVideoMiniatureFromUrl,
updateLocalVideoMiniatureFromExisting,
updateRemoteVideoThumbnail,

View file

@ -27,13 +27,14 @@ async function autoBlacklistVideoIfNeeded (parameters: {
user?: MUser
isRemote: boolean
isNew: boolean
isNewFile: boolean
notify?: boolean
transaction?: Transaction
}) {
const { video, user, isRemote, isNew, notify = true, transaction } = parameters
const { video, user, isRemote, isNew, isNewFile, notify = true, transaction } = parameters
const doAutoBlacklist = await Hooks.wrapFun(
autoBlacklistNeeded,
{ video, user, isRemote, isNew },
{ video, user, isRemote, isNew, isNewFile },
'filter:video.auto-blacklist.result'
)
@ -128,14 +129,15 @@ function autoBlacklistNeeded (parameters: {
video: MVideoWithBlacklistLight
isRemote: boolean
isNew: boolean
isNewFile: boolean
user?: MUser
}) {
const { user, video, isRemote, isNew } = parameters
const { user, video, isRemote, isNew, isNewFile } = parameters
// Already blacklisted
if (video.VideoBlacklist) return false
if (!CONFIG.AUTO_BLACKLIST.VIDEOS.OF_USERS.ENABLED || !user) return false
if (isRemote || isNew === false) return false
if (isRemote || (isNew === false && isNewFile === false)) return false
if (user.hasRight(UserRight.MANAGE_VIDEO_BLACKLIST) || user.hasAdminFlag(UserAdminFlag.BYPASS_VIDEO_AUTO_BLACKLIST)) return false

View file

@ -89,6 +89,7 @@ async function insertFromImportIntoDB (parameters: {
notify: false,
isRemote: false,
isNew: true,
isNewFile: true,
transaction: t
})

View file

@ -65,6 +65,8 @@ const customConfigUpdateValidator = [
body('videoStudio.enabled').isBoolean(),
body('videoStudio.remoteRunners.enabled').isBoolean(),
body('videoFile.update.enabled').isBoolean(),
body('import.videos.concurrency').isInt({ min: 0 }),
body('import.videos.http.enabled').isBoolean(),
body('import.videos.torrent.enabled').isBoolean(),

View file

@ -1,12 +1,13 @@
export * from './video-blacklist'
export * from './video-captions'
export * from './video-channel-sync'
export * from './video-channels'
export * from './video-comments'
export * from './video-files'
export * from './video-imports'
export * from './video-live'
export * from './video-ownership-changes'
export * from './video-view'
export * from './video-passwords'
export * from './video-rates'
export * from './video-shares'
export * from './video-source'
@ -14,6 +15,5 @@ export * from './video-stats'
export * from './video-studio'
export * from './video-token'
export * from './video-transcoding'
export * from './video-view'
export * from './videos'
export * from './video-channel-sync'
export * from './video-passwords'

View file

@ -0,0 +1,2 @@
export * from './upload'
export * from './video-validators'

View file

@ -0,0 +1,39 @@
import express from 'express'
import { logger } from '@server/helpers/logger'
import { getVideoStreamDuration } from '@shared/ffmpeg'
import { HttpStatusCode } from '@shared/models'
export async function addDurationToVideoFileIfNeeded (options: {
res: express.Response
videoFile: { path: string, duration?: number }
middlewareName: string
}) {
const { res, middlewareName, videoFile } = options
try {
if (!videoFile.duration) await addDurationToVideo(videoFile)
} catch (err) {
logger.error('Invalid input file in ' + middlewareName, { err })
res.fail({
status: HttpStatusCode.UNPROCESSABLE_ENTITY_422,
message: 'Video file unreadable.'
})
return false
}
return true
}
// ---------------------------------------------------------------------------
// Private
// ---------------------------------------------------------------------------
async function addDurationToVideo (videoFile: { path: string, duration?: number }) {
const duration = await getVideoStreamDuration(videoFile.path)
// FFmpeg may not be able to guess video duration
// For example with m2v files: https://trac.ffmpeg.org/ticket/9726#comment:2
if (isNaN(duration)) videoFile.duration = 0
else videoFile.duration = duration
}

View file

@ -0,0 +1,104 @@
import express from 'express'
import { isVideoFileMimeTypeValid, isVideoFileSizeValid } from '@server/helpers/custom-validators/videos'
import { logger } from '@server/helpers/logger'
import { CONSTRAINTS_FIELDS } from '@server/initializers/constants'
import { isLocalVideoFileAccepted } from '@server/lib/moderation'
import { Hooks } from '@server/lib/plugins/hooks'
import { MUserAccountId, MVideo } from '@server/types/models'
import { HttpStatusCode, ServerErrorCode, ServerFilterHookName, VideoState } from '@shared/models'
import { checkUserQuota } from '../../shared'
export async function commonVideoFileChecks (options: {
res: express.Response
user: MUserAccountId
videoFileSize: number
files: express.UploadFilesForCheck
}): Promise<boolean> {
const { res, user, videoFileSize, files } = options
if (!isVideoFileMimeTypeValid(files)) {
res.fail({
status: HttpStatusCode.UNSUPPORTED_MEDIA_TYPE_415,
message: 'This file is not supported. Please, make sure it is of the following type: ' +
CONSTRAINTS_FIELDS.VIDEOS.EXTNAME.join(', ')
})
return false
}
if (!isVideoFileSizeValid(videoFileSize.toString())) {
res.fail({
status: HttpStatusCode.PAYLOAD_TOO_LARGE_413,
message: 'This file is too large. It exceeds the maximum file size authorized.',
type: ServerErrorCode.MAX_FILE_SIZE_REACHED
})
return false
}
if (await checkUserQuota(user, videoFileSize, res) === false) return false
return true
}
export async function isVideoFileAccepted (options: {
req: express.Request
res: express.Response
videoFile: express.VideoUploadFile
hook: Extract<ServerFilterHookName, 'filter:api.video.upload.accept.result' | 'filter:api.video.update-file.accept.result'>
}) {
const { req, res, videoFile } = options
// Check we accept this video
const acceptParameters = {
videoBody: req.body,
videoFile,
user: res.locals.oauth.token.User
}
const acceptedResult = await Hooks.wrapFun(
isLocalVideoFileAccepted,
acceptParameters,
'filter:api.video.upload.accept.result'
)
if (!acceptedResult || acceptedResult.accepted !== true) {
logger.info('Refused local video file.', { acceptedResult, acceptParameters })
res.fail({
status: HttpStatusCode.FORBIDDEN_403,
message: acceptedResult.errorMessage || 'Refused local video file'
})
return false
}
return true
}
export function checkVideoFileCanBeEdited (video: MVideo, res: express.Response) {
if (video.isLive) {
res.fail({
status: HttpStatusCode.BAD_REQUEST_400,
message: 'Cannot edit a live video'
})
return false
}
if (video.state === VideoState.TO_TRANSCODE || video.state === VideoState.TO_EDIT) {
res.fail({
status: HttpStatusCode.CONFLICT_409,
message: 'Cannot edit video that is already waiting for transcoding/edition'
})
return false
}
const validStates = new Set([ VideoState.PUBLISHED, VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED, VideoState.TRANSCODING_FAILED ])
if (!validStates.has(video.state)) {
res.fail({
status: HttpStatusCode.BAD_REQUEST_400,
message: 'Video state is not compatible with edition'
})
return false
}
return true
}

View file

@ -1,20 +1,31 @@
import express from 'express'
import { body, header } from 'express-validator'
import { getResumableUploadPath } from '@server/helpers/upload'
import { getVideoWithAttributes } from '@server/helpers/video'
import { CONFIG } from '@server/initializers/config'
import { uploadx } from '@server/lib/uploadx'
import { VideoSourceModel } from '@server/models/video/video-source'
import { MVideoFullLight } from '@server/types/models'
import { HttpStatusCode, UserRight } from '@shared/models'
import { Metadata as UploadXMetadata } from '@uploadx/core'
import { logger } from '../../../helpers/logger'
import { areValidationErrors, checkUserCanManageVideo, doesVideoExist, isValidVideoIdParam } from '../shared'
import { addDurationToVideoFileIfNeeded, checkVideoFileCanBeEdited, commonVideoFileChecks, isVideoFileAccepted } from './shared'
const videoSourceGetValidator = [
export const videoSourceGetLatestValidator = [
isValidVideoIdParam('id'),
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
if (areValidationErrors(req, res)) return
if (!await doesVideoExist(req.params.id, res, 'for-api')) return
if (!await doesVideoExist(req.params.id, res, 'all')) return
const video = getVideoWithAttributes(res) as MVideoFullLight
res.locals.videoSource = await VideoSourceModel.loadByVideoId(video.id)
const user = res.locals.oauth.token.User
if (!checkUserCanManageVideo(user, video, UserRight.UPDATE_ANY_VIDEO, res)) return
res.locals.videoSource = await VideoSourceModel.loadLatest(video.id)
if (!res.locals.videoSource) {
return res.fail({
status: HttpStatusCode.NOT_FOUND_404,
@ -22,13 +33,98 @@ const videoSourceGetValidator = [
})
}
const user = res.locals.oauth.token.User
if (!checkUserCanManageVideo(user, video, UserRight.UPDATE_ANY_VIDEO, res)) return
return next()
}
]
export const replaceVideoSourceResumableValidator = [
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
const body: express.CustomUploadXFile<UploadXMetadata> = req.body
const file = { ...body, duration: undefined, path: getResumableUploadPath(body.name), filename: body.metadata.filename }
const cleanup = () => uploadx.storage.delete(file).catch(err => logger.error('Cannot delete the file %s', file.name, { err }))
if (!await checkCanUpdateVideoFile({ req, res })) {
return cleanup()
}
if (!await addDurationToVideoFileIfNeeded({ videoFile: file, res, middlewareName: 'updateVideoFileResumableValidator' })) {
return cleanup()
}
if (!await isVideoFileAccepted({ req, res, videoFile: file, hook: 'filter:api.video.update-file.accept.result' })) {
return cleanup()
}
res.locals.updateVideoFileResumable = { ...file, originalname: file.filename }
return next()
}
]
export {
videoSourceGetValidator
export const replaceVideoSourceResumableInitValidator = [
body('filename')
.exists(),
header('x-upload-content-length')
.isNumeric()
.exists()
.withMessage('Should specify the file length'),
header('x-upload-content-type')
.isString()
.exists()
.withMessage('Should specify the file mimetype'),
async (req: express.Request, res: express.Response, next: express.NextFunction) => {
const user = res.locals.oauth.token.User
logger.debug('Checking updateVideoFileResumableInitValidator parameters and headers', {
parameters: req.body,
headers: req.headers
})
if (areValidationErrors(req, res, { omitLog: true })) return
if (!await checkCanUpdateVideoFile({ req, res })) return
const videoFileMetadata = {
mimetype: req.headers['x-upload-content-type'] as string,
size: +req.headers['x-upload-content-length'],
originalname: req.body.filename
}
const files = { videofile: [ videoFileMetadata ] }
if (await commonVideoFileChecks({ res, user, videoFileSize: videoFileMetadata.size, files }) === false) return
return next()
}
]
// ---------------------------------------------------------------------------
// Private
// ---------------------------------------------------------------------------
async function checkCanUpdateVideoFile (options: {
req: express.Request
res: express.Response
}) {
const { req, res } = options
if (!CONFIG.VIDEO_FILE.UPDATE.ENABLED) {
res.fail({
status: HttpStatusCode.FORBIDDEN_403,
message: 'Updating the file of an existing video is not allowed on this instance'
})
return false
}
if (!await doesVideoExist(req.params.id, res)) return false
const user = res.locals.oauth.token.User
const video = res.locals.videoAll
if (!checkUserCanManageVideo(user, video, UserRight.UPDATE_ANY_VIDEO, res)) return false
if (!checkVideoFileCanBeEdited(video, res)) return false
return true
}

View file

@ -11,8 +11,9 @@ import { cleanUpReqFiles } from '@server/helpers/express-utils'
import { CONFIG } from '@server/initializers/config'
import { approximateIntroOutroAdditionalSize, getTaskFileFromReq } from '@server/lib/video-studio'
import { isAudioFile } from '@shared/ffmpeg'
import { HttpStatusCode, UserRight, VideoState, VideoStudioCreateEdition, VideoStudioTask } from '@shared/models'
import { HttpStatusCode, UserRight, VideoStudioCreateEdition, VideoStudioTask } from '@shared/models'
import { areValidationErrors, checkUserCanManageVideo, checkUserQuota, doesVideoExist } from '../shared'
import { checkVideoFileCanBeEdited } from './shared'
const videoStudioAddEditionValidator = [
param('videoId')
@ -66,14 +67,7 @@ const videoStudioAddEditionValidator = [
if (!await doesVideoExist(req.params.videoId, res)) return cleanUpReqFiles(req)
const video = res.locals.videoAll
if (video.state === VideoState.TO_TRANSCODE || video.state === VideoState.TO_EDIT) {
res.fail({
status: HttpStatusCode.CONFLICT_409,
message: 'Cannot edit video that is already waiting for transcoding/edition'
})
return cleanUpReqFiles(req)
}
if (!checkVideoFileCanBeEdited(video, res)) return cleanUpReqFiles(req)
const user = res.locals.oauth.token.User
if (!checkUserCanManageVideo(user, video, UserRight.UPDATE_ANY_VIDEO, res)) return cleanUpReqFiles(req)

View file

@ -2,13 +2,12 @@ import express from 'express'
import { body, header, param, query, ValidationChain } from 'express-validator'
import { isTestInstance } from '@server/helpers/core-utils'
import { getResumableUploadPath } from '@server/helpers/upload'
import { uploadx } from '@server/lib/uploadx'
import { Redis } from '@server/lib/redis'
import { uploadx } from '@server/lib/uploadx'
import { getServerActor } from '@server/models/application/application'
import { ExpressPromiseHandler } from '@server/types/express-handler'
import { MUserAccountId, MVideoFullLight } from '@server/types/models'
import { arrayify, getAllPrivacies } from '@shared/core-utils'
import { getVideoStreamDuration } from '@shared/ffmpeg'
import { HttpStatusCode, ServerErrorCode, UserRight, VideoInclude, VideoState } from '@shared/models'
import {
exists,
@ -27,8 +26,6 @@ import {
isValidPasswordProtectedPrivacy,
isVideoCategoryValid,
isVideoDescriptionValid,
isVideoFileMimeTypeValid,
isVideoFileSizeValid,
isVideoFilterValid,
isVideoImageValid,
isVideoIncludeValid,
@ -44,21 +41,19 @@ import { logger } from '../../../helpers/logger'
import { getVideoWithAttributes } from '../../../helpers/video'
import { CONFIG } from '../../../initializers/config'
import { CONSTRAINTS_FIELDS, OVERVIEWS } from '../../../initializers/constants'
import { isLocalVideoAccepted } from '../../../lib/moderation'
import { Hooks } from '../../../lib/plugins/hooks'
import { VideoModel } from '../../../models/video/video'
import {
areValidationErrors,
checkCanAccessVideoStaticFiles,
checkCanSeeVideo,
checkUserCanManageVideo,
checkUserQuota,
doesVideoChannelOfAccountExist,
doesVideoExist,
doesVideoFileOfVideoExist,
isValidVideoIdParam,
isValidVideoPasswordHeader
} from '../shared'
import { addDurationToVideoFileIfNeeded, commonVideoFileChecks, isVideoFileAccepted } from './shared'
const videosAddLegacyValidator = getCommonVideoEditAttributes().concat([
body('videofile')
@ -83,26 +78,15 @@ const videosAddLegacyValidator = getCommonVideoEditAttributes().concat([
const videoFile: express.VideoUploadFile = req.files['videofile'][0]
const user = res.locals.oauth.token.User
if (!await commonVideoChecksPass({ req, res, user, videoFileSize: videoFile.size, files: req.files })) {
if (
!await commonVideoChecksPass({ req, res, user, videoFileSize: videoFile.size, files: req.files }) ||
!isValidPasswordProtectedPrivacy(req, res) ||
!await addDurationToVideoFileIfNeeded({ videoFile, res, middlewareName: 'videosAddvideosAddLegacyValidatorResumableValidator' }) ||
!await isVideoFileAccepted({ req, res, videoFile, hook: 'filter:api.video.upload.accept.result' })
) {
return cleanUpReqFiles(req)
}
if (!isValidPasswordProtectedPrivacy(req, res)) return cleanUpReqFiles(req)
try {
if (!videoFile.duration) await addDurationToVideo(videoFile)
} catch (err) {
logger.error('Invalid input file in videosAddLegacyValidator.', { err })
res.fail({
status: HttpStatusCode.UNPROCESSABLE_ENTITY_422,
message: 'Video file unreadable.'
})
return cleanUpReqFiles(req)
}
if (!await isVideoAccepted(req, res, videoFile)) return cleanUpReqFiles(req)
return next()
}
])
@ -146,22 +130,10 @@ const videosAddResumableValidator = [
await Redis.Instance.setUploadSession(uploadId)
if (!await doesVideoChannelOfAccountExist(file.metadata.channelId, user, res)) return cleanup()
if (!await addDurationToVideoFileIfNeeded({ videoFile: file, res, middlewareName: 'videosAddResumableValidator' })) return cleanup()
if (!await isVideoFileAccepted({ req, res, videoFile: file, hook: 'filter:api.video.upload.accept.result' })) return cleanup()
try {
if (!file.duration) await addDurationToVideo(file)
} catch (err) {
logger.error('Invalid input file in videosAddResumableValidator.', { err })
res.fail({
status: HttpStatusCode.UNPROCESSABLE_ENTITY_422,
message: 'Video file unreadable.'
})
return cleanup()
}
if (!await isVideoAccepted(req, res, file)) return cleanup()
res.locals.videoFileResumable = { ...file, originalname: file.filename }
res.locals.uploadVideoFileResumable = { ...file, originalname: file.filename }
return next()
}
@ -604,76 +576,20 @@ function areErrorsInScheduleUpdate (req: express.Request, res: express.Response)
return false
}
async function commonVideoChecksPass (parameters: {
async function commonVideoChecksPass (options: {
req: express.Request
res: express.Response
user: MUserAccountId
videoFileSize: number
files: express.UploadFilesForCheck
}): Promise<boolean> {
const { req, res, user, videoFileSize, files } = parameters
const { req, res, user } = options
if (areErrorsInScheduleUpdate(req, res)) return false
if (!await doesVideoChannelOfAccountExist(req.body.channelId, user, res)) return false
if (!isVideoFileMimeTypeValid(files)) {
res.fail({
status: HttpStatusCode.UNSUPPORTED_MEDIA_TYPE_415,
message: 'This file is not supported. Please, make sure it is of the following type: ' +
CONSTRAINTS_FIELDS.VIDEOS.EXTNAME.join(', ')
})
return false
}
if (!isVideoFileSizeValid(videoFileSize.toString())) {
res.fail({
status: HttpStatusCode.PAYLOAD_TOO_LARGE_413,
message: 'This file is too large. It exceeds the maximum file size authorized.',
type: ServerErrorCode.MAX_FILE_SIZE_REACHED
})
return false
}
if (await checkUserQuota(user, videoFileSize, res) === false) return false
if (!await commonVideoFileChecks(options)) return false
return true
}
export async function isVideoAccepted (
req: express.Request,
res: express.Response,
videoFile: express.VideoUploadFile
) {
// Check we accept this video
const acceptParameters = {
videoBody: req.body,
videoFile,
user: res.locals.oauth.token.User
}
const acceptedResult = await Hooks.wrapFun(
isLocalVideoAccepted,
acceptParameters,
'filter:api.video.upload.accept.result'
)
if (!acceptedResult || acceptedResult.accepted !== true) {
logger.info('Refused local video.', { acceptedResult, acceptParameters })
res.fail({
status: HttpStatusCode.FORBIDDEN_403,
message: acceptedResult.errorMessage || 'Refused local video'
})
return false
}
return true
}
async function addDurationToVideo (videoFile: { path: string, duration?: number }) {
const duration = await getVideoStreamDuration(videoFile.path)
// FFmpeg may not be able to guess video duration
// For example with m2v files: https://trac.ffmpeg.org/ticket/9726#comment:2
if (isNaN(duration)) videoFile.duration = 0
else videoFile.duration = duration
}

View file

@ -76,6 +76,8 @@ export function videoModelToActivityPubObject (video: MVideoAP): VideoObject {
updated: video.updatedAt.toISOString(),
uploadDate: video.inputFileUpdatedAt?.toISOString(),
tag: buildTags(video),
mediaType: 'text/markdown',

View file

@ -149,6 +149,7 @@ export function videoModelToFormattedDetailsJSON (video: MVideoFormattableDetail
commentsEnabled: video.commentsEnabled,
downloadEnabled: video.downloadEnabled,
waitTranscoding: video.waitTranscoding,
inputFileUpdatedAt: video.inputFileUpdatedAt,
state: {
id: video.state,
label: getStateLabel(video.state)

View file

@ -263,6 +263,7 @@ export class VideoTableAttributes {
'state',
'publishedAt',
'originallyPublishedAt',
'inputFileUpdatedAt',
'channelId',
'createdAt',
'updatedAt',

View file

@ -1,27 +1,18 @@
import { Op } from 'sequelize'
import {
AllowNull,
BelongsTo,
Column,
CreatedAt,
ForeignKey,
Model,
Table,
UpdatedAt
} from 'sequelize-typescript'
import { Transaction } from 'sequelize'
import { AllowNull, BelongsTo, Column, CreatedAt, ForeignKey, Model, Table, UpdatedAt } from 'sequelize-typescript'
import { VideoSource } from '@shared/models/videos/video-source'
import { AttributesOnly } from '@shared/typescript-utils'
import { getSort } from '../shared'
import { VideoModel } from './video'
@Table({
tableName: 'videoSource',
indexes: [
{
fields: [ 'videoId' ],
where: {
videoId: {
[Op.ne]: null
}
}
fields: [ 'videoId' ]
},
{
fields: [ { name: 'createdAt', order: 'DESC' } ]
}
]
})
@ -40,16 +31,26 @@ export class VideoSourceModel extends Model<Partial<AttributesOnly<VideoSourceMo
@Column
videoId: number
@BelongsTo(() => VideoModel)
@BelongsTo(() => VideoModel, {
foreignKey: {
allowNull: false
},
onDelete: 'cascade'
})
Video: VideoModel
static loadByVideoId (videoId) {
return VideoSourceModel.findOne({ where: { videoId } })
static loadLatest (videoId: number, transaction?: Transaction) {
return VideoSourceModel.findOne({
where: { videoId },
order: getSort('-createdAt'),
transaction
})
}
toFormattedJSON () {
toFormattedJSON (): VideoSource {
return {
filename: this.filename
filename: this.filename,
createdAt: this.createdAt.toISOString()
}
}
}

View file

@ -546,6 +546,12 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
@Column
state: VideoState
// We already have the information in videoSource table for local videos, but we prefer to normalize it for performance
// And also to store the info from remote instances
@AllowNull(true)
@Column
inputFileUpdatedAt: Date
@CreatedAt
createdAt: Date
@ -610,7 +616,7 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
@HasOne(() => VideoSourceModel, {
foreignKey: {
name: 'videoId',
allowNull: true
allowNull: false
},
onDelete: 'CASCADE'
})

View file

@ -170,6 +170,11 @@ describe('Test config API validators', function () {
enabled: true
}
},
videoFile: {
update: {
enabled: true
}
},
import: {
videos: {
concurrency: 1,

View file

@ -1,5 +1,12 @@
import { HttpStatusCode } from '@shared/models'
import { cleanupTests, createSingleServer, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands'
import {
cleanupTests,
createSingleServer,
PeerTubeServer,
setAccessTokensToServers,
setDefaultVideoChannel,
waitJobs
} from '@shared/server-commands'
describe('Test video sources API validator', function () {
let server: PeerTubeServer = null
@ -7,35 +14,138 @@ describe('Test video sources API validator', function () {
let userToken: string
before(async function () {
this.timeout(30000)
this.timeout(120000)
server = await createSingleServer(1)
await setAccessTokensToServers([ server ])
await setDefaultVideoChannel([ server ])
const created = await server.videos.quickUpload({ name: 'video' })
uuid = created.uuid
userToken = await server.users.generateUserAndToken('user')
userToken = await server.users.generateUserAndToken('user1')
})
it('Should fail without a valid uuid', async function () {
await server.videos.getSource({ id: '4da6fde3-88f7-4d16-b119-108df563d0b0', expectedStatus: HttpStatusCode.NOT_FOUND_404 })
describe('When getting latest source', function () {
before(async function () {
const created = await server.videos.quickUpload({ name: 'video' })
uuid = created.uuid
})
it('Should fail without a valid uuid', async function () {
await server.videos.getSource({ id: '4da6fde3-88f7-4d16-b119-108df563d0b0', expectedStatus: HttpStatusCode.NOT_FOUND_404 })
})
it('Should receive 404 when passing a non existing video id', async function () {
await server.videos.getSource({ id: '4da6fde3-88f7-4d16-b119-108df5630b06', expectedStatus: HttpStatusCode.NOT_FOUND_404 })
})
it('Should not get the source as unauthenticated', async function () {
await server.videos.getSource({ id: uuid, expectedStatus: HttpStatusCode.UNAUTHORIZED_401, token: null })
})
it('Should not get the source with another user', async function () {
await server.videos.getSource({ id: uuid, expectedStatus: HttpStatusCode.FORBIDDEN_403, token: userToken })
})
it('Should succeed with the correct parameters get the source as another user', async function () {
await server.videos.getSource({ id: uuid })
})
})
it('Should receive 404 when passing a non existing video id', async function () {
await server.videos.getSource({ id: '4da6fde3-88f7-4d16-b119-108df5630b06', expectedStatus: HttpStatusCode.NOT_FOUND_404 })
})
describe('When updating source video file', function () {
let userAccessToken: string
let userId: number
it('Should not get the source as unauthenticated', async function () {
await server.videos.getSource({ id: uuid, expectedStatus: HttpStatusCode.UNAUTHORIZED_401, token: null })
})
let videoId: string
let userVideoId: string
it('Should not get the source with another user', async function () {
await server.videos.getSource({ id: uuid, expectedStatus: HttpStatusCode.FORBIDDEN_403, token: userToken })
})
before(async function () {
const res = await server.users.generate('user2')
userAccessToken = res.token
userId = res.userId
it('Should succeed with the correct parameters get the source as another user', async function () {
await server.videos.getSource({ id: uuid })
const { uuid } = await server.videos.quickUpload({ name: 'video' })
videoId = uuid
await waitJobs([ server ])
})
it('Should fail if not enabled on the instance', async function () {
await server.config.disableFileUpdate()
await server.videos.replaceSourceFile({ videoId, fixture: 'video_short.mp4', expectedStatus: HttpStatusCode.FORBIDDEN_403 })
})
it('Should fail on an unknown video', async function () {
await server.config.enableFileUpdate()
await server.videos.replaceSourceFile({ videoId: 404, fixture: 'video_short.mp4', expectedStatus: HttpStatusCode.NOT_FOUND_404 })
})
it('Should fail with an invalid video', async function () {
await server.config.enableLive({ allowReplay: false })
const { video } = await server.live.quickCreate({ saveReplay: false, permanentLive: true })
await server.videos.replaceSourceFile({
videoId: video.uuid,
fixture: 'video_short.mp4',
expectedStatus: HttpStatusCode.BAD_REQUEST_400
})
})
it('Should fail without token', async function () {
await server.videos.replaceSourceFile({
token: null,
videoId,
fixture: 'video_short.mp4',
expectedStatus: HttpStatusCode.UNAUTHORIZED_401
})
})
it('Should fail with another user', async function () {
await server.videos.replaceSourceFile({
token: userAccessToken,
videoId,
fixture: 'video_short.mp4',
expectedStatus: HttpStatusCode.FORBIDDEN_403
})
})
it('Should fail with an incorrect input file', async function () {
await server.videos.replaceSourceFile({
fixture: 'video_short_fake.webm',
videoId,
expectedStatus: HttpStatusCode.UNPROCESSABLE_ENTITY_422
})
await server.videos.replaceSourceFile({
fixture: 'video_short.mkv',
videoId,
expectedStatus: HttpStatusCode.UNSUPPORTED_MEDIA_TYPE_415
})
})
it('Should fail if quota is exceeded', async function () {
this.timeout(60000)
const { uuid } = await server.videos.quickUpload({ name: 'user video' })
userVideoId = uuid
await waitJobs([ server ])
await server.users.update({ userId, videoQuota: 1 })
await server.videos.replaceSourceFile({
token: userAccessToken,
videoId: uuid,
fixture: 'video_short.mp4',
expectedStatus: HttpStatusCode.FORBIDDEN_403
})
})
it('Should succeed with the correct params', async function () {
this.timeout(60000)
await server.users.update({ userId, videoQuota: 1000 * 1000 * 1000 })
await server.videos.replaceSourceFile({ videoId: userVideoId, fixture: 'video_short.mp4' })
})
})
after(async function () {

View file

@ -105,6 +105,8 @@ function checkInitialConfig (server: PeerTubeServer, data: CustomConfig) {
expect(data.videoStudio.enabled).to.be.false
expect(data.videoStudio.remoteRunners.enabled).to.be.false
expect(data.videoFile.update.enabled).to.be.false
expect(data.import.videos.concurrency).to.equal(2)
expect(data.import.videos.http.enabled).to.be.true
expect(data.import.videos.torrent.enabled).to.be.true
@ -216,6 +218,8 @@ function checkUpdatedConfig (data: CustomConfig) {
expect(data.videoStudio.enabled).to.be.true
expect(data.videoStudio.remoteRunners.enabled).to.be.true
expect(data.videoFile.update.enabled).to.be.true
expect(data.import.videos.concurrency).to.equal(4)
expect(data.import.videos.http.enabled).to.be.false
expect(data.import.videos.torrent.enabled).to.be.false
@ -386,6 +390,11 @@ const newCustomConfig: CustomConfig = {
enabled: true
}
},
videoFile: {
update: {
enabled: true
}
},
import: {
videos: {
concurrency: 4,

View file

@ -13,11 +13,11 @@ import './video-imports'
import './video-nsfw'
import './video-playlists'
import './video-playlist-thumbnails'
import './video-source'
import './video-privacy'
import './video-schedule-update'
import './videos-common-filters'
import './videos-history'
import './videos-overview'
import './video-source'
import './video-static-file-privacy'
import './video-storyboard'

View file

@ -11,6 +11,7 @@ import { cleanupTests, createSingleServer, PeerTubeServer, setAccessTokensToServ
// Most classic resumable upload tests are done in other test suites
describe('Test resumable upload', function () {
const path = '/api/v1/videos/upload-resumable'
const defaultFixture = 'video_short.mp4'
let server: PeerTubeServer
let rootId: number
@ -44,7 +45,7 @@ describe('Test resumable upload', function () {
const mimetype = 'video/mp4'
const res = await server.videos.prepareResumableUpload({ token, attributes, size, mimetype, originalName, lastModified })
const res = await server.videos.prepareResumableUpload({ path, token, attributes, size, mimetype, originalName, lastModified })
return res.header['location'].split('?')[1]
}
@ -66,6 +67,7 @@ describe('Test resumable upload', function () {
return server.videos.sendResumableChunks({
token,
path,
pathUploadId,
videoFilePath: absoluteFilePath,
size,
@ -125,7 +127,7 @@ describe('Test resumable upload', function () {
it('Should correctly delete files after an upload', async function () {
const uploadId = await prepareUpload()
await sendChunks({ pathUploadId: uploadId })
await server.videos.endResumableUpload({ pathUploadId: uploadId })
await server.videos.endResumableUpload({ path, pathUploadId: uploadId })
expect(await countResumableUploads()).to.equal(0)
})
@ -251,7 +253,7 @@ describe('Test resumable upload', function () {
const uploadId1 = await prepareUpload({ originalName, lastModified, token: server.accessToken })
await sendChunks({ pathUploadId: uploadId1 })
await server.videos.endResumableUpload({ pathUploadId: uploadId1 })
await server.videos.endResumableUpload({ path, pathUploadId: uploadId1 })
const uploadId2 = await prepareUpload({ originalName, lastModified, token: server.accessToken })
expect(uploadId1).to.equal(uploadId2)

View file

@ -1,36 +1,447 @@
import { expect } from 'chai'
import { cleanupTests, createSingleServer, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands'
import { expectStartWith } from '@server/tests/shared'
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import { areMockObjectStorageTestsDisabled, getAllFiles } from '@shared/core-utils'
import { HttpStatusCode } from '@shared/models'
import {
cleanupTests,
createMultipleServers,
doubleFollow,
makeGetRequest,
makeRawRequest,
ObjectStorageCommand,
PeerTubeServer,
setAccessTokensToServers,
setDefaultAccountAvatar,
setDefaultVideoChannel,
waitJobs
} from '@shared/server-commands'
describe('Test video source', () => {
let server: PeerTubeServer = null
const fixture = 'video_short.webm'
describe('Test a video file replacement', function () {
let servers: PeerTubeServer[] = []
let replaceDate: Date
let userToken: string
let uuid: string
before(async function () {
this.timeout(30000)
this.timeout(50000)
server = await createSingleServer(1)
await setAccessTokensToServers([ server ])
servers = await createMultipleServers(2)
// Get the access tokens
await setAccessTokensToServers(servers)
await setDefaultVideoChannel(servers)
await setDefaultAccountAvatar(servers)
await servers[0].config.enableFileUpdate()
userToken = await servers[0].users.generateUserAndToken('user1')
// Server 1 and server 2 follow each other
await doubleFollow(servers[0], servers[1])
})
it('Should get the source filename with legacy upload', async function () {
this.timeout(30000)
describe('Getting latest video source', () => {
const fixture = 'video_short.webm'
const uuids: string[] = []
const { uuid } = await server.videos.upload({ attributes: { name: 'my video', fixture }, mode: 'legacy' })
it('Should get the source filename with legacy upload', async function () {
this.timeout(30000)
const source = await server.videos.getSource({ id: uuid })
expect(source.filename).to.equal(fixture)
const { uuid } = await servers[0].videos.upload({ attributes: { name: 'my video', fixture }, mode: 'legacy' })
uuids.push(uuid)
const source = await servers[0].videos.getSource({ id: uuid })
expect(source.filename).to.equal(fixture)
})
it('Should get the source filename with resumable upload', async function () {
this.timeout(30000)
const { uuid } = await servers[0].videos.upload({ attributes: { name: 'my video', fixture }, mode: 'resumable' })
uuids.push(uuid)
const source = await servers[0].videos.getSource({ id: uuid })
expect(source.filename).to.equal(fixture)
})
after(async function () {
this.timeout(60000)
for (const uuid of uuids) {
await servers[0].videos.remove({ id: uuid })
}
await waitJobs(servers)
})
})
it('Should get the source filename with resumable upload', async function () {
this.timeout(30000)
describe('Updating video source', function () {
const { uuid } = await server.videos.upload({ attributes: { name: 'my video', fixture }, mode: 'resumable' })
describe('Filesystem', function () {
const source = await server.videos.getSource({ id: uuid })
expect(source.filename).to.equal(fixture)
it('Should replace a video file with transcoding disabled', async function () {
this.timeout(120000)
await servers[0].config.disableTranscoding()
const { uuid } = await servers[0].videos.quickUpload({ name: 'fs without transcoding', fixture: 'video_short_720p.mp4' })
await waitJobs(servers)
for (const server of servers) {
const video = await server.videos.get({ id: uuid })
const files = getAllFiles(video)
expect(files).to.have.lengthOf(1)
expect(files[0].resolution.id).to.equal(720)
}
await servers[0].videos.replaceSourceFile({ videoId: uuid, fixture: 'video_short_360p.mp4' })
await waitJobs(servers)
for (const server of servers) {
const video = await server.videos.get({ id: uuid })
const files = getAllFiles(video)
expect(files).to.have.lengthOf(1)
expect(files[0].resolution.id).to.equal(360)
}
})
it('Should replace a video file with transcoding enabled', async function () {
this.timeout(120000)
const previousPaths: string[] = []
await servers[0].config.enableTranscoding(true, true, true)
const { uuid: videoUUID } = await servers[0].videos.quickUpload({ name: 'fs with transcoding', fixture: 'video_short_720p.mp4' })
uuid = videoUUID
await waitJobs(servers)
for (const server of servers) {
const video = await server.videos.get({ id: uuid })
expect(video.inputFileUpdatedAt).to.be.null
const files = getAllFiles(video)
expect(files).to.have.lengthOf(6 * 2)
// Grab old paths to ensure we'll regenerate
previousPaths.push(video.previewPath)
previousPaths.push(video.thumbnailPath)
for (const file of files) {
previousPaths.push(file.fileUrl)
previousPaths.push(file.torrentUrl)
previousPaths.push(file.metadataUrl)
const metadata = await server.videos.getFileMetadata({ url: file.metadataUrl })
previousPaths.push(JSON.stringify(metadata))
}
const { storyboards } = await server.storyboard.list({ id: uuid })
for (const s of storyboards) {
previousPaths.push(s.storyboardPath)
}
}
replaceDate = new Date()
await servers[0].videos.replaceSourceFile({ videoId: uuid, fixture: 'video_short_360p.mp4' })
await waitJobs(servers)
for (const server of servers) {
const video = await server.videos.get({ id: uuid })
expect(video.inputFileUpdatedAt).to.not.be.null
expect(new Date(video.inputFileUpdatedAt)).to.be.above(replaceDate)
const files = getAllFiles(video)
expect(files).to.have.lengthOf(4 * 2)
expect(previousPaths).to.not.include(video.previewPath)
expect(previousPaths).to.not.include(video.thumbnailPath)
await makeGetRequest({ url: server.url, path: video.previewPath, expectedStatus: HttpStatusCode.OK_200 })
await makeGetRequest({ url: server.url, path: video.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 })
for (const file of files) {
expect(previousPaths).to.not.include(file.fileUrl)
expect(previousPaths).to.not.include(file.torrentUrl)
expect(previousPaths).to.not.include(file.metadataUrl)
await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
await makeRawRequest({ url: file.torrentUrl, expectedStatus: HttpStatusCode.OK_200 })
const metadata = await server.videos.getFileMetadata({ url: file.metadataUrl })
expect(previousPaths).to.not.include(JSON.stringify(metadata))
}
const { storyboards } = await server.storyboard.list({ id: uuid })
for (const s of storyboards) {
expect(previousPaths).to.not.include(s.storyboardPath)
await makeGetRequest({ url: server.url, path: s.storyboardPath, expectedStatus: HttpStatusCode.OK_200 })
}
}
await servers[0].config.enableMinimumTranscoding()
})
it('Should have cleaned up old files', async function () {
{
const count = await servers[0].servers.countFiles('storyboards')
expect(count).to.equal(2)
}
{
const count = await servers[0].servers.countFiles('web-videos')
expect(count).to.equal(5 + 1) // +1 for private directory
}
{
const count = await servers[0].servers.countFiles('streaming-playlists/hls')
expect(count).to.equal(1 + 1) // +1 for private directory
}
{
const count = await servers[0].servers.countFiles('torrents')
expect(count).to.equal(9)
}
})
it('Should have the correct source input', async function () {
const source = await servers[0].videos.getSource({ id: uuid })
expect(source.filename).to.equal('video_short_360p.mp4')
expect(new Date(source.createdAt)).to.be.above(replaceDate)
})
it('Should not have regenerated miniatures that were previously uploaded', async function () {
this.timeout(120000)
const { uuid } = await servers[0].videos.upload({
attributes: {
name: 'custom miniatures',
thumbnailfile: 'custom-thumbnail.jpg',
previewfile: 'custom-preview.jpg'
}
})
await waitJobs(servers)
const previousPaths: string[] = []
for (const server of servers) {
const video = await server.videos.get({ id: uuid })
previousPaths.push(video.previewPath)
previousPaths.push(video.thumbnailPath)
await makeGetRequest({ url: server.url, path: video.previewPath, expectedStatus: HttpStatusCode.OK_200 })
await makeGetRequest({ url: server.url, path: video.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 })
}
await servers[0].videos.replaceSourceFile({ videoId: uuid, fixture: 'video_short_360p.mp4' })
await waitJobs(servers)
for (const server of servers) {
const video = await server.videos.get({ id: uuid })
expect(previousPaths).to.include(video.previewPath)
expect(previousPaths).to.include(video.thumbnailPath)
await makeGetRequest({ url: server.url, path: video.previewPath, expectedStatus: HttpStatusCode.OK_200 })
await makeGetRequest({ url: server.url, path: video.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 })
}
})
})
describe('Autoblacklist', function () {
function updateAutoBlacklist (enabled: boolean) {
return servers[0].config.updateExistingSubConfig({
newConfig: {
autoBlacklist: {
videos: {
ofUsers: {
enabled
}
}
}
}
})
}
async function expectBlacklist (uuid: string, value: boolean) {
const video = await servers[0].videos.getWithToken({ id: uuid })
expect(video.blacklisted).to.equal(value)
}
before(async function () {
await updateAutoBlacklist(true)
})
it('Should auto blacklist an unblacklisted video after file replacement', async function () {
this.timeout(120000)
const { uuid } = await servers[0].videos.quickUpload({ token: userToken, name: 'user video' })
await waitJobs(servers)
await expectBlacklist(uuid, true)
await servers[0].blacklist.remove({ videoId: uuid })
await expectBlacklist(uuid, false)
await servers[0].videos.replaceSourceFile({ videoId: uuid, token: userToken, fixture: 'video_short_360p.mp4' })
await waitJobs(servers)
await expectBlacklist(uuid, true)
})
it('Should auto blacklist an already blacklisted video after file replacement', async function () {
this.timeout(120000)
const { uuid } = await servers[0].videos.quickUpload({ token: userToken, name: 'user video' })
await waitJobs(servers)
await expectBlacklist(uuid, true)
await servers[0].videos.replaceSourceFile({ videoId: uuid, token: userToken, fixture: 'video_short_360p.mp4' })
await waitJobs(servers)
await expectBlacklist(uuid, true)
})
it('Should not auto blacklist if auto blacklist has been disabled between the upload and the replacement', async function () {
this.timeout(120000)
const { uuid } = await servers[0].videos.quickUpload({ token: userToken, name: 'user video' })
await waitJobs(servers)
await expectBlacklist(uuid, true)
await servers[0].blacklist.remove({ videoId: uuid })
await expectBlacklist(uuid, false)
await updateAutoBlacklist(false)
await servers[0].videos.replaceSourceFile({ videoId: uuid, token: userToken, fixture: 'video_short1.webm' })
await waitJobs(servers)
await expectBlacklist(uuid, false)
})
})
describe('With object storage enabled', function () {
if (areMockObjectStorageTestsDisabled()) return
const objectStorage = new ObjectStorageCommand()
before(async function () {
this.timeout(120000)
const configOverride = objectStorage.getDefaultMockConfig()
await objectStorage.prepareDefaultMockBuckets()
await servers[0].kill()
await servers[0].run(configOverride)
})
it('Should replace a video file with transcoding disabled', async function () {
this.timeout(120000)
await servers[0].config.disableTranscoding()
const { uuid } = await servers[0].videos.quickUpload({
name: 'object storage without transcoding',
fixture: 'video_short_720p.mp4'
})
await waitJobs(servers)
for (const server of servers) {
const video = await server.videos.get({ id: uuid })
const files = getAllFiles(video)
expect(files).to.have.lengthOf(1)
expect(files[0].resolution.id).to.equal(720)
expectStartWith(files[0].fileUrl, objectStorage.getMockWebVideosBaseUrl())
}
await servers[0].videos.replaceSourceFile({ videoId: uuid, fixture: 'video_short_360p.mp4' })
await waitJobs(servers)
for (const server of servers) {
const video = await server.videos.get({ id: uuid })
const files = getAllFiles(video)
expect(files).to.have.lengthOf(1)
expect(files[0].resolution.id).to.equal(360)
expectStartWith(files[0].fileUrl, objectStorage.getMockWebVideosBaseUrl())
}
})
it('Should replace a video file with transcoding enabled', async function () {
this.timeout(120000)
const previousPaths: string[] = []
await servers[0].config.enableTranscoding(true, true, true)
const { uuid: videoUUID } = await servers[0].videos.quickUpload({
name: 'object storage with transcoding',
fixture: 'video_short_360p.mp4'
})
uuid = videoUUID
await waitJobs(servers)
for (const server of servers) {
const video = await server.videos.get({ id: uuid })
const files = getAllFiles(video)
expect(files).to.have.lengthOf(4 * 2)
for (const file of files) {
previousPaths.push(file.fileUrl)
}
for (const file of video.files) {
expectStartWith(file.fileUrl, objectStorage.getMockWebVideosBaseUrl())
}
for (const file of video.streamingPlaylists[0].files) {
expectStartWith(file.fileUrl, objectStorage.getMockPlaylistBaseUrl())
}
}
await servers[0].videos.replaceSourceFile({ videoId: uuid, fixture: 'video_short_240p.mp4' })
await waitJobs(servers)
for (const server of servers) {
const video = await server.videos.get({ id: uuid })
const files = getAllFiles(video)
expect(files).to.have.lengthOf(3 * 2)
for (const file of files) {
expect(previousPaths).to.not.include(file.fileUrl)
}
for (const file of video.files) {
expectStartWith(file.fileUrl, objectStorage.getMockWebVideosBaseUrl())
}
for (const file of video.streamingPlaylists[0].files) {
expectStartWith(file.fileUrl, objectStorage.getMockPlaylistBaseUrl())
}
}
})
})
})
after(async function () {
await cleanupTests([ server ])
await cleanupTests(servers)
})
})

View file

@ -19,12 +19,6 @@ import {
waitJobs
} from '@shared/server-commands'
async function countFiles (server: PeerTubeServer, directory: string) {
const files = await readdir(server.servers.buildDirectory(directory))
return files.length
}
async function assertNotExists (server: PeerTubeServer, directory: string, substring: string) {
const files = await readdir(server.servers.buildDirectory(directory))
@ -35,28 +29,28 @@ async function assertNotExists (server: PeerTubeServer, directory: string, subst
async function assertCountAreOkay (servers: PeerTubeServer[]) {
for (const server of servers) {
const videosCount = await countFiles(server, 'web-videos')
const videosCount = await server.servers.countFiles('web-videos')
expect(videosCount).to.equal(9) // 2 videos with 4 resolutions + private directory
const privateVideosCount = await countFiles(server, 'web-videos/private')
const privateVideosCount = await server.servers.countFiles('web-videos/private')
expect(privateVideosCount).to.equal(4)
const torrentsCount = await countFiles(server, 'torrents')
const torrentsCount = await server.servers.countFiles('torrents')
expect(torrentsCount).to.equal(24)
const previewsCount = await countFiles(server, 'previews')
const previewsCount = await server.servers.countFiles('previews')
expect(previewsCount).to.equal(3)
const thumbnailsCount = await countFiles(server, 'thumbnails')
const thumbnailsCount = await server.servers.countFiles('thumbnails')
expect(thumbnailsCount).to.equal(5) // 3 local videos, 1 local playlist, 2 remotes videos (lazy downloaded) and 1 remote playlist
const avatarsCount = await countFiles(server, 'avatars')
const avatarsCount = await server.servers.countFiles('avatars')
expect(avatarsCount).to.equal(4)
const hlsRootCount = await countFiles(server, join('streaming-playlists', 'hls'))
const hlsRootCount = await server.servers.countFiles(join('streaming-playlists', 'hls'))
expect(hlsRootCount).to.equal(3) // 2 videos + private directory
const hlsPrivateRootCount = await countFiles(server, join('streaming-playlists', 'hls', 'private'))
const hlsPrivateRootCount = await server.servers.countFiles(join('streaming-playlists', 'hls', 'private'))
expect(hlsPrivateRootCount).to.equal(1)
}
}

View file

@ -277,7 +277,7 @@ function checkUploadVideoParam (
) {
return mode === 'legacy'
? server.videos.buildLegacyUpload({ token, attributes, expectedStatus })
: server.videos.buildResumeUpload({ token, attributes, expectedStatus })
: server.videos.buildResumeUpload({ token, attributes, expectedStatus, path: '/api/v1/videos/upload-resumable' })
}
// serverNumber starts from 1

View file

@ -86,13 +86,15 @@ declare module 'express' {
// Our custom UploadXFile object using our custom metadata
export type CustomUploadXFile <T extends Metadata> = UploadXFile & { metadata: T }
export type EnhancedUploadXFile = CustomUploadXFile<UploadXFileMetadata> & {
export type EnhancedUploadXFile = CustomUploadXFile<Metadata> & {
duration: number
path: string
filename: string
originalname: string
}
export type UploadNewVideoUploadXFile = EnhancedUploadXFile & CustomUploadXFile<UploadXFileMetadata>
// Extends Response with added functions and potential variables passed by middlewares
interface Response {
fail: (options: {
@ -139,7 +141,8 @@ declare module 'express' {
videoFile?: MVideoFile
videoFileResumable?: EnhancedUploadXFile
uploadVideoFileResumable?: UploadNewVideoUploadXFile
updateVideoFileResumable?: EnhancedUploadXFile
videoImport?: MVideoImportDefault

View file

@ -31,9 +31,11 @@ export interface VideoObject {
downloadEnabled: boolean
waitTranscoding: boolean
state: VideoState
published: string
originallyPublishedAt: string
updated: string
uploadDate: string
mediaType: 'text/markdown'
content: string

View file

@ -64,6 +64,7 @@ export const serverFilterHookObject = {
'filter:api.video.pre-import-torrent.accept.result': true,
'filter:api.video.post-import-url.accept.result': true,
'filter:api.video.post-import-torrent.accept.result': true,
'filter:api.video.update-file.accept.result': true,
// Filter the result of the accept comment (thread or reply) functions
// If the functions return false then the user cannot post its comment
'filter:api.video-thread.create.accept.result': true,
@ -155,6 +156,9 @@ export const serverActionHookObject = {
// Fired when a local video is viewed
'action:api.video.viewed': true,
// Fired when a local video file has been replaced by a new one
'action:api.video.file-updated': true,
// Fired when a video channel is created
'action:api.video-channel.created': true,
// Fired when a video channel is updated

View file

@ -175,6 +175,12 @@ export interface CustomConfig {
}
}
videoFile: {
update: {
enabled: boolean
}
}
import: {
videos: {
concurrency: number

View file

@ -192,6 +192,12 @@ export interface ServerConfig {
}
}
videoFile: {
update: {
enabled: boolean
}
}
import: {
videos: {
http: {

View file

@ -1,3 +1,4 @@
export interface VideoSource {
filename: string
createdAt: string | Date
}

View file

@ -94,4 +94,6 @@ export interface VideoDetails extends Video {
files: VideoFile[]
streamingPlaylists: VideoStreamingPlaylist[]
inputFileUpdatedAt: string | Date
}

View file

@ -74,6 +74,28 @@ export class ConfigCommand extends AbstractCommand {
// ---------------------------------------------------------------------------
disableFileUpdate () {
return this.setFileUpdateEnabled(false)
}
enableFileUpdate () {
return this.setFileUpdateEnabled(true)
}
private setFileUpdateEnabled (enabled: boolean) {
return this.updateExistingSubConfig({
newConfig: {
videoFile: {
update: {
enabled
}
}
}
})
}
// ---------------------------------------------------------------------------
enableChannelSync () {
return this.setChannelSyncEnabled(true)
}
@ -466,6 +488,11 @@ export class ConfigCommand extends AbstractCommand {
enabled: false
}
},
videoFile: {
update: {
enabled: false
}
},
import: {
videos: {
concurrency: 3,

View file

@ -1,5 +1,5 @@
import { exec } from 'child_process'
import { copy, ensureDir, readFile, remove } from 'fs-extra'
import { copy, ensureDir, readFile, readdir, remove } from 'fs-extra'
import { basename, join } from 'path'
import { isGithubCI, root, wait } from '@shared/core-utils'
import { getFileSize } from '@shared/extra-utils'
@ -77,6 +77,12 @@ export class ServersCommand extends AbstractCommand {
return join(root(), 'test' + this.server.internalServerNumber, directory)
}
async countFiles (directory: string) {
const files = await readdir(this.buildDirectory(directory))
return files.length
}
buildWebVideoFilePath (fileUrl: string) {
return this.buildDirectory(join('web-videos', basename(fileUrl)))
}

View file

@ -32,6 +32,7 @@ export type VideoEdit = Partial<Omit<VideoCreate, 'thumbnailfile' | 'previewfile
}
export class VideosCommand extends AbstractCommand {
getCategories (options: OverrideCommandOptions = {}) {
const path = '/api/v1/videos/categories'
@ -424,7 +425,7 @@ export class VideosCommand extends AbstractCommand {
const created = mode === 'legacy'
? await this.buildLegacyUpload({ ...options, attributes })
: await this.buildResumeUpload({ ...options, attributes })
: await this.buildResumeUpload({ ...options, path: '/api/v1/videos/upload-resumable', attributes })
// Wait torrent generation
const expectedStatus = this.buildExpectedStatus({ ...options, defaultExpectedStatus: HttpStatusCode.OK_200 })
@ -458,9 +459,10 @@ export class VideosCommand extends AbstractCommand {
}
async buildResumeUpload (options: OverrideCommandOptions & {
attributes: VideoEdit
path: string
attributes: { fixture?: string } & { [id: string]: any }
}): Promise<VideoCreateResult> {
const { attributes, expectedStatus } = options
const { path, attributes, expectedStatus } = options
let size = 0
let videoFilePath: string
@ -478,7 +480,15 @@ export class VideosCommand extends AbstractCommand {
}
// Do not check status automatically, we'll check it manually
const initializeSessionRes = await this.prepareResumableUpload({ ...options, expectedStatus: null, attributes, size, mimetype })
const initializeSessionRes = await this.prepareResumableUpload({
...options,
path,
expectedStatus: null,
attributes,
size,
mimetype
})
const initStatus = initializeSessionRes.status
if (videoFilePath && initStatus === HttpStatusCode.CREATED_201) {
@ -487,10 +497,23 @@ export class VideosCommand extends AbstractCommand {
const pathUploadId = locationHeader.split('?')[1]
const result = await this.sendResumableChunks({ ...options, pathUploadId, videoFilePath, size })
const result = await this.sendResumableChunks({
...options,
path,
pathUploadId,
videoFilePath,
size
})
if (result.statusCode === HttpStatusCode.OK_200) {
await this.endResumableUpload({ ...options, expectedStatus: HttpStatusCode.NO_CONTENT_204, pathUploadId })
await this.endResumableUpload({
...options,
expectedStatus: HttpStatusCode.NO_CONTENT_204,
path,
pathUploadId
})
}
return result.body?.video || result.body as any
@ -506,18 +529,19 @@ export class VideosCommand extends AbstractCommand {
}
async prepareResumableUpload (options: OverrideCommandOptions & {
attributes: VideoEdit
path: string
attributes: { fixture?: string } & { [id: string]: any }
size: number
mimetype: string
originalName?: string
lastModified?: number
}) {
const { attributes, originalName, lastModified, size, mimetype } = options
const { path, attributes, originalName, lastModified, size, mimetype } = options
const path = '/api/v1/videos/upload-resumable'
const attaches = this.buildUploadAttaches(omit(options.attributes, [ 'fixture' ]))
return this.postUploadRequest({
const uploadOptions = {
...options,
path,
@ -538,11 +562,16 @@ export class VideosCommand extends AbstractCommand {
implicitToken: true,
defaultExpectedStatus: null
})
}
if (Object.keys(attaches).length === 0) return this.postBodyRequest(uploadOptions)
return this.postUploadRequest(uploadOptions)
}
sendResumableChunks (options: OverrideCommandOptions & {
pathUploadId: string
path: string
videoFilePath: string
size: number
contentLength?: number
@ -550,6 +579,7 @@ export class VideosCommand extends AbstractCommand {
digestBuilder?: (chunk: any) => string
}) {
const {
path,
pathUploadId,
videoFilePath,
size,
@ -559,7 +589,6 @@ export class VideosCommand extends AbstractCommand {
expectedStatus = HttpStatusCode.OK_200
} = options
const path = '/api/v1/videos/upload-resumable'
let start = 0
const token = this.buildCommonRequestToken({ ...options, implicitToken: true })
@ -610,12 +639,13 @@ export class VideosCommand extends AbstractCommand {
}
endResumableUpload (options: OverrideCommandOptions & {
path: string
pathUploadId: string
}) {
return this.deleteRequest({
...options,
path: '/api/v1/videos/upload-resumable',
path: options.path,
rawQuery: options.pathUploadId,
implicitToken: true,
defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204
@ -657,6 +687,21 @@ export class VideosCommand extends AbstractCommand {
// ---------------------------------------------------------------------------
replaceSourceFile (options: OverrideCommandOptions & {
videoId: number | string
fixture: string
}) {
return this.buildResumeUpload({
...options,
path: '/api/v1/videos/' + options.videoId + '/source/replace-resumable',
attributes: { fixture: options.fixture },
expectedStatus: HttpStatusCode.NO_CONTENT_204
})
}
// ---------------------------------------------------------------------------
removeHLSPlaylist (options: OverrideCommandOptions & {
videoId: number | string
}) {

View file

@ -2641,22 +2641,6 @@ paths:
example: |
**[Want to help to translate this video?](https://weblate.framasoft.org/projects/what-is-peertube-video/)**\r\n\r\n**Take back the control of your videos! [#JoinPeertube](https://joinpeertube.org)**
'/api/v1/videos/{id}/source':
post:
summary: Get video source file metadata
operationId: getVideoSource
tags:
- Video
parameters:
- $ref: '#/components/parameters/idOrUUID'
responses:
'200':
description: successful operation
content:
application/json:
schema:
$ref: '#/components/schemas/VideoSource'
'/api/v1/videos/{id}/views':
post:
summary: Notify user is watching a video
@ -2871,21 +2855,8 @@ paths:
- Video
- Video Upload
parameters:
- name: X-Upload-Content-Length
in: header
schema:
type: number
example: 2469036
required: true
description: Number of bytes that will be uploaded in subsequent requests. Set this value to the size of the file you are uploading.
- name: X-Upload-Content-Type
in: header
schema:
type: string
format: mimetype
example: video/mp4
required: true
description: MIME type of the file that you are uploading. Depending on your instance settings, acceptable values might vary.
- $ref: '#/components/parameters/resumableUploadInitContentLengthHeader'
- $ref: '#/components/parameters/resumableUploadInitContentTypeHeader'
requestBody:
content:
application/json:
@ -2924,36 +2895,9 @@ paths:
- Video
- Video Upload
parameters:
- name: upload_id
in: query
required: true
description: |
Created session id to proceed with. If you didn't send chunks in the last hour, it is
not valid anymore and you need to initialize a new upload.
schema:
type: string
- name: Content-Range
in: header
schema:
type: string
example: bytes 0-262143/2469036
required: true
description: |
Specifies the bytes in the file that the request is uploading.
For example, a value of `bytes 0-262143/1000000` shows that the request is sending the first
262144 bytes (256 x 1024) in a 2,469,036 byte file.
- name: Content-Length
in: header
schema:
type: number
example: 262144
required: true
description: |
Size of the chunk that the request is sending.
Remember that larger chunks are more efficient. PeerTube's web client uses chunks varying from
1048576 bytes (~1MB) and increases or reduces size depending on connection health.
- $ref: '#/components/parameters/resumableUploadId'
- $ref: '#/components/parameters/resumableUploadChunkContentRangeHeader'
- $ref: '#/components/parameters/resumableUploadChunkContentLengthHeader'
requestBody:
content:
application/octet-stream:
@ -3009,14 +2953,7 @@ paths:
- Video
- Video Upload
parameters:
- name: upload_id
in: query
required: true
description: |
Created session id to proceed with. If you didn't send chunks in the last 12 hours, it is
not valid anymore and the upload session has already been deleted with its data ;-)
schema:
type: string
- $ref: '#/components/parameters/resumableUploadId'
- name: Content-Length
in: header
required: true
@ -3286,6 +3223,140 @@ paths:
schema:
$ref: '#/components/schemas/LiveVideoSessionResponse'
'/api/v1/videos/{id}/source':
get:
summary: Get video source file metadata
operationId: getVideoSource
tags:
- Video
parameters:
- $ref: '#/components/parameters/idOrUUID'
responses:
'200':
description: successful operation
content:
application/json:
schema:
$ref: '#/components/schemas/VideoSource'
'/api/v1/videos/{id}/source/replace-resumable':
post:
summary: Initialize the resumable replacement of a video
description: Uses [a resumable protocol](https://github.com/kukhariev/node-uploadx/blob/master/proto.md) to initialize the replacement of a video
operationId: replaceVideoSourceResumableInit
security:
- OAuth2: []
tags:
- Video
- Video Upload
parameters:
- $ref: '#/components/parameters/resumableUploadInitContentLengthHeader'
- $ref: '#/components/parameters/resumableUploadInitContentTypeHeader'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/VideoReplaceSourceRequestResumable'
responses:
'200':
description: file already exists, send a [`resume`](https://github.com/kukhariev/node-uploadx/blob/master/proto.md) request instead
'201':
description: created
headers:
Location:
schema:
type: string
format: url
Content-Length:
schema:
type: number
example: 0
'413':
x-summary: video file too large, due to quota, absolute max file size or concurrent partial upload limit
description: |
Disambiguate via `type`:
- `max_file_size_reached` for the absolute file size limit
- `quota_reached` for quota limits whether daily or global
'415':
description: video type unsupported
put:
summary: Send chunk for the resumable replacement of a video
description: Uses [a resumable protocol](https://github.com/kukhariev/node-uploadx/blob/master/proto.md) to continue, pause or resume the replacement of a video
operationId: replaceVideoSourceResumable
security:
- OAuth2: []
tags:
- Video
- Video Upload
parameters:
- $ref: '#/components/parameters/resumableUploadId'
- $ref: '#/components/parameters/resumableUploadChunkContentRangeHeader'
- $ref: '#/components/parameters/resumableUploadChunkContentLengthHeader'
requestBody:
content:
application/octet-stream:
schema:
type: string
format: binary
responses:
'204':
description: 'last chunk received: successful operation'
'308':
description: resume incomplete
headers:
Range:
schema:
type: string
example: bytes=0-262143
Content-Length:
schema:
type: number
example: 0
'403':
description: video didn't pass file replacement filter
'404':
description: replace upload not found
'409':
description: chunk doesn't match range
'422':
description: video unreadable
'429':
description: too many concurrent requests
'503':
description: upload is already being processed
headers:
'Retry-After':
schema:
type: number
example: 300
delete:
summary: Cancel the resumable replacement of a video
description: Uses [a resumable protocol](https://github.com/kukhariev/node-uploadx/blob/master/proto.md) to cancel the replacement of a video
operationId: replaceVideoSourceResumableCancel
security:
- OAuth2: []
tags:
- Video
- Video Upload
parameters:
- $ref: '#/components/parameters/resumableUploadId'
- name: Content-Length
in: header
required: true
schema:
type: number
example: 0
responses:
'204':
description: source file replacement cancelled
headers:
Content-Length:
schema:
type: number
example: 0
'404':
description: source file replacement not found
/api/v1/users/me/abuses:
get:
summary: List my abuses
@ -6640,6 +6711,58 @@ components:
required: false
schema:
type: string
resumableUploadInitContentLengthHeader:
name: X-Upload-Content-Length
in: header
schema:
type: number
example: 2469036
required: true
description: Number of bytes that will be uploaded in subsequent requests. Set this value to the size of the file you are uploading.
resumableUploadInitContentTypeHeader:
name: X-Upload-Content-Type
in: header
schema:
type: string
format: mimetype
example: video/mp4
required: true
description: MIME type of the file that you are uploading. Depending on your instance settings, acceptable values might vary.
resumableUploadChunkContentRangeHeader:
name: Content-Range
in: header
schema:
type: string
example: bytes 0-262143/2469036
required: true
description: |
Specifies the bytes in the file that the request is uploading.
For example, a value of `bytes 0-262143/1000000` shows that the request is sending the first
262144 bytes (256 x 1024) in a 2,469,036 byte file.
resumableUploadChunkContentLengthHeader:
name: Content-Length
in: header
schema:
type: number
example: 262144
required: true
description: |
Size of the chunk that the request is sending.
Remember that larger chunks are more efficient. PeerTube's web client uses chunks varying from
1048576 bytes (~1MB) and increases or reduces size depending on connection health.
resumableUploadId:
name: upload_id
in: query
required: true
description: |
Created session id to proceed with. If you didn't send chunks in the last hour, it is
not valid anymore and you need to initialize a new upload.
schema:
type: string
securitySchemes:
OAuth2:
description: |
@ -7209,6 +7332,11 @@ components:
type: boolean
downloadEnabled:
type: boolean
inputFileUpdatedAt:
type: string
format: date-time
nullable: true
description: Latest input file update. Null if the file has never been replaced since the original upload
trackerUrls:
type: array
items:
@ -7554,6 +7682,9 @@ components:
properties:
filename:
type: string
createdAt:
type: string
format: date-time
ActorImage:
properties:
path:
@ -8403,6 +8534,13 @@ components:
$ref: '#/components/schemas/Video/properties/uuid'
shortUUID:
$ref: '#/components/schemas/Video/properties/shortUUID'
VideoReplaceSourceRequestResumable:
properties:
filename:
description: Video filename including extension
type: string
format: filename
example: what_is_peertube.mp4
CommentThreadResponse:
properties:
total: