diff --git a/.gitignore b/.gitignore index 05416e8fa364..143d81e28a5c 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,4 @@ license_en.txt !.yarn/sdks !.yarn/versions chromedriver.log +shared-resources/ai-highlighter/* diff --git a/app/components-react/highlighter/BlankSlate.tsx b/app/components-react/highlighter/BlankSlate.tsx deleted file mode 100644 index 40348cbc75f0..000000000000 --- a/app/components-react/highlighter/BlankSlate.tsx +++ /dev/null @@ -1,193 +0,0 @@ -import React, { useEffect, useRef, useState } from 'react'; -import { CheckCircleOutlined, InfoCircleOutlined } from '@ant-design/icons'; -import HotkeyBinding, { getBindingString } from 'components-react/shared/HotkeyBinding'; -import { IHotkey } from 'services/hotkeys'; -import { Services } from 'components-react/service-provider'; -import { useVuex } from 'components-react/hooks'; -import { Button } from 'antd'; -import { SUPPORTED_FILE_TYPES } from 'services/highlighter/constants'; -import { SliderInput } from 'components-react/shared/inputs'; -import Form from 'components-react/shared/inputs/Form'; -import Scrollable from 'components-react/shared/Scrollable'; -import styles from '../pages/Highlighter.m.less'; -import { $t } from 'services/i18n'; -import Translate from 'components-react/shared/Translate'; - -export default function BlankSlate(p: { close: () => void }) { - const { HotkeysService, SettingsService, StreamingService } = Services; - const [hotkey, setHotkey] = useState(null); - const hotkeyRef = useRef(null); - const v = useVuex(() => ({ - settingsValues: SettingsService.views.values, - isStreaming: StreamingService.isStreaming, - })); - - const correctlyConfigured = - v.settingsValues.Output.RecRB && - v.settingsValues.General.ReplayBufferWhileStreaming && - !v.settingsValues.General.KeepReplayBufferStreamStops && - SUPPORTED_FILE_TYPES.includes(v.settingsValues.Output.RecFormat); - - function configure() { - SettingsService.actions.setSettingsPatch({ - General: { - ReplayBufferWhileStreaming: true, - KeepReplayBufferStreamStops: false, - }, - Output: { - RecRB: true, - }, - }); - - // We will only set recording format to mp4 if the user isn't already on - // a supported format. i.e. don't switch them from mov to mp4, but we will - // switch from flv to mp4. - if (!SUPPORTED_FILE_TYPES.includes(v.settingsValues.Output.RecFormat)) { - SettingsService.actions.setSettingsPatch({ Output: { RecFormat: 'mp4' } }); - } - } - - useEffect(() => { - HotkeysService.actions.return.getGeneralHotkeyByName('SAVE_REPLAY').then(hotkey => { - if (hotkey) setHotkey(hotkey); - }); - }, []); - - useEffect(() => { - if (!v.isStreaming) { - HotkeysService.actions.unregisterAll(); - - return () => { - if (hotkeyRef.current) { - // Implies a bind all - HotkeysService.actions.applyGeneralHotkey(hotkeyRef.current); - } else { - HotkeysService.actions.bindHotkeys(); - } - }; - } - }, [v.isStreaming]); - - function completedStepHeading(title: string) { - return ( -

- - {title} -

- ); - } - - function incompleteStepHeading(title: string) { - return ( -

- - {title} -

- ); - } - - function setReplayTime(time: number) { - SettingsService.actions.setSettingsPatch({ Output: { RecRBTime: time } }); - } - - return ( -
- -

{$t('Highlighter')}

-

- {$t( - 'The highlighter allows you to clip the best moments from your livestream and edit them together into an exciting highlight video you can upload directly to YouTube.', - )} -

-
-
-

{$t('Get Started')}

- {!v.isStreaming && ( -
- {correctlyConfigured - ? completedStepHeading($t('Configure the replay buffer')) - : incompleteStepHeading($t('Configure the replay buffer'))} - {correctlyConfigured ? ( -
{$t('The replay buffer is correctly configured')}
- ) : ( - - )} -
- )} - {!v.isStreaming && ( -
- {completedStepHeading($t('Adjust replay duration'))} -
- {$t('Set the duration of captured replays. You can always trim them down later.')} -
-
- `${v}s`} - /> - -
- )} - {!v.isStreaming && ( -
- {hotkey?.bindings.length - ? completedStepHeading($t('Set a hotkey to capture replays')) - : incompleteStepHeading($t('Set a hotkey to capture replays'))} - {hotkey && ( - { - const newHotkey = { ...hotkey }; - newHotkey.bindings.splice(0, 1, binding); - setHotkey(newHotkey); - hotkeyRef.current = newHotkey; - }} - /> - )} -
- )} -
- {incompleteStepHeading($t('Capture a replay'))} - {!!hotkey?.bindings.length && ( -
- -
- )} - {!hotkey?.bindings.length && ( -
- {$t('Start streaming and capture a replay. Check back here after your stream.')} -
- )} -
- {$t('Or, import a clip from your computer')} -
-
-
-
- -
- ); -} diff --git a/app/components-react/highlighter/ClipPreview.m.less b/app/components-react/highlighter/ClipPreview.m.less index 10a6aa22c253..1c3e75046124 100644 --- a/app/components-react/highlighter/ClipPreview.m.less +++ b/app/components-react/highlighter/ClipPreview.m.less @@ -87,7 +87,7 @@ .highlighter-icon { font-size: 19px; - transform: translateY(1px); + transform: translateX(-6px); } .preview-clip-bottom-bar { @@ -104,3 +104,16 @@ align-items: center; pointer-events: auto; } + +.round-tag { + padding: 4px 6px; + background-color: #00000070; + border-radius: 4px; + color: white; +} + +.flame-hypescore-wrapper { + position: absolute; + top: 7px; + right: 9px; +} diff --git a/app/components-react/highlighter/ClipPreview.tsx b/app/components-react/highlighter/ClipPreview.tsx index b38050eaded5..24a63b9bce86 100644 --- a/app/components-react/highlighter/ClipPreview.tsx +++ b/app/components-react/highlighter/ClipPreview.tsx @@ -6,6 +6,8 @@ import { BoolButtonInput } from 'components-react/shared/inputs/BoolButtonInput' import styles from './ClipPreview.m.less'; import { Button } from 'antd'; import { $t } from 'services/i18n'; +import { isAiClip } from './utils'; +import { InputEmojiSection } from './InputEmojiSection'; import { useVuex } from 'components-react/hooks'; export default function ClipPreview(props: { @@ -24,7 +26,7 @@ export default function ClipPreview(props: { const enabled = v.clip.deleted ? false : v.clip.enabled; if (!v.clip) { - return
deleted
; + return <>deleted; } function mouseMove(e: React.MouseEvent) { @@ -66,6 +68,9 @@ export default function ClipPreview(props: { )} +
+ {isAiClip(v.clip) && } +
+ {/* left */}
{formatSecondsToHMS(v.clip.duration! - (v.clip.startTrim + v.clip.endTrim) || 0)}
-
- + {/* right */} +
+
+ {isAiClip(v.clip) ? ( + + ) : ( +
+ +
+ )} +
+ {isAiClip(v.clip) && v.clip.aiInfo?.metadata?.round && ( +
{`Round: ${v.clip.aiInfo.metadata.round}`}
+ )}
@@ -120,3 +148,25 @@ export function formatSecondsToHMS(seconds: number): string { minutes !== 0 ? minutes.toString() + 'm ' : '' }${remainingSeconds !== 0 ? remainingSeconds.toString() + 's' : ''}`; } + +function FlameHypeScore({ score }: { score: number }) { + if (score === undefined) { + return <>; + } + const normalizedScore = Math.min(1, Math.max(0, score)); + const fullFlames = Math.ceil(normalizedScore * 5); + + return ( +
+ {Array.from({ length: fullFlames }).map((_, index) => ( + ðŸ”Ĩ + ))} + + {Array.from({ length: 5 - fullFlames }).map((_, index) => ( + + ðŸ”Ĩ + + ))} +
+ ); +} diff --git a/app/components-react/highlighter/ClipTrimmer.tsx b/app/components-react/highlighter/ClipTrimmer.tsx index a2b892c08505..d88a6bca706d 100644 --- a/app/components-react/highlighter/ClipTrimmer.tsx +++ b/app/components-react/highlighter/ClipTrimmer.tsx @@ -151,14 +151,17 @@ export default function ClipTrimmer(props: { clip: TClip }) { function stopDragging() { if (isDragging.current === 'start') { HighlighterService.actions.setStartTrim(props.clip.path, localStartTrim); - UsageStatisticsService.actions.recordAnalyticsEvent('Highlighter', { type: 'Trim' }); } else if (isDragging.current === 'end') { HighlighterService.actions.setEndTrim(props.clip.path, localEndTrim); - UsageStatisticsService.actions.recordAnalyticsEvent('Highlighter', { type: 'Trim' }); } isDragging.current = null; playAt(localStartTrim); + + UsageStatisticsService.actions.recordAnalyticsEvent( + HighlighterService.state.useAiHighlighter ? 'AIHighlighter' : 'Highlighter', + { type: 'Trim' }, + ); } const scrubHeight = 100; diff --git a/app/components-react/highlighter/ClipsView.tsx b/app/components-react/highlighter/ClipsView.tsx index 38082f9fbc15..870cd0fd9dfe 100644 --- a/app/components-react/highlighter/ClipsView.tsx +++ b/app/components-react/highlighter/ClipsView.tsx @@ -3,6 +3,7 @@ import * as remote from '@electron/remote'; import { Services } from 'components-react/service-provider'; import styles from './ClipsView.m.less'; import { EHighlighterView, IAiClip, IViewState, TClip } from 'services/highlighter'; +import ClipPreview, { formatSecondsToHMS } from 'components-react/highlighter/ClipPreview'; import { ReactSortable } from 'react-sortablejs'; import Scrollable from 'components-react/shared/Scrollable'; import { EditingControls } from './EditingControls'; @@ -18,7 +19,10 @@ import { Button } from 'antd'; import { SUPPORTED_FILE_TYPES } from 'services/highlighter/constants'; import { $t } from 'services/i18n'; import path from 'path'; -import ClipPreview from './ClipPreview'; +import MiniClipPreview from './MiniClipPreview'; +import HighlightGenerator from './HighlightGenerator'; +import { EAvailableFeatures } from 'services/incremental-rollout'; + export type TModalClipsView = 'trim' | 'export' | 'preview' | 'remove'; interface IClipsViewProps { @@ -33,7 +37,10 @@ export default function ClipsView({ props: IClipsViewProps; emitSetView: (data: IViewState) => void; }) { - const { HighlighterService, UsageStatisticsService } = Services; + const { HighlighterService, UsageStatisticsService, IncrementalRolloutService } = Services; + const aiHighlighterEnabled = IncrementalRolloutService.views.featureIsEnabled( + EAvailableFeatures.aiHighlighter, + ); const clipsAmount = useVuex(() => HighlighterService.views.clips.length); const [clips, setClips] = useState<{ ordered: { id: string }[]; @@ -48,26 +55,18 @@ export default function ClipsView({ setClipsLoaded(true); }, []); + const getClips = useCallback(() => { + return HighlighterService.getClips(HighlighterService.views.clips, props.id); + }, [props.id]); + useEffect(() => { setClipsLoaded(false); - setClips( - sortAndFilterClips( - HighlighterService.getClips(HighlighterService.views.clips, props.id), - props.id, - activeFilter, - ), - ); + setClips(sortAndFilterClips(getClips(), props.id, activeFilter)); loadClips(props.id); }, [props.id, clipsAmount]); useEffect(() => { - setClips( - sortAndFilterClips( - HighlighterService.getClips(HighlighterService.views.clips, props.id), - props.id, - activeFilter, - ), - ); + setClips(sortAndFilterClips(getClips(), props.id, activeFilter)); }, [activeFilter]); useEffect(() => UsageStatisticsService.actions.recordFeatureUsage('Highlighter'), []); @@ -119,8 +118,10 @@ export default function ClipsView({ ); setClips({ - ordered: newClipArray.map(c => ({ id: c })), - orderedFiltered: filterClipsBySource(updatedClips, activeFilter).map(c => ({ id: c.path })), + ordered: newClipArray.map(clipPath => ({ id: clipPath })), + orderedFiltered: filterClipsBySource(updatedClips, activeFilter).map(clip => ({ + id: clip.path, + })), }); return; } @@ -165,13 +166,25 @@ export default function ClipsView({

emitSetView({ view: EHighlighterView.SETTINGS })} + onClick={() => + emitSetView( + streamId + ? { view: EHighlighterView.STREAM } + : { view: EHighlighterView.SETTINGS }, + ) + } > {props.streamTitle ?? $t('All highlight clips')}

@@ -185,13 +198,7 @@ export default function ClipsView({ { - setClips( - sortAndFilterClips( - HighlighterService.getClips(HighlighterService.views.clips, props.id), - props.id, - activeFilter, - ), - ); + setClips(sortAndFilterClips(getClips(), props.id, activeFilter)); }} />
@@ -204,15 +211,38 @@ export default function ClipsView({ { - setClips( - sortAndFilterClips( - HighlighterService.getClips(HighlighterService.views.clips, props.id), - props.id, - activeFilter, - ), - ); + setClips(sortAndFilterClips(getClips(), props.id, activeFilter)); }} /> + {streamId && + aiHighlighterEnabled && + HighlighterService.getClips(HighlighterService.views.clips, props.id) + .filter(clip => clip.source === 'AiClip') + .every(clip => (clip as IAiClip).aiInfo.metadata?.round) && ( + { + const clips = HighlighterService.getClips( + HighlighterService.views.clips, + props.id, + ); + const filteredClips = aiFilterClips(clips, streamId, filterOptions); + const filteredClipPaths = new Set(filteredClips.map(c => c.path)); + + clips.forEach(clip => { + const shouldBeEnabled = filteredClipPaths.has(clip.path); + const isEnabled = clip.enabled; + + if (shouldBeEnabled && !isEnabled) { + HighlighterService.enableClip(clip.path, true); + } else if (!shouldBeEnabled && isEnabled) { + HighlighterService.disableClip(clip.path); + } + }); + }} + combinedClipsDuration={getCombinedClipsDuration(getClips())} + roundDetails={HighlighterService.getRoundDetails(getClips())} + /> + )}
+ HighlighterService.getClips(HighlighterService.views.clips, streamId), + ); + + const totalDuration = clips + .filter(clip => clip.enabled) + .reduce((acc, clip) => acc + clip.duration! - clip.startTrim! - clip.endTrim!, 0); + + return {formatSecondsToHMS(totalDuration)}; +} + function AddClip({ streamId, addedClips, diff --git a/app/components-react/highlighter/ClipsViewModal.tsx b/app/components-react/highlighter/ClipsViewModal.tsx index 9ca8b5f04ee6..9b4aa8aa30a9 100644 --- a/app/components-react/highlighter/ClipsViewModal.tsx +++ b/app/components-react/highlighter/ClipsViewModal.tsx @@ -7,7 +7,6 @@ import styles from './ClipsView.m.less'; import ClipTrimmer from 'components-react/highlighter/ClipTrimmer'; import { Modal, Alert, Button } from 'antd'; import ExportModal from 'components-react/highlighter/ExportModal'; - import { $t } from 'services/i18n'; import PreviewModal from './PreviewModal'; diff --git a/app/components-react/highlighter/EditingControls.tsx b/app/components-react/highlighter/EditingControls.tsx index 1375d0cfc48d..ad9397c5f1a4 100644 --- a/app/components-react/highlighter/EditingControls.tsx +++ b/app/components-react/highlighter/EditingControls.tsx @@ -11,7 +11,7 @@ import { TModalClipsView } from './ClipsView'; import { useVuex } from 'components-react/hooks'; import { Services } from 'components-react/service-provider'; import { SUPPORTED_FILE_TYPES } from 'services/highlighter/constants'; -import { Clip } from 'services/highlighter/clip'; +import { RenderingClip } from 'services/highlighter/clip'; export function EditingControls({ emitSetShowModal, @@ -45,7 +45,7 @@ export function EditingControls({ async function setVideoFile(file: string, type: 'intro' | 'outro') { if (!videoExtensions.map(e => `.${e}`).includes(path.parse(file).ext)) return; - const tempClip = new Clip(file); + const tempClip = new RenderingClip(file); await tempClip.init(); HighlighterService.actions.setVideo({ [type]: { path: file, duration: tempClip.duration } }); } diff --git a/app/components-react/highlighter/ExportModal.m.less b/app/components-react/highlighter/ExportModal.m.less index 42ce669e2680..0d3302d9ddd9 100644 --- a/app/components-react/highlighter/ExportModal.m.less +++ b/app/components-react/highlighter/ExportModal.m.less @@ -1,5 +1,5 @@ .crossclip-container { - height: 300px; + min-height: 300px; display: flex; flex-direction: column; align-items: center; @@ -15,8 +15,8 @@ .sign-up-title { text-align: center; - font-size: 32px; - font-weight: 300; + font-size: 32px; + font-weight: 300; } .log-in { diff --git a/app/components-react/highlighter/ExportModal.tsx b/app/components-react/highlighter/ExportModal.tsx index 5ad46dba9989..a2d30f159afb 100644 --- a/app/components-react/highlighter/ExportModal.tsx +++ b/app/components-react/highlighter/ExportModal.tsx @@ -1,5 +1,5 @@ import React, { useState, useEffect, useMemo } from 'react'; -import { EExportStep, TFPS, TResolution, TPreset } from 'services/highlighter'; +import { EExportStep, TFPS, TResolution, TPreset, TOrientation } from 'services/highlighter'; import { Services } from 'components-react/service-provider'; import { FileInput, TextInput, ListInput } from 'components-react/shared/inputs'; import Form from 'components-react/shared/inputs/Form'; @@ -23,10 +23,19 @@ class ExportController { get exportInfo() { return this.service.views.exportInfo; } + getStreamTitle(streamId?: string) { + return ( + this.service.views.highlightedStreams.find(stream => stream.id === streamId)?.title || + 'My Video' + ); + } dismissError() { return this.service.actions.dismissError(); } + resetExportedState() { + return this.service.actions.resetExportedState(); + } setResolution(value: string) { this.service.actions.setResolution(parseInt(value, 10) as TResolution); @@ -44,8 +53,8 @@ class ExportController { this.service.actions.setExportFile(exportFile); } - exportCurrentFile(streamId: string | undefined) { - this.service.actions.export(false, streamId); + exportCurrentFile(streamId: string | undefined, orientation: TOrientation = 'horizontal') { + this.service.actions.export(false, streamId, orientation); } cancelExport() { @@ -79,14 +88,31 @@ export default function ExportModalProvider({ } function ExportModal({ close, streamId }: { close: () => void; streamId: string | undefined }) { - const { exportInfo, dismissError } = useController(ExportModalCtx); + const { exportInfo, dismissError, resetExportedState, getStreamTitle } = useController( + ExportModalCtx, + ); + const [videoName, setVideoName] = useState(getStreamTitle(streamId) + ' - highlights'); + + const unmount = () => { + dismissError(); + resetExportedState(); + }; // Clear all errors when this component unmounts - useEffect(dismissError, []); + useEffect(() => unmount, []); if (exportInfo.exporting) return ; - if (!exportInfo.exported) return ; - return ; + if (!exportInfo.exported) { + return ( + + ); + } + return ; } function ExportProgress() { @@ -128,7 +154,17 @@ function ExportProgress() { ); } -function ExportOptions({ close, streamId }: { close: () => void; streamId: string | undefined }) { +function ExportOptions({ + close, + streamId, + videoName, + onVideoNameChange, +}: { + close: () => void; + streamId: string | undefined; + videoName: string; + onVideoNameChange: (name: string) => void; +}) { const { UsageStatisticsService } = Services; const { exportInfo, @@ -139,10 +175,12 @@ function ExportOptions({ close, streamId }: { close: () => void; streamId: strin fileExists, setExport, exportCurrentFile, - store, + getStreamTitle, } = useController(ExportModalCtx); - const videoName = store.useState(s => s.videoName); + // Video name and export file are kept in sync + const [exportFile, setExportFile] = useState(getExportFileFromVideoName(videoName)); + function getExportFileFromVideoName(videoName: string) { const parsed = path.parse(exportInfo.file); const sanitized = videoName.replace(/[/\\?%*:|"<>\.,;=#]/g, ''); @@ -152,8 +190,27 @@ function ExportOptions({ close, streamId }: { close: () => void; streamId: strin function getVideoNameFromExportFile(exportFile: string) { return path.parse(exportFile).name; } - // Video name and export file are kept in sync - const [exportFile, setExportFile] = useState(getExportFileFromVideoName(videoName)); + + async function startExport(orientation: TOrientation) { + if (await fileExists(exportFile)) { + if ( + !(await confirmAsync({ + title: $t('Overwite File?'), + content: $t('%{filename} already exists. Would you like to overwrite it?', { + filename: path.basename(exportFile), + }), + okText: $t('Overwrite'), + })) + ) { + return; + } + } + + UsageStatisticsService.actions.recordFeatureUsage('HighlighterExport'); + + setExport(exportFile); + exportCurrentFile(streamId, orientation); + } return (
@@ -163,9 +220,7 @@ function ExportOptions({ close, streamId }: { close: () => void; streamId: strin label={$t('Video Name')} value={videoName} onInput={name => { - store.setState(s => { - s.videoName = name; - }); + onVideoNameChange(name); setExportFile(getExportFileFromVideoName(name)); }} uncontrolled={false} @@ -178,9 +233,7 @@ function ExportOptions({ close, streamId }: { close: () => void; streamId: strin value={exportFile} onChange={file => { setExportFile(file); - store.setState(s => { - s.videoName = getVideoNameFromExportFile(file); - }); + onVideoNameChange(getVideoNameFromExportFile(file)); }} /> void; streamId: strin - +
@@ -259,9 +294,8 @@ function ExportOptions({ close, streamId }: { close: () => void; streamId: strin ); } -function PlatformSelect(p: { onClose: () => void }) { - const { store, clearUpload } = useController(ExportModalCtx); - const videoName = store.useState(s => s.videoName); +function PlatformSelect({ onClose, videoName }: { onClose: () => void; videoName: string }) { + const { store, clearUpload, getStreamTitle } = useController(ExportModalCtx); const { UserService } = Services; const { isYoutubeLinked } = useVuex(() => ({ isYoutubeLinked: !!UserService.state.auth?.platforms.youtube, @@ -291,8 +325,8 @@ function PlatformSelect(p: { onClose: () => void }) { nowrap options={platformOptions} /> - {platform === 'youtube' && } - {platform !== 'youtube' && } + {platform === 'youtube' && } + {platform !== 'youtube' && } ); } diff --git a/app/components-react/highlighter/HighlightGenerator.m.less b/app/components-react/highlighter/HighlightGenerator.m.less new file mode 100644 index 000000000000..fa504f2f6e4b --- /dev/null +++ b/app/components-react/highlighter/HighlightGenerator.m.less @@ -0,0 +1,48 @@ +.wrapper { + display: flex; + align-items: center; + gap: 8px; + padding: 4px 16px; + background: var(--border); + border-radius: 8px; + width: fit-content; +} + +.dropdown { + color: white; +} +.option { + width: 100%; + padding: 8px 12px; + margin-bottom: 4px; + border-radius: 8px; + overflow: hidden; + opacity: 1; +} + +.tag { + padding: 3px 8px; + width: fit-content; + font-size: 12px; + background-color: var(--border); + border-radius: 4px; + margin-right: 4px; + color: #ffffff; +} + +.info-tag { + border-radius: 4px; + color: #ffffff; + background-color: #3a484f; + padding: 2px 6px; +} + +.reset-button { + display: flex; + align-items: center; + justify-content: center; + width: 32px; + height: 32px; + border-radius: 16px; + padding: 0; +} diff --git a/app/components-react/highlighter/HighlightGenerator.tsx b/app/components-react/highlighter/HighlightGenerator.tsx new file mode 100644 index 000000000000..6e1d93188527 --- /dev/null +++ b/app/components-react/highlighter/HighlightGenerator.tsx @@ -0,0 +1,185 @@ +import React, { useEffect, useRef, useState } from 'react'; +import { Button, Select, Checkbox, Typography } from 'antd'; +import { DownOutlined, RobotOutlined } from '@ant-design/icons'; +import { IFilterOptions } from './utils'; +import { IInput } from 'services/highlighter'; +import { getPlacementFromInputs, InputEmojiSection } from './InputEmojiSection'; +import { EHighlighterInputTypes } from 'services/highlighter/ai-highlighter/ai-highlighter'; +import styles from './HighlightGenerator.m.less'; +import { formatSecondsToHMS } from './ClipPreview'; +import { $t } from 'services/i18n'; +const { Option } = Select; + +const selectStyles = { + width: '220px', + borderRadius: '8px', +}; + +const dropdownStyles = { + borderRadius: '10px', + padding: '4px 4px', +}; + +const checkboxStyles = { + borderRadius: '8px', + width: '100%', +}; + +export default function HighlightGenerator({ + combinedClipsDuration, + roundDetails, + emitSetFilter, +}: { + combinedClipsDuration: number; // Maximum duration the highlight reel can be long - only used to restrict the targetDuration options + roundDetails: { + round: number; + inputs: IInput[]; + duration: number; + hypeScore: number; + }[]; + emitSetFilter: (filter: IFilterOptions) => void; +}) { + // console.log('reHIGHUI'); + + const [selectedRounds, setSelectedRounds] = useState([0]); + const [filterType, setFilterType] = useState<'duration' | 'hypescore'>('duration'); + const [targetDuration, setTargetDuration] = useState(combinedClipsDuration + 100); + const options = [ + { + value: 1, + label: $t('%{duration} minute', { duration: 1 }), + }, + ...[2, 5, 10, 12, 15, 20, 30].map(value => ({ + value, + label: $t('%{duration} minutes', { duration: value }), + })), + ]; + const filteredOptions = options.filter(option => option.value * 60 <= combinedClipsDuration); + const isFirstRender = useRef(true); + useEffect(() => { + // To not emit on first render + if (isFirstRender.current) { + isFirstRender.current = false; + return; + } + + emitSetFilter({ + rounds: selectedRounds, + targetDuration: filterType === 'duration' ? targetDuration * 60 : 9999, + includeAllEvents: true, + }); + }, [selectedRounds, filterType, targetDuration]); + + function roundDropdownDetails(roundDetails: { + round: number; + inputs: IInput[]; + duration: number; + hypeScore: number; + }) { + const combinedKillAndKnocked = roundDetails.inputs.reduce((count, input) => { + if ( + input.type === EHighlighterInputTypes.KILL || + input.type === EHighlighterInputTypes.KNOCKED + ) { + return count + 1; + } + return count; + }, 0); + const won = roundDetails.inputs.some(input => input.type === EHighlighterInputTypes.VICTORY); + let rank = null; + if (!won) { + rank = getPlacementFromInputs(roundDetails.inputs); + } + return ( +
+
Round {roundDetails.round}
+
+
{combinedKillAndKnocked} ðŸ”Ŧ
+ {won ? ( +
1st 🏆
+ ) : ( +
{`${rank ? '#' + rank : ''} ðŸŠĶ`}
+ )} +
{`${roundDetails.hypeScore} ðŸ”Ĩ`}
+
{`${formatSecondsToHMS(roundDetails.duration)}`}
+
+
+ ); + } + + return ( +
+

+ ðŸĪ– {$t('Create highlight video of')} +

+ +

{$t('with a duration of')}

+ +
+ ); +} diff --git a/app/components-react/highlighter/InputEmojiSection.m.less b/app/components-react/highlighter/InputEmojiSection.m.less new file mode 100644 index 000000000000..c1d7d0f50749 --- /dev/null +++ b/app/components-react/highlighter/InputEmojiSection.m.less @@ -0,0 +1,3 @@ +.description { + text-wrap: nowrap; +} diff --git a/app/components-react/highlighter/InputEmojiSection.tsx b/app/components-react/highlighter/InputEmojiSection.tsx new file mode 100644 index 000000000000..6ce9326fd2c2 --- /dev/null +++ b/app/components-react/highlighter/InputEmojiSection.tsx @@ -0,0 +1,215 @@ +import React from 'react'; +import { IAiClip, IDeathMetadata, IInput, IKillMetadata, TClip } from 'services/highlighter'; +import { isAiClip } from './utils'; +import { EHighlighterInputTypes } from 'services/highlighter/ai-highlighter/ai-highlighter'; +import styles from './InputEmojiSection.m.less'; + +interface TypeWording { + emoji: string; + description: string; +} +const TYPE_WORDING_MAP: Record TypeWording> = { + [EHighlighterInputTypes.KILL]: count => ({ + emoji: 'ðŸ”Ŧ', + description: count > 1 ? 'eliminations' : 'elimination', + }), + [EHighlighterInputTypes.KNOCKED]: count => ({ + emoji: 'ðŸĨŠ', + description: count > 1 ? 'knocks' : 'knocked', + }), + [EHighlighterInputTypes.DEATH]: count => ({ + emoji: 'ðŸŠĶ', + description: count > 1 ? 'deaths' : 'death', + }), + [EHighlighterInputTypes.VICTORY]: count => ({ + emoji: '🏆', + description: count > 1 ? 'wins' : 'win', + }), + [EHighlighterInputTypes.DEPLOY]: count => ({ + emoji: '🊂', + description: count > 1 ? 'deploys' : 'deploy', + }), + [EHighlighterInputTypes.PLAYER_KNOCKED]: () => ({ + emoji: 'ðŸ˜ĩ', + description: 'got knocked', + }), + BOT_KILL: count => ({ + emoji: 'ðŸĪ–', + description: count > 1 ? 'bot eliminations' : 'bot elimination', + }), + rounds: count => ({ + emoji: '🏁', + description: count === 0 || count > 1 ? `rounds ${count === 0 ? 'detected' : ''}` : 'round', + }), +}; + +function getTypeWordingFromType( + type: string, + count: number, +): { emoji: string; description: string } { + return TYPE_WORDING_MAP[type]?.(count) ?? { emoji: '', description: '?' }; +} + +function getInputTypeCount(clips: TClip[]): { [type: string]: number } { + const typeCounts: { [type: string]: number } = {}; + if (clips.length === 0) { + return typeCounts; + } + clips.forEach(clip => { + if (isAiClip(clip)) { + clip.aiInfo.inputs?.forEach(input => { + const type = input.type; + if (type === EHighlighterInputTypes.KILL) { + if ((input?.metadata as IKillMetadata)?.bot_kill === true) { + const currentCount = typeCounts['BOT_KILL']; + typeCounts['BOT_KILL'] = currentCount ? currentCount + 1 : 1; + return; + } + } + if (typeCounts[type]) { + typeCounts[type] += 1; + } else { + typeCounts[type] = 1; + } + }); + } + }); + return typeCounts; +} +function isDeath(type: string): boolean { + return type === EHighlighterInputTypes.DEATH; +} + +function getGamePlacement(clips: TClip[]): number | null { + const deathClip = clips.find( + clip => + isAiClip(clip) && + clip.aiInfo.inputs.some(input => input.type === EHighlighterInputTypes.DEATH), + ) as IAiClip; + + return getPlacementFromInputs(deathClip.aiInfo.inputs); +} +function getAmountOfRounds(clips: TClip[]): number { + const rounds: number[] = []; + clips.filter(isAiClip).forEach(clip => { + rounds.push(clip.aiInfo.metadata?.round || 1); + }); + return Math.max(0, ...rounds); +} + +export function getPlacementFromInputs(inputs: IInput[]): number | null { + const deathInput = inputs.find(input => input.type === EHighlighterInputTypes.DEATH); + return (deathInput?.metadata as IDeathMetadata)?.place || null; +} + +export function InputEmojiSection({ + clips, + includeRounds, + includeDeploy, + showCount, + showDescription, +}: { + clips: TClip[]; + includeRounds: boolean; + includeDeploy: boolean; + showCount?: boolean; + showDescription?: boolean; +}): JSX.Element { + const excludeTypes = [ + EHighlighterInputTypes.GAME_SEQUENCE, + EHighlighterInputTypes.GAME_START, + EHighlighterInputTypes.GAME_END, + EHighlighterInputTypes.VOICE_ACTIVITY, + EHighlighterInputTypes.META_DURATION, + EHighlighterInputTypes.LOW_HEALTH, + ]; + + const inputTypeMap = Object.entries(getInputTypeCount(clips)); + const filteredInputTypeMap = inputTypeMap.filter( + ([type]) => + !excludeTypes.includes(type as EHighlighterInputTypes) && + (inputTypeMap.length <= 2 || includeDeploy || type !== 'deploy'), + ); + + return ( +
+ {includeRounds && } + {filteredInputTypeMap.map(([type, count]) => ( + + ))} + + {inputTypeMap.length > 3 ? '...' : ''} +
+ ); +} + +export function RoundTag({ clips }: { clips: TClip[] }): JSX.Element { + const rounds = getAmountOfRounds(clips); + const { emoji, description } = getTypeWordingFromType('rounds', rounds); + return ( +
+ {emoji} + + {rounds} {description} + +
+ ); +} + +export function AiMomentTag({ + type, + count, + clips, + showCount, + showDescription, + includeRounds, +}: { + type: string; + count: number; + clips: TClip[]; + showCount?: boolean; + showDescription?: boolean; + includeRounds?: boolean; +}): JSX.Element { + const { emoji, description } = getTypeWordingFromType(type, count); + return ( +
+ {emoji} + {(showCount !== false || showDescription !== false) && ( + + {showCount !== false && `${count} `} + {showDescription !== false && description} + {!includeRounds && isDeath(type) && getGamePlacement(clips) + ? '#' + getGamePlacement(clips) + : ''} + + )} +
+ ); +} + +export function ManualClipTag({ clips }: { clips: TClip[] }): JSX.Element { + const manualClips = clips.filter( + clip => clip.source === 'ReplayBuffer' || clip.source === 'Manual', + ); + if (manualClips.length === 0) { + return <>; + } + return ( +
+ 🎎 + {`${manualClips.length} ${ + manualClips.length === 1 ? 'manual' : 'manuals' + }`} +
+ ); +} diff --git a/app/components-react/highlighter/PreviewModal.m.less b/app/components-react/highlighter/PreviewModal.m.less new file mode 100644 index 000000000000..486e76d206ca --- /dev/null +++ b/app/components-react/highlighter/PreviewModal.m.less @@ -0,0 +1,29 @@ +.timeline { + width: 100%; + padding-left: 8px; + padding-right: 8px; + display: flex; + overflow-x: auto; +} + +.timeline-item { + cursor: pointer; + border-radius: 6px; + width: fit-content; + border: solid 2px transparent; +} + +.timeline-item-wrapper { + display: flex; + gap: 4px; + padding-bottom: 8px; + justify-content: center; +} + +.video-player { + position: absolute; + top: 0; + left: 0; + width: 100%; + height: 100%; +} diff --git a/app/components-react/highlighter/PreviewModal.tsx b/app/components-react/highlighter/PreviewModal.tsx index b8425f807600..f6331cf1e9ae 100644 --- a/app/components-react/highlighter/PreviewModal.tsx +++ b/app/components-react/highlighter/PreviewModal.tsx @@ -1,9 +1,11 @@ -import { useVuex } from 'components-react/hooks'; -import React, { useEffect, useRef } from 'react'; +import React, { useEffect, useRef, useState } from 'react'; import { Services } from 'components-react/service-provider'; -import { Progress, Alert } from 'antd'; import { $t } from 'services/i18n'; - +import { TClip } from 'services/highlighter'; +import { sortClipsByOrder } from './utils'; +import MiniClipPreview from './MiniClipPreview'; +import { PauseButton, PlayButton } from './StreamCard'; +import styles from './PreviewModal.m.less'; export default function PreviewModal({ close, streamId, @@ -11,76 +13,274 @@ export default function PreviewModal({ close: () => void; streamId: string | undefined; }) { + if (streamId === undefined) { + close(); + console.error('streamId is required'); + } const { HighlighterService } = Services; - const v = useVuex(() => ({ - exportInfo: HighlighterService.views.exportInfo, - })); + const clips = HighlighterService.getClips(HighlighterService.views.clips, streamId); + const { intro, outro } = HighlighterService.views.video; + const audioSettings = HighlighterService.views.audio; + const [currentClipIndex, setCurrentClipIndex] = useState(0); + const currentClipIndexRef = useRef(0); + const sortedClips = [...sortClipsByOrder(clips, streamId).filter(c => c.enabled)]; - useEffect(() => { - HighlighterService.actions.export(true, streamId); + const playlist = [ + ...(intro.duration + ? [ + { + src: intro.path, + path: intro.path, + start: 0, + end: intro.duration!, + type: 'video/mp4', + }, + ] + : []), + ...sortedClips.map((clip: TClip) => ({ + src: clip.path + `#t=${clip.startTrim},${clip.duration! - clip.endTrim}`, + path: clip.path, + start: clip.startTrim, + end: clip.duration! - clip.endTrim, + type: 'video/mp4', + })), + ...(outro.duration && outro.path + ? [ + { + src: outro.path, + path: outro.path, + start: 0, + end: outro.duration!, + type: 'video/mp4', + }, + ] + : []), + ]; + const videoPlayer = useRef(null); + const containerRef = useRef(null); + const audio = useRef(null); + const isChangingClip = useRef(false); + const [isPlaying, setIsPlaying] = useState(true); - return () => HighlighterService.actions.cancelExport(); - }, []); + function isRoughlyEqual(a: number, b: number, tolerance: number = 0.3): boolean { + return Math.abs(a - b) <= tolerance; + } - // Clear all errors when this component unmounts useEffect(() => { - return () => HighlighterService.actions.dismissError(); - }, []); + if (!videoPlayer.current) { + return; + } + //Pause gets also triggered when the video ends. We dont want to change the clip in that case + const nextClip = () => { + if (!isChangingClip.current) { + isChangingClip.current = true; + + setCurrentClipIndex(prevIndex => { + const newIndex = (prevIndex + 1) % playlist.length; + + videoPlayer.current!.src = playlist[currentClipIndex].src; + videoPlayer.current!.load(); + + playAudio(newIndex, newIndex === prevIndex + 1); + + return newIndex; + }); + + setTimeout(() => { + isChangingClip.current = false; + }, 500); + } + }; + + const handleEnded = () => { + nextClip(); + }; - // Kind of hacky but used to know if we ever were exporting at any point - const didStartExport = useRef(false); - if (v.exportInfo.exporting) didStartExport.current = true; + const handlePause = () => { + // sometimes player fires paused event before ended, in this case we need to compare timestamps + // and check if we are at the end of the clip + const currentTime = videoPlayer.current!.currentTime; + const endTime = playlist[currentClipIndexRef.current].end; + + if (currentTime >= endTime || isRoughlyEqual(currentTime, endTime)) { + nextClip(); + } + }; + + const handlePlay = () => { + setIsPlaying(true); + }; + + const handleAudioEnd = () => { + audio.current!.currentTime = 0; + audio.current!.play().catch(e => console.error('Error playing audio:', e)); + }; + + videoPlayer.current.addEventListener('ended', handleEnded); + videoPlayer.current.addEventListener('play', handlePlay); + videoPlayer.current.addEventListener('pause', handlePause); + + if (audioSettings.musicEnabled && audioSettings.musicPath && playlist.length > 0) { + audio.current = new Audio(audioSettings.musicPath); + audio.current.volume = audioSettings.musicVolume / 100; + audio.current.autoplay = true; + audio.current.addEventListener('ended', handleAudioEnd); + } + + return () => { + videoPlayer.current?.removeEventListener('ended', handleEnded); + videoPlayer.current?.removeEventListener('play', handlePlay); + videoPlayer.current?.removeEventListener('pause', handlePause); + if (audio.current) { + audio.current.pause(); + audio.current.removeEventListener('ended', handleAudioEnd); + audio.current = null; + } + }; + }, [playlist.length, videoPlayer.current]); useEffect(() => { - // Close the window immediately if we stopped exporting due to cancel - if (!v.exportInfo.exporting && v.exportInfo.cancelRequested && didStartExport.current) { - close(); + currentClipIndexRef.current = currentClipIndex; + if (videoPlayer.current === null || playlist.length === 0) { + return; + } + videoPlayer.current!.src = playlist[currentClipIndex].src; + videoPlayer.current!.load(); + videoPlayer.current!.play().catch(e => console.error('Error playing video:', e)); + + // currently its done by querying DOM, don't want to store a giant array of refs + // that wont be used otherwise + document.getElementById('preview-' + currentClipIndex)?.scrollIntoView({ + behavior: 'smooth', + block: 'center', + inline: 'center', + }); + }, [currentClipIndex]); + + function togglePlay() { + const currentPlayer = videoPlayer.current; + if (currentPlayer?.paused) { + currentPlayer.play().catch(e => console.error('Error playing video:', e)); + if (audio.current && audio.current.currentTime > 0) { + audio.current?.play().catch(e => console.error('Error playing audio:', e)); + } + } else { + setIsPlaying(false); + currentPlayer?.pause(); + audio.current?.pause(); } - }, [v.exportInfo.exporting, v.exportInfo.cancelRequested]); + } + + function playPauseButton() { + if (isPlaying) { + return ; + } else { + return ; + } + } + + function jumpToClip(index: number) { + if (currentClipIndex === index) { + return; + } + setCurrentClipIndex(index); + + const clip = playlist[index]; + videoPlayer.current!.src = clip.src; + videoPlayer.current!.load(); + + playAudio(index); + } + + function playAudio(index: number, continuation = false) { + // if its a continuation of a previous segment, no need to seek + // and introduce playback lag + if (continuation || !audio.current) { + return; + } + + // clips don't have absolute timestamps, we need to calculate the start time + // in relation to previous clips + const startTime = playlist + .filter((_, i) => i < index) + .reduce((acc, curr) => acc + (curr.end - curr.start), 0); + + if (startTime < audio.current!.duration) { + audio.current!.currentTime = startTime; + } else { + const start = startTime % audio.current!.duration; + audio.current!.currentTime = start; + // audio.current?.pause(); + } + audio.current!.play().catch(e => console.error('Error playing audio:', e)); + } + + const handleScroll = (event: { deltaY: any }) => { + if (containerRef.current) { + containerRef.current.scrollLeft += event.deltaY; + } + }; + + if (playlist.length === 0) { + return ( +
+

{$t('Preview')}

+

Select at least one clip to preview your video

+
+ ); + } return (
-

{$t('Render Preview')}

+

{$t('Preview')}

- {$t( - 'The render preview shows a low-quality preview of the final rendered video. The final exported video will be higher resolution, framerate, and quality.', - )} + This is just a preview of your highlight reel. Loading times between clips are possible.

- {v.exportInfo.exporting && ( - - )} - {v.exportInfo.exporting && v.exportInfo.cancelRequested && {$t('Canceling...')}} - {v.exportInfo.exporting &&
} - {v.exportInfo.exporting && ( - - )} - {!v.exportInfo.exporting && v.exportInfo.error && ( - - )} - {!v.exportInfo.exporting && - !v.exportInfo.cancelRequested && - !v.exportInfo.error && - didStartExport.current && ( -
); } diff --git a/app/components-react/highlighter/SettingsView.m.less b/app/components-react/highlighter/SettingsView.m.less index bfe4255e162c..fc9ea0c282ce 100644 --- a/app/components-react/highlighter/SettingsView.m.less +++ b/app/components-react/highlighter/SettingsView.m.less @@ -19,12 +19,20 @@ .inner-scroll-wrapper { display: flex; - background-color: #09161d; + background-color: var(--section); padding: 56px; border-radius: 24px; gap: 24px; } +.headerbar-tag { + margin: 0; + margin-left: 4px; + font-size: 14px; + padding: 2px 6px; + border-radius: 4px; + background-color: var(--section); +} .card-wrapper { display: flex; flex-direction: column; @@ -39,12 +47,24 @@ flex-direction: column; gap: 24px; position: relative; - background-color: #17242d; + background-color: var(--border); padding: 40px; border-radius: 16px; border: solid 2px #2b5bd7; } +.recommended-corner { + position: absolute; + right: 0; + bottom: 0; + border-radius: 16px 0 9px 0; + padding: 8px; + padding-bottom: 5px; + background-color: #2b5bd7; + color: white; + height: fit-content; +} + .manual-card { display: flex; flex-direction: column; @@ -61,9 +81,8 @@ } .setting-section { - background-color: #17242d; + background-color: var(--border); h3 { - color: #bdc2c4; } padding: 24px; @@ -83,3 +102,9 @@ background-size: contain; background-repeat: no-repeat; } + +.card-headerbar { + display: flex; + align-items: center; + gap: 8px; +} diff --git a/app/components-react/highlighter/SettingsView.tsx b/app/components-react/highlighter/SettingsView.tsx index 77339085970e..e5193866be2d 100644 --- a/app/components-react/highlighter/SettingsView.tsx +++ b/app/components-react/highlighter/SettingsView.tsx @@ -12,6 +12,7 @@ import Scrollable from 'components-react/shared/Scrollable'; import styles from './SettingsView.m.less'; import { $t } from 'services/i18n'; import { EHighlighterView, IViewState } from 'services/highlighter'; +import { EAvailableFeatures } from 'services/incremental-rollout'; export default function SettingsView({ emitSetView, @@ -20,13 +21,23 @@ export default function SettingsView({ emitSetView: (data: IViewState) => void; close: () => void; }) { - const { HotkeysService, SettingsService, StreamingService, HighlighterService } = Services; + const { + HotkeysService, + SettingsService, + StreamingService, + HighlighterService, + IncrementalRolloutService, + } = Services; + const aiHighlighterEnabled = IncrementalRolloutService.views.featureIsEnabled( + EAvailableFeatures.aiHighlighter, + ); const [hotkey, setHotkey] = useState(null); const hotkeyRef = useRef(null); const v = useVuex(() => ({ settingsValues: SettingsService.views.values, isStreaming: StreamingService.isStreaming, + useAiHighlighter: HighlighterService.views.useAiHighlighter, })); const correctlyConfigured = @@ -97,6 +108,10 @@ export default function SettingsView({ SettingsService.actions.setSettingsPatch({ Output: { RecRBTime: time } }); } + function toggleUseAiHighlighter() { + HighlighterService.actions.toggleAiHighlighter(); + } + return (
@@ -109,9 +124,14 @@ export default function SettingsView({

+ {aiHighlighterEnabled && ( + + )} {/* New button coming with next PR */}
@@ -119,11 +139,38 @@ export default function SettingsView({
+ {aiHighlighterEnabled && ( +
+
+
+ +

{$t('AI Highlighter')}

+

{$t('For Fortnite streams (Beta)')}

+
+
+ +

+ {$t( + 'Automatically capture the best moments from your livestream and turn them into a highlight video.', + )} +

+ + +
{$t('Recommended')}
+
+ )}
-

{$t('Manual Highlighter')}

+

+ {aiHighlighterEnabled ? 'Or use the manual highlighter ' : 'Manual highlighter'} +

{$t( - 'The hotkey highlighter allows you to clip the best moments during your livestream manually and edit them together afterwards.', + 'Manually capture the best moments from your livestream with a hotkey command, and automatically turn them into a highlight video.', )}

diff --git a/app/components-react/highlighter/StreamCard.m.less b/app/components-react/highlighter/StreamCard.m.less new file mode 100644 index 000000000000..849a491aca4e --- /dev/null +++ b/app/components-react/highlighter/StreamCard.m.less @@ -0,0 +1,183 @@ +@import '../../styles/index'; + +.stream-card { + overflow: hidden; + background: var(--border); + display: flex; + flex-direction: column; + border-radius: 10px; + gap: 16px; + height: fit-content; + width: 422px; + cursor: pointer; +} + +.clips-amount { + position: absolute; + top: 50%; + left: 50%; + color: white; + text-align: center; + font-size: 14px; + z-index: 99; + display: flex; + gap: 3px; + padding-right: 3px; + text-shadow: 0px 0px 6px black; + transform: translate(-24px, 15px); +} + +.centered-overlay-item { + position: absolute; + top: 50%; + left: 50%; + color: white; + text-align: center; + font-size: 14px; + z-index: 99; + display: flex; + gap: 3px; + padding-right: 3px; + transform: translate(-50%, -50%); +} + +.thumbnail-wrapper { + position: relative; + + --thumbWith: 192px; + --thumbHeight: 108px; + overflow-x: clip; + + width: calc(var(--thumbWith) * 2.2); + height: calc(var(--thumbHeight) * 2.2); + background: var(--Day-Colors-Dark-4, #4f5e65); +} + +.thumbnail-wrapper-small { + position: absolute; + --thumbWith: 192px; + --thumbHeight: 108px; + overflow-x: clip; + border-radius: 4px; + + width: calc(var(--thumbWith) * 0.35); // Destroys the aspect ratio but is nicer in the ui + height: calc(var(--thumbHeight) * 0.4); + border: 2px #22292d solid; +} + +.progressbar-background { + display: flex; + position: relative; + justify-content: space-between; + width: 100%; + height: 40px; + border-radius: 4px; + overflow: hidden; + background: var(--Day-Colors-Dark-4, #4f5e65); +} + +.progressbar-progress { + height: 100%; + width: 100%; + transform: scaleX(0); + + background: var(--Night-Colors-Light-2, #f5f8fa); +} + +.progressbar-text { + height: 40px; + display: flex; + align-items: center; + padding-left: 16px; + position: absolute; + color: black; + font-size: 16px; + z-index: 1; +} + +.loader { + border: 2px solid #f3f3f3; /* Light grey */ + border-top: 2px solid #3e3e3e; /* Blue */ + border-radius: 50%; + width: 16px; + height: 16px; + animation: spin 2s linear infinite; +} + +@keyframes spin { + 0% { + transform: rotate(0deg); + } + 100% { + transform: rotate(360deg); + } +} + +.delete-button { + display: flex; + top: 8px; + left: 8px; + gap: 8px; + align-items: center; + opacity: 0; + backdrop-filter: blur(6px); +} + +.stream-card:hover .delete-button { + opacity: 1; +} + +.streaminfo-wrapper { + display: flex; + flex-direction: column; + gap: 8px; + padding: 20px; + padding-top: 0px; +} + +.streamcard-title { + margin: 0; + width: 275px; + text-overflow: ellipsis; + overflow: hidden; + white-space: nowrap; +} + +.title-date-wrapper { + display: flex; + flex-direction: column; + gap: 4px; + height: fit-content; +} + +.title-rotated-clips-wrapper { + display: flex; + justify-content: space-between; + gap: 8px; + height: fit-content; +} + +.emoji-wrapper { + padding-top: 6px; + padding-bottom: 6px; + margin: 0; + display: flex; + gap: 8px; + justify-content: start; +} + +.cancel-button { + border: none; + background-color: transparent; + color: black; + display: flex; + position: absolute; + right: 0; + align-items: center; +} + +.button-bar-wrapper { + display: flex; + gap: 4px; + justify-content: space-between; +} diff --git a/app/components-react/highlighter/StreamCard.tsx b/app/components-react/highlighter/StreamCard.tsx new file mode 100644 index 000000000000..ff287fe06945 --- /dev/null +++ b/app/components-react/highlighter/StreamCard.tsx @@ -0,0 +1,366 @@ +import React from 'react'; +import { + EAiDetectionState, + EHighlighterView, + IHighlightedStream, + IViewState, + StreamInfoForAiHighlighter, + TClip, +} from 'services/highlighter'; +import styles from './StreamCard.m.less'; +import { Button } from 'antd'; +import { Services } from 'components-react/service-provider'; +import { isAiClip } from './utils'; +import { useVuex } from 'components-react/hooks'; +import { InputEmojiSection } from './InputEmojiSection'; +import { $t } from 'services/i18n'; + +export default function StreamCard({ + streamId, + clipsOfStreamAreLoading, + emitSetView, + emitGeneratePreview, + emitExportVideo, + emitRemoveStream, + emitCancelHighlightGeneration, +}: { + streamId: string; + clipsOfStreamAreLoading: string | null; + emitSetView: (data: IViewState) => void; + emitGeneratePreview: () => void; + emitExportVideo: () => void; + emitRemoveStream: () => void; + emitCancelHighlightGeneration: () => void; +}) { + const { HighlighterService } = Services; + const clips = useVuex(() => + HighlighterService.views.clips + .filter(c => c.streamInfo?.[streamId]) + .map(clip => { + if (isAiClip(clip) && (clip.aiInfo as any).moments) { + clip.aiInfo.inputs = (clip.aiInfo as any).moments; + } + return clip; + }), + ); + const stream = useVuex(() => + HighlighterService.views.highlightedStreams.find(s => s.id === streamId), + ); + if (!stream) { + return <>; + } + + function showStreamClips() { + if (stream?.state.type !== EAiDetectionState.IN_PROGRESS) { + emitSetView({ view: EHighlighterView.CLIPS, id: stream?.id }); + } + } + + return ( +
{ + showStreamClips(); + }} + > + +
+
+
+

{stream.title}

+

{new Date(stream.date).toDateString()}

+
+ +
+

+ {stream.state.type === EAiDetectionState.FINISHED ? ( + + ) : ( +
+ )} +

+ { + HighlighterService.actions.restartAiDetection(stream.path, stream); + }} + emitSetView={emitSetView} + /> +
+
+ ); +} + +function ActionBar({ + stream, + clips, + clipsOfStreamAreLoading, + emitCancelHighlightGeneration, + emitExportVideo, + emitShowStreamClips, + emitRestartAiDetection, + emitSetView, +}: { + stream: IHighlightedStream; + clips: TClip[]; + clipsOfStreamAreLoading: string | null; + emitCancelHighlightGeneration: () => void; + emitExportVideo: () => void; + emitShowStreamClips: () => void; + emitRestartAiDetection: () => void; + emitSetView: (data: IViewState) => void; +}): JSX.Element { + function getFailedText(state: EAiDetectionState): string { + switch (state) { + case EAiDetectionState.ERROR: + return $t('Highlights failed'); + case EAiDetectionState.CANCELED_BY_USER: + return $t('Highlights cancelled'); + default: + return ''; + } + } + + // In Progress + if (stream?.state.type === EAiDetectionState.IN_PROGRESS) { + return ( +
+
{$t('Searching for highlights...')}
+
+ + +
+ ); + } + + // If finished + if (stream && clips.length > 0) { + return ( +
+ + + {/* TODO: What clips should be included when user clicks this button + bring normal export modal in here */} + +
+ ); + } + + //if failed or no clips + return ( +
+
+ {getFailedText(stream.state.type)} +
+
+ {stream?.state.type === EAiDetectionState.CANCELED_BY_USER ? ( + + ) : ( + + )} +
+
+ ); +} + +export function Thumbnail({ + clips, + clipsOfStreamAreLoading, + stream, + emitGeneratePreview, + emitCancelHighlightGeneration, + emitRemoveStream, +}: { + clips: TClip[]; + clipsOfStreamAreLoading: string | null; + stream: IHighlightedStream; + emitGeneratePreview: () => void; + emitCancelHighlightGeneration: () => void; + emitRemoveStream: () => void; +}) { + function getThumbnailText(state: EAiDetectionState): JSX.Element | string { + if (clipsOfStreamAreLoading === stream?.id) { + return
; + } + + if (clips.length > 0) { + return ; + } + switch (state) { + case EAiDetectionState.IN_PROGRESS: + return $t('Searching for highlights...'); + case EAiDetectionState.FINISHED: + if (clips.length === 0) { + return $t('Not enough highlights found'); + } + return ; + case EAiDetectionState.CANCELED_BY_USER: + return $t('Highlights cancelled'); + case EAiDetectionState.ERROR: + return $t('Highlights cancelled'); + default: + return ''; + } + } + + return ( +
+ + { + if (stream.state.type !== EAiDetectionState.IN_PROGRESS) { + emitGeneratePreview(); + e.stopPropagation(); + } + }} + style={{ height: '100%' }} + src={ + clips.find(clip => clip?.streamInfo?.[stream.id]?.orderPosition === 0)?.scrubSprite || + clips.find(clip => clip.scrubSprite)?.scrubSprite + } + alt="" + /> +
+
{ + if (stream.state.type !== EAiDetectionState.IN_PROGRESS) { + emitGeneratePreview(); + e.stopPropagation(); + } + }} + > + {getThumbnailText(stream.state.type)} +
+
+
+ ); +} + +export function RotatedClips({ clips }: { clips: TClip[] }) { + return ( +
+ {clips.length > 0 ? ( +
+
+ {clips.length} + clips +
+ {clips.slice(0, 3).map((clip, index) => ( +
+ +
+ ))} +
+ ) : ( + '' + )} +
+ ); +} + +export const PlayButton = () => ( + + + +); +export const PauseButton = () => ( + + + + +); diff --git a/app/components-react/highlighter/StreamView.m.less b/app/components-react/highlighter/StreamView.m.less new file mode 100644 index 000000000000..03d1a01412ff --- /dev/null +++ b/app/components-react/highlighter/StreamView.m.less @@ -0,0 +1,76 @@ +@import '../../styles/index'; + +.stream-view-root { + position: relative; + + :global(.ant-modal-wrap), + :global(.ant-modal-mask) { + position: absolute; + } + + :global(.os-content-glue) { + width: 100% !important; + height: 100% !important; + } +} + +.streams-wrapper { + display: grid; + grid-template-columns: auto auto auto; + + gap: 16px; +} + +.stream-view-root { + position: relative; + + :global(.ant-modal-wrap), + :global(.ant-modal-mask) { + position: absolute; + } + + :global(.os-content-glue) { + width: 100% !important; + height: 100% !important; + } +} + +.upload-wrapper { + display: flex; + padding: 16px; + padding-left: 22px; + margin-top: -17px; //1px bcs of border + align-items: center; + gap: 16px; + border-radius: 8px; + border: 1px dashed var(--Day-Colors-Dark-4, #4f5e65); +} + +.manual-upload-wrapper { + width: 100%; + display: flex; + flex-direction: column; + gap: 16px; + padding: 8px; +} +.title-input-wrapper { + width: 100%; + display: flex; + flex-direction: column; + gap: 8px; +} + +.streamcards-wrapper { + width: 100%; + display: flex; + flex-wrap: wrap; + gap: 8px; +} + +.period-divider { + border-bottom: 1px solid var(--border); + margin: 20px 0; + padding-bottom: 10px; + font-size: 18px; + font-weight: bold; +} diff --git a/app/components-react/highlighter/StreamView.tsx b/app/components-react/highlighter/StreamView.tsx new file mode 100644 index 000000000000..121ec780f436 --- /dev/null +++ b/app/components-react/highlighter/StreamView.tsx @@ -0,0 +1,402 @@ +import { useVuex } from 'components-react/hooks'; +import React, { useRef, useState } from 'react'; +import { Services } from 'components-react/service-provider'; +import styles from './StreamView.m.less'; +import { EHighlighterView, IViewState, StreamInfoForAiHighlighter } from 'services/highlighter'; +import isEqual from 'lodash/isEqual'; +import { Modal, Button, Alert } from 'antd'; +import ExportModal from 'components-react/highlighter/ExportModal'; +import { SUPPORTED_FILE_TYPES } from 'services/highlighter/constants'; +import Scrollable from 'components-react/shared/Scrollable'; +import { $t } from 'services/i18n'; +import * as remote from '@electron/remote'; +import uuid from 'uuid'; +import StreamCard from './StreamCard'; +import path from 'path'; +import PreviewModal from './PreviewModal'; +import moment from 'moment'; + +type TModalStreamView = + | { type: 'export'; id: string | undefined } + | { type: 'preview'; id: string | undefined } + | { type: 'upload' } + | { type: 'remove'; id: string | undefined } + | null; + +export default function StreamView({ emitSetView }: { emitSetView: (data: IViewState) => void }) { + const { HighlighterService, HotkeysService, UsageStatisticsService } = Services; + const v = useVuex(() => ({ + exportInfo: HighlighterService.views.exportInfo, + error: HighlighterService.views.error, + uploadInfo: HighlighterService.views.uploadInfo, + })); + + // Below is only used because useVueX doesnt work as expected + // there probably is a better way to do this + const currentStreams = useRef<{ id: string; date: string }[]>(); + const highlightedStreams = useVuex(() => { + const newStreamIds = [ + ...HighlighterService.views.highlightedStreams.map(stream => { + return { id: stream.id, date: stream.date }; + }), + ]; + + if (currentStreams.current === undefined || !isEqual(currentStreams.current, newStreamIds)) { + currentStreams.current = newStreamIds; + } + return currentStreams.current.sort( + (a, b) => new Date(b.date).getTime() - new Date(a.date).getTime(), + ); + }); + + const currentAiDetectionState = useRef(); + + const aiDetectionInProgress = useVuex(() => { + const newDetectionInProgress = HighlighterService.views.highlightedStreams.some( + stream => stream.state.type === 'detection-in-progress', + ); + + if ( + currentAiDetectionState.current === undefined || + !isEqual(currentAiDetectionState.current, newDetectionInProgress) + ) { + currentAiDetectionState.current = newDetectionInProgress; + } + return currentAiDetectionState.current; + }); + + const [showModal, rawSetShowModal] = useState(null); + const [modalWidth, setModalWidth] = useState('700px'); + const [clipsOfStreamAreLoading, setClipsOfStreamAreLoading] = useState(null); + + // This is kind of weird, but ensures that modals stay the right + // size while the closing animation is played. This is why modal + // width has its own state. This makes sure we always set the right + // size whenever displaying a modal. + function setShowModal(modal: TModalStreamView | null) { + rawSetShowModal(modal); + + if (modal && modal.type) { + setModalWidth( + { + trim: '60%', + preview: '700px', + export: '700px', + remove: '400px', + upload: '400px', + }[modal.type], + ); + } + } + + async function previewVideo(id: string) { + setClipsOfStreamAreLoading(id); + + try { + await HighlighterService.actions.return.loadClips(id); + setClipsOfStreamAreLoading(null); + rawSetShowModal({ type: 'preview', id }); + } catch (error: unknown) { + console.error('Error loading clips for preview export', error); + setClipsOfStreamAreLoading(null); + } + } + + async function exportVideo(id: string) { + setClipsOfStreamAreLoading(id); + + try { + await HighlighterService.actions.return.loadClips(id); + setClipsOfStreamAreLoading(null); + rawSetShowModal({ type: 'export', id }); + } catch (error: unknown) { + console.error('Error loading clips for export', error); + setClipsOfStreamAreLoading(null); + } + } + + function ImportStreamModal({ close }: { close: () => void }) { + const { HighlighterService } = Services; + const [inputValue, setInputValue] = useState(''); + + function handleInputChange(event: any) { + setInputValue(event.target.value); + } + + async function startAiDetection(title: string) { + const streamInfo: StreamInfoForAiHighlighter = { + id: 'manual_' + uuid(), + title, + game: 'Fortnite', + }; + + let filePath: string[] | undefined = []; + + try { + filePath = await importStreamFromDevice(); + if (filePath && filePath.length > 0) { + HighlighterService.actions.flow(filePath[0], streamInfo); + close(); + } else { + // No file selected + } + } catch (error: unknown) { + console.error('Error importing file from device', error); + } + } + + return ( + <> +
+
+

{$t('Import Fortnite stream')}

+ +
+
+ + +
+
+ + ); + } + + async function importStreamFromDevice() { + const selections = await remote.dialog.showOpenDialog(remote.getCurrentWindow(), { + properties: ['openFile'], + filters: [{ name: $t('Video Files'), extensions: SUPPORTED_FILE_TYPES }], + }); + + if (selections && selections.filePaths) { + return selections.filePaths; + } + } + + function closeModal() { + // Do not allow closing export modal while export/upload operations are in progress + if (v.exportInfo.exporting) return; + if (v.uploadInfo.uploading) return; + + setShowModal(null); + + if (v.error) HighlighterService.actions.dismissError(); + } + + function onDrop(e: React.DragEvent) { + const extensions = SUPPORTED_FILE_TYPES.map(e => `.${e}`); + const files: string[] = []; + let fi = e.dataTransfer.files.length; + while (fi--) { + const file = e.dataTransfer.files.item(fi)?.path; + if (file) files.push(file); + } + + const filtered = files.filter(f => extensions.includes(path.parse(f).ext)); + if (filtered.length) { + const StreamInfoForAiHighlighter: StreamInfoForAiHighlighter = { + id: 'manual_' + uuid(), + game: 'Fortnite', + }; + HighlighterService.actions.flow(filtered[0], StreamInfoForAiHighlighter); + } + + e.preventDefault(); + e.stopPropagation(); + } + + return ( +
onDrop(event)} + > +
+
+

My stream highlights

+
+
+
!aiDetectionInProgress && setShowModal({ type: 'upload' })} + > + + {$t('Select your Fornite recording')} + +
+ +
+
+ + + {highlightedStreams.length === 0 ? ( + <>No highlight clips created from streams // TODO: Add empty state + ) : ( + Object.entries(groupStreamsByTimePeriod(highlightedStreams)).map( + ([period, streams]) => + streams.length > 0 && ( + +
{period}
+
+ {streams.map(stream => ( + emitSetView(data)} + emitGeneratePreview={() => previewVideo(stream.id)} + emitExportVideo={() => exportVideo(stream.id)} + emitRemoveStream={() => setShowModal({ type: 'remove', id: stream.id })} + clipsOfStreamAreLoading={clipsOfStreamAreLoading} + emitCancelHighlightGeneration={() => { + HighlighterService.actions.cancelHighlightGeneration(stream.id); + }} + /> + ))} +
+ + ), + ) + )} + + + + {!!v.error && } + {showModal?.type === 'upload' && } + {showModal?.type === 'export' && } + {showModal?.type === 'preview' && ( + + )} + {showModal?.type === 'remove' && ( + + )} + +
+ ); +} + +function RemoveStream(p: { streamId: string | undefined; close: () => void }) { + const { HighlighterService } = Services; + + return ( +
+

{$t('Delete highlighted stream?')}

+

+ {$t( + 'Are you sure you want to delete this stream and all its associated clips? This action cannot be undone.', + )} +

+ + +
+ ); +} + +export function groupStreamsByTimePeriod(streams: { id: string; date: string }[]) { + const now = moment(); + const groups: { [key: string]: typeof streams } = { + Today: [], + Yesterday: [], + 'This week': [], + 'Last week': [], + 'This month': [], + 'Last month': [], + }; + const monthGroups: { [key: string]: typeof streams } = {}; + + streams.forEach(stream => { + const streamDate = moment(stream.date); + if (streamDate.isSame(now, 'day')) { + groups['Today'].push(stream); + } else if (streamDate.isSame(now.clone().subtract(1, 'day'), 'day')) { + groups['Yesterday'].push(stream); + } else if (streamDate.isSame(now, 'week')) { + groups['This week'].push(stream); + } else if (streamDate.isSame(now.clone().subtract(1, 'week'), 'week')) { + groups['Last week'].push(stream); + } else if (streamDate.isSame(now, 'month')) { + groups['This month'].push(stream); + } else if (streamDate.isSame(now.clone().subtract(1, 'month'), 'month')) { + groups['Last month'].push(stream); + } else { + const monthKey = streamDate.format('MMMM YYYY'); + if (!monthGroups[monthKey]) { + monthGroups[monthKey] = []; + } + monthGroups[monthKey].push(stream); + } + }); + + return { ...groups, ...monthGroups }; +} + +function FortniteIcon(): JSX.Element { + return ( + + + + + + + + + + + + + + + + + + + ); +} diff --git a/app/components-react/highlighter/UpdateModal.m.less b/app/components-react/highlighter/UpdateModal.m.less new file mode 100644 index 000000000000..bfae124209ad --- /dev/null +++ b/app/components-react/highlighter/UpdateModal.m.less @@ -0,0 +1,51 @@ +.overlay { + position: fixed; + top: 20; + left: 0; + width: 100%; + height: 100%; + background-color: rgba(23, 36, 45, 0.3); + backdrop-filter: blur(4px); + display: flex; + align-items: center; + justify-content: center; + z-index: 1000; +} + +.modal { + background-color: #09161d; + padding: 52px; + border-radius: 16px; + width: 400px; + text-align: center; +} + +.title { + font-size: 20px; + color: #ffffff; + font-style: normal; + font-weight: 500; + margin: 0 0 20px 0; +} + +.subtitle { + font-size: 14px; + color: #ffffff; + font-style: normal; + font-weight: 400; + margin: 0 0 36px 0; +} + +.progressBarContainer { + background-color: #2b383f; + border-radius: 8px; + overflow: hidden; + height: 12px; + margin: 20px 0; +} + +.progressBar { + background-color: #80f5d2; + height: 100%; + width: 0; +} diff --git a/app/components-react/highlighter/UpdateModal.tsx b/app/components-react/highlighter/UpdateModal.tsx new file mode 100644 index 000000000000..026448c85e47 --- /dev/null +++ b/app/components-react/highlighter/UpdateModal.tsx @@ -0,0 +1,36 @@ +import React from 'react'; +import styles from './UpdateModal.m.less'; + +export default function Modal({ + version, + progress, + isVisible, +}: { + version: string; + progress: number; + isVisible: boolean; +}) { + if (!isVisible) return null; + + let subtitle; + if (progress >= 100) { + subtitle =

Installing...

; + } else { + subtitle =

{Math.round(progress)}% complete

; + } + + return ( +
+
+

Downloading version {version}

+ {subtitle} +
+
+
+
+
+ ); +} diff --git a/app/components-react/pages/Highlighter.tsx b/app/components-react/pages/Highlighter.tsx index 02552861033c..e3007af82817 100644 --- a/app/components-react/pages/Highlighter.tsx +++ b/app/components-react/pages/Highlighter.tsx @@ -3,27 +3,60 @@ import { useVuex } from 'components-react/hooks'; import React, { useState } from 'react'; import { EHighlighterView, IViewState } from 'services/highlighter'; import { Services } from 'components-react/service-provider'; +import StreamView from 'components-react/highlighter/StreamView'; import ClipsView from 'components-react/highlighter/ClipsView'; +import UpdateModal from 'components-react/highlighter/UpdateModal'; +import { EAvailableFeatures } from 'services/incremental-rollout'; export default function Highlighter(props: { params?: { view: string } }) { - const openViewFromParams = props?.params?.view || ''; - - const { HighlighterService } = Services; + const { HighlighterService, IncrementalRolloutService } = Services; + const aiHighlighterEnabled = IncrementalRolloutService.views.featureIsEnabled( + EAvailableFeatures.aiHighlighter, + ); const v = useVuex(() => ({ - dismissedTutorial: HighlighterService.views.dismissedTutorial, - clips: HighlighterService.views.clips, + useAiHighlighter: HighlighterService.views.useAiHighlighter, + isUpdaterRunning: HighlighterService.views.isUpdaterRunning, + highlighterVersion: HighlighterService.views.highlighterVersion, + progress: HighlighterService.views.updaterProgress, + clipsAmount: HighlighterService.views.clips.length, + streamAmount: HighlighterService.views.highlightedStreams.length, })); - const [viewState, setViewState] = useState( - v.clips.length === 0 - ? { view: EHighlighterView.SETTINGS } - : { view: EHighlighterView.CLIPS, id: undefined }, + let initialViewState: IViewState; + + if (v.streamAmount > 0 && v.clipsAmount > 0 && aiHighlighterEnabled) { + initialViewState = { view: EHighlighterView.STREAM }; + } else if (v.clipsAmount > 0) { + initialViewState = { view: EHighlighterView.CLIPS, id: undefined }; + } else { + initialViewState = { view: EHighlighterView.SETTINGS }; + } + + const [viewState, setViewState] = useState(initialViewState); + const updaterModal = ( + ); switch (viewState.view) { + case EHighlighterView.STREAM: + return ( + <> + {aiHighlighterEnabled && updaterModal} + { + setViewFromEmit(data); + }} + /> + + ); case EHighlighterView.CLIPS: return ( <> + {aiHighlighterEnabled && updaterModal} { setViewFromEmit(data); @@ -40,6 +73,7 @@ export default function Highlighter(props: { params?: { view: string } }) { default: return ( <> + {aiHighlighterEnabled && updaterModal} { HighlighterService.actions.dismissTutorial(); diff --git a/app/components-react/pages/RecordingHistory.tsx b/app/components-react/pages/RecordingHistory.tsx index 88365c9cfd36..2549b0902dcf 100644 --- a/app/components-react/pages/RecordingHistory.tsx +++ b/app/components-react/pages/RecordingHistory.tsx @@ -13,8 +13,12 @@ import { Services } from '../service-provider'; import { initStore, useController } from '../hooks/zustand'; import { useVuex } from '../hooks'; import Translate from 'components-react/shared/Translate'; +import uuid from 'uuid/v4'; +import { EMenuItemKey } from 'services/side-nav'; import { $i } from 'services/utils'; import { IRecordingEntry } from 'services/recording-mode'; +import { EAiDetectionState, EHighlighterView } from 'services/highlighter'; +import { EAvailableFeatures } from 'services/incremental-rollout'; interface IRecordingHistoryStore { showSLIDModal: boolean; @@ -29,6 +33,9 @@ class RecordingHistoryController { private UserService = Services.UserService; private SharedStorageService = Services.SharedStorageService; private NotificationsService = Services.NotificationsService; + private HighlighterService = Services.HighlighterService; + private NavigationService = Services.NavigationService; + private IncrementalRolloutService = Services.IncrementalRolloutService; store = initStore({ showSLIDModal: false, showEditModal: false, @@ -51,8 +58,19 @@ class RecordingHistoryController { return this.RecordingModeService.state.uploadInfo; } + get aiDetectionInProgress() { + return this.HighlighterService.views.highlightedStreams.some( + stream => stream.state.type === EAiDetectionState.IN_PROGRESS, + ); + } + get uploadOptions() { const opts = [ + { + label: `${$t('Get highlights (Fortnite only))')}`, + value: 'highlighter', + icon: 'icon-highlighter', + }, { label: $t('Edit'), value: 'edit', @@ -113,6 +131,20 @@ class RecordingHistoryController { this.postError($t('Upload already in progress')); return; } + if (platform === 'highlighter') { + if (this.aiDetectionInProgress) return; + this.HighlighterService.actions.flow(recording.filename, { + game: 'forntnite', + id: 'rec_' + uuid(), + }); + this.NavigationService.actions.navigate( + 'Highlighter', + { view: EHighlighterView.STREAM }, + EMenuItemKey.Highlighter, + ); + return; + } + if (platform === 'youtube') return this.uploadToYoutube(recording.filename); if (platform === 'remove') return this.removeEntry(recording.timestamp); if (this.hasSLID) { @@ -168,8 +200,12 @@ export default function RecordingHistoryPage() { export function RecordingHistory() { const controller = useController(RecordingHistoryCtx); const { formattedTimestamp, showFile, handleSelect, postError } = controller; - const { uploadInfo, uploadOptions, recordings, hasSLID } = useVuex(() => ({ + const aiHighlighterEnabled = Services.IncrementalRolloutService.views.featureIsEnabled( + EAvailableFeatures.aiHighlighter, + ); + const { uploadInfo, uploadOptions, recordings, hasSLID, aiDetectionInProgress } = useVuex(() => ({ recordings: controller.recordings, + aiDetectionInProgress: controller.aiDetectionInProgress, uploadOptions: controller.uploadOptions, uploadInfo: controller.uploadInfo, hasSLID: controller.hasSLID, @@ -193,18 +229,32 @@ export function RecordingHistory() { function UploadActions(p: { recording: IRecordingEntry }) { return ( - {uploadOptions.map(opt => ( - handleSelect(p.recording, opt.value)} - > - -   - {opt.label} - - ))} + {uploadOptions + .map(option => { + if (option.value === 'highlighter' && !aiHighlighterEnabled) { + return null; + } + return ( + handleSelect(p.recording, option.value)} + > + +   + {option.label} + + ); + }) + .filter(Boolean)} ); } diff --git a/app/components-react/sidebar/FeaturesNav.tsx b/app/components-react/sidebar/FeaturesNav.tsx index 964efafd97ba..803ef1a87349 100644 --- a/app/components-react/sidebar/FeaturesNav.tsx +++ b/app/components-react/sidebar/FeaturesNav.tsx @@ -260,7 +260,15 @@ function FeaturesNavItem(p: { handleNavigation: (menuItem: IMenuItem, key?: string) => void; className?: string; }) { - const { SideNavService, TransitionsService, DualOutputService } = Services; + const { + SideNavService, + TransitionsService, + DualOutputService, + IncrementalRolloutService, + } = Services; + const aiHighlighterEnabled = IncrementalRolloutService.views.featureIsEnabled( + EAvailableFeatures.aiHighlighter, + ); const { isSubMenuItem, menuItem, handleNavigation, className } = p; const { currentMenuItem, isOpen, studioMode, dualOutputMode } = useVuex(() => ({ @@ -312,7 +320,14 @@ function FeaturesNavItem(p: { } }} > - {title} +
+ {title} + {menuItem.key === EMenuItemKey.Highlighter && aiHighlighterEnabled && ( +
+

beta

+
+ )} +
); } diff --git a/app/components-react/sidebar/SideNav.m.less b/app/components-react/sidebar/SideNav.m.less index 3fa209dc7a92..ce81f89a9884 100644 --- a/app/components-react/sidebar/SideNav.m.less +++ b/app/components-react/sidebar/SideNav.m.less @@ -248,3 +248,11 @@ button.sidenav-button { padding: 4px 16px; } } + +.beta-tag { + border-radius: 4px; + background-color: var(--button); + color: var(--paragraph); + padding: 2px 6px; + margin-left: 8px; +} diff --git a/app/components-react/windows/go-live/AiHighlighterToggle.m.less b/app/components-react/windows/go-live/AiHighlighterToggle.m.less new file mode 100644 index 000000000000..6e2a61e538d3 --- /dev/null +++ b/app/components-react/windows/go-live/AiHighlighterToggle.m.less @@ -0,0 +1,58 @@ +@import '../../../styles/index'; + +.ai-highlighter-box { + background-color: #000; + display: flex; + flex-direction: column; + max-width: 66.666666667%; + justify-content: space-between; + width: 100%; + border: 2px solid var(--Day-Colors-Dark-Blue, #2b5bd7); + padding: 16px 20px 16px 20px; + align-items: center; + border-radius: 16px; +} + +.headline-wrapper { + cursor: pointer; + display: flex; + justify-content: space-between; + width: 100%; + align-items: center; +} + +.image { + width: 182px; + height: 187px; + display: grid; + place-content: center; + background-image: url(https://slobs-cdn.streamlabs.com/media/highlighter-image.png); + background-position: center; + background-size: contain; + background-repeat: no-repeat; +} +.toggle-text-wrapper { + display: flex; + flex-grow: 1; + flex-direction: column; + justify-content: space-between; +} +.beta-tag { + padding: 4px; + border-radius: 4px; + padding-top: 1px; + padding-bottom: 1px; + padding-left: 6px; + padding-right: 6px; + width: fit-content; + margin-bottom: 8px; + background-color: #2b5bd7; +} + +.expanded-wrapper { + display: flex; + justify-content: space-between; + gap: 16px; + margin-top: 16px; + width: 100%; +} diff --git a/app/components-react/windows/go-live/AiHighlighterToggle.tsx b/app/components-react/windows/go-live/AiHighlighterToggle.tsx new file mode 100644 index 000000000000..b763534cda5a --- /dev/null +++ b/app/components-react/windows/go-live/AiHighlighterToggle.tsx @@ -0,0 +1,101 @@ +import { SwitchInput } from 'components-react/shared/inputs/SwitchInput'; +import React, { useEffect, useRef, useState } from 'react'; +import styles from './AiHighlighterToggle.m.less'; + +import { Services } from 'components-react/service-provider'; +import Highlighter from 'components-react/pages/Highlighter'; +import { useVuex } from 'components-react/hooks'; +import { DownOutlined, UpOutlined } from '@ant-design/icons'; + +export default function AiHighlighterToggle({ + game, + cardIsExpanded, +}: { + game: string | undefined; + cardIsExpanded: boolean; +}) { + //TODO M: Probably good way to integrate the highlighter in to GoLiveSettings + const { HighlighterService } = Services; + const useHighlighter = useVuex(() => HighlighterService.views.useAiHighlighter); + + function getInitialExpandedState() { + if (game === 'Fortnite') { + return true; + } else { + if (useHighlighter) { + return true; + } else { + return cardIsExpanded; + } + } + } + const initialExpandedState = getInitialExpandedState(); + const [isExpanded, setIsExpanded] = useState(initialExpandedState); + + useEffect(() => { + if (game === 'Fortnite') { + setIsExpanded(true); + } + if (game !== 'Fortnite' && game !== undefined && useHighlighter) { + HighlighterService.actions.setAiHighlighter(false); + } + }, [game]); + + return ( +
+ {game === undefined || game === 'Fortnite' ? ( +
+
+ +
+
setIsExpanded(!isExpanded)}> +

+ Streaming Fortnite? Try AI Highlighter! +

+ {isExpanded ? ( + + ) : ( + + )} +
+ {isExpanded ? ( + <> +
+
+
+

+ Auto-create +
highlights +

+
Beta
+
+ HighlighterService.actions.toggleAiHighlighter()} + /> +
+
+
+ + ) : ( + <> + )} +
+
+ ) : ( + <> + )} +
+ ); +} diff --git a/app/components-react/windows/go-live/CommonPlatformFields.tsx b/app/components-react/windows/go-live/CommonPlatformFields.tsx index aee5eac88d9b..d76d2d4f014a 100644 --- a/app/components-react/windows/go-live/CommonPlatformFields.tsx +++ b/app/components-react/windows/go-live/CommonPlatformFields.tsx @@ -7,6 +7,8 @@ import InputWrapper from '../../shared/inputs/InputWrapper'; import Animate from 'rc-animate'; import { TLayoutMode } from './platforms/PlatformSettingsLayout'; import { Services } from '../../service-provider'; +import AiHighlighterToggle from './AiHighlighterToggle'; +import { EAvailableFeatures } from 'services/incremental-rollout'; interface ICommonPlatformSettings { title: string; @@ -22,6 +24,7 @@ interface IProps { layoutMode?: TLayoutMode; value: ICommonPlatformSettings; descriptionIsRequired?: boolean; + enabledPlatforms?: TPlatform[]; onChange: (newValue: ICommonPlatformSettings) => unknown; } @@ -55,6 +58,9 @@ export const CommonPlatformFields = InputComponent((rawProps: IProps) => { } const view = Services.StreamingService.views; + const aiHighlighterEnabled = Services.IncrementalRolloutService.views.featureIsEnabled( + EAvailableFeatures.aiHighlighter, + ); const hasCustomCheckbox = p.layoutMode === 'multiplatformAdvanced'; const fieldsAreVisible = !hasCustomCheckbox || p.value.useCustomFields; const descriptionIsRequired = @@ -125,6 +131,10 @@ export const CommonPlatformFields = InputComponent((rawProps: IProps) => { required={descriptionIsRequired} /> )} + + {aiHighlighterEnabled && enabledPlatforms && !enabledPlatforms.includes('twitch') && ( + + )}
)} diff --git a/app/components-react/windows/go-live/PlatformSettings.tsx b/app/components-react/windows/go-live/PlatformSettings.tsx index ee6da64d6ac5..aa0cefef76db 100644 --- a/app/components-react/windows/go-live/PlatformSettings.tsx +++ b/app/components-react/windows/go-live/PlatformSettings.tsx @@ -80,6 +80,7 @@ export default function PlatformSettings() { descriptionIsRequired={descriptionIsRequired} value={commonFields} onChange={updateCommonFields} + enabledPlatforms={enabledPlatforms} /> )} diff --git a/app/components-react/windows/go-live/platforms/TwitchEditStreamInfo.tsx b/app/components-react/windows/go-live/platforms/TwitchEditStreamInfo.tsx index ae2b94eca6b4..d3c5ddff9d26 100644 --- a/app/components-react/windows/go-live/platforms/TwitchEditStreamInfo.tsx +++ b/app/components-react/windows/go-live/platforms/TwitchEditStreamInfo.tsx @@ -12,10 +12,15 @@ import Message from '../../../shared/Message'; import { Row, Col, Select } from 'antd'; import { IListOption } from 'components/shared/inputs'; import TwitchContentClassificationInput from './TwitchContentClassificationInput'; +import AiHighlighterToggle from '../AiHighlighterToggle'; +import { Services } from 'components-react/service-provider'; +import { EAvailableFeatures } from 'services/incremental-rollout'; export function TwitchEditStreamInfo(p: IPlatformComponentParams<'twitch'>) { const twSettings = p.value; - + const aiHighlighterEnabled = Services.IncrementalRolloutService.views.featureIsEnabled( + EAvailableFeatures.aiHighlighter, + ); function updateSettings(patch: Partial) { p.onChange({ ...twSettings, ...patch }); } @@ -44,7 +49,14 @@ export function TwitchEditStreamInfo(p: IPlatformComponentParams<'twitch'>) { onChange={updateSettings} /> } - requiredFields={} + requiredFields={ + + + {aiHighlighterEnabled && ( + + )} + + } optionalFields={optionalFields} /> diff --git a/app/i18n/en-US/highlighter.json b/app/i18n/en-US/highlighter.json index 92bdb190184c..b9660aaac5ec 100644 --- a/app/i18n/en-US/highlighter.json +++ b/app/i18n/en-US/highlighter.json @@ -103,7 +103,7 @@ "Record your screen with Streamlabs Desktop. Once recording is complete, it will be displayed here. Access your files or edit further with Streamlabs tools.": "Record your screen with Streamlabs Desktop. Once recording is complete, it will be displayed here. Access your files or edit further with Streamlabs tools.", "Recordings": "Recordings", "Manual Highlighter": "Manual Highlighter", - "The hotkey highlighter allows you to clip the best moments during your livestream manually and edit them together afterwards.": "The hotkey highlighter allows you to clip the best moments during your livestream manually and edit them together afterwards.", + "Manually capture the best moments from your livestream with a hotkey command, and automatically turn them into a highlight video.": "Manually capture the best moments from your livestream with a hotkey command, and automatically turn them into a highlight video.", "End your stream to change the Hotkey or the replay duration.": "End your stream to change the Hotkey or the replay duration.", "No clips found": "No clips found", "All highlight clips": "All highlight clips", @@ -111,5 +111,34 @@ "Trim": "Trim", "Intro": "Intro", "Outro": "Outro", - "All clips": "All clips" + "All Clips": "All Clips", + "Export highlight reel": "Export highlight reel", + "Restart": "Restart", + "Add Clips": "Add Clips", + "Edit Clips": "Edit Clips", + "Searching for highlights...": "Searching for highlights...", + "Not enough highlights found": "Not enough highlights found", + "Highlights cancelled": "Highlights cancelled", + "Highlights failed": "Highlights failed", + "Import Fortnite stream": "Import Fortnite stream", + "Select video to start import": "Select video to start import", + "Select your Fornite recording": "Select your Fornite recording", + "Settings": "Settings", + "Delete highlighted stream?": "Delete highlighted stream?", + "Are you sure you want to delete this stream and all its associated clips? This action cannot be undone.": "Are you sure you want to delete this stream and all its associated clips? This action cannot be undone.", + "Set a title for your stream": "Set a title for your stream", + "Create highlight video of": "Create highlight video of", + "All rounds": "All rounds", + "unlimited": "unlimited", + "%{duration} minutes": "%{duration} minutes", + "%{duration} minute": "%{duration} minute", + "with a duration of": "with a duration of", + "AI Highlighter": "AI Highlighter", + "For Fortnite streams (Beta)": "For Fortnite streams (Beta)", + "Automatically capture the best moments from your livestream and turn them into a highlight video.": "Automatically capture the best moments from your livestream and turn them into a highlight video.", + "Recommended": "Recommended", + "Stream Highlights": "Stream Highlights", + "Export Vertical": "Export Vertical", + "Export Horizontal": "Export Horizontal", + "Get highlights (Fortnite only))": "Get highlights (Fortnite only))" } \ No newline at end of file diff --git a/app/services/highlighter/ai-highlighter/ai-highlighter.ts b/app/services/highlighter/ai-highlighter/ai-highlighter.ts index 6164dc16874b..467fb919c3ff 100644 --- a/app/services/highlighter/ai-highlighter/ai-highlighter.ts +++ b/app/services/highlighter/ai-highlighter/ai-highlighter.ts @@ -1,6 +1,9 @@ import * as child from 'child_process'; import EventEmitter from 'events'; +import { AiHighlighterUpdater } from './updater'; import { duration } from 'moment'; +import { ICoordinates } from '..'; +import kill from 'tree-kill'; export enum EHighlighterInputTypes { KILL = 'kill', @@ -16,7 +19,6 @@ export enum EHighlighterInputTypes { LOW_HEALTH = 'low_health', PLAYER_KNOCKED = 'player_knocked', } - export type DeathMetadata = { place: number; }; @@ -33,7 +35,7 @@ export interface IHighlight { input_types: EHighlighterInputTypes[]; inputs: IHighlighterInput[]; score: number; - metadata: { round: number }; + metadata: { round: number; webcam_coordinates: ICoordinates }; } export type EHighlighterMessageTypes = @@ -41,9 +43,225 @@ export type EHighlighterMessageTypes = | 'inputs' | 'inputs_partial' | 'highlights' - | 'highlights_partial'; + | 'milestone'; export interface IHighlighterMessage { type: EHighlighterMessageTypes; json: {}; } +interface IHighlighterProgressMessage { + progress: number; +} + +export interface IHighlighterMilestone { + name: string; + weight: number; + data: IHighlighterMessage[] | null; +} + +const START_TOKEN = '>>>>'; +const END_TOKEN = '<<<<'; + +// Buffer management class to handle split messages +class MessageBufferHandler { + private buffer: string = ''; + + hasCompleteMessage(): boolean { + const hasStart = this.buffer.includes(START_TOKEN); + const hasEnd = this.buffer.includes(END_TOKEN); + return hasStart && hasEnd; + } + + isMessageComplete(message: string): boolean { + const combined = this.buffer + message; + const hasStart = combined.includes(START_TOKEN); + const hasEnd = combined.includes(END_TOKEN); + return hasStart && hasEnd; + } + + appendToBuffer(message: string) { + this.buffer += message; + } + + extractCompleteMessages(): string[] { + const messages = []; + while (this.hasCompleteMessage()) { + const start = this.buffer.indexOf(START_TOKEN); + const end = this.buffer.indexOf(END_TOKEN); + + if (start !== -1 && end !== -1 && start < end) { + const completeMessage = this.buffer.substring(start, end + END_TOKEN.length); + // Clear the buffer of the extracted message + this.buffer = this.buffer.substring(end + END_TOKEN.length); + messages.push(completeMessage); + } else { + // Message not complete + } + } + return messages; + } + + clear() { + this.buffer = ''; + } +} + +export function getHighlightClips( + videoUri: string, + renderHighlights: (highlightClips: IHighlight[]) => void, + cancelSignal: AbortSignal, + progressUpdate?: (progress: number) => void, + milestonesPath?: string, + milestoneUpdate?: (milestone: IHighlighterMilestone) => void, +): Promise { + return new Promise((resolve, reject) => { + console.log(`Get highlight clips for ${videoUri}`); + + const partialInputsRendered = false; + console.log('Start Ai analysis'); + + const childProcess: child.ChildProcess = AiHighlighterUpdater.startHighlighterProcess( + videoUri, + milestonesPath, + ); + const messageBuffer = new MessageBufferHandler(); + + if (cancelSignal) { + cancelSignal.addEventListener('abort', () => { + console.log('ending highlighter process'); + messageBuffer.clear(); + kill(childProcess.pid!, 'SIGINT'); + reject(new Error('Highlight generation canceled')); + }); + } + + childProcess.stdout?.on('data', (data: Buffer) => { + const message = data.toString(); + messageBuffer.appendToBuffer(message); + + // Try to extract a complete message + const completeMessages = messageBuffer.extractCompleteMessages(); + + for (const completeMessage of completeMessages) { + // messageBuffer.clear(); + const aiHighlighterMessage = parseAiHighlighterMessage(completeMessage); + if (typeof aiHighlighterMessage === 'string' || aiHighlighterMessage instanceof String) { + console.log('message type of string', aiHighlighterMessage); + } else if (aiHighlighterMessage) { + switch (aiHighlighterMessage.type) { + case 'progress': + progressUpdate?.((aiHighlighterMessage.json as IHighlighterProgressMessage).progress); + break; + case 'highlights': + if (!partialInputsRendered) { + console.log('call Render highlights:'); + renderHighlights?.(aiHighlighterMessage.json as IHighlight[]); + } + resolve(aiHighlighterMessage.json as IHighlight[]); + break; + case 'milestone': + milestoneUpdate?.(aiHighlighterMessage.json as IHighlighterMilestone); + break; + default: + // console.log('\n\n'); + // console.log('Unrecognized message type:', aiHighlighterMessage); + // console.log('\n\n'); + break; + } + } + } + }); + + childProcess.stderr?.on('data', (data: Buffer) => { + console.log('Debug logs:', data.toString()); + }); + + childProcess.on('error', error => { + messageBuffer.clear(); + reject(new Error(`Child process threw an error. Error message: ${error.message}.`)); + }); + + childProcess.on('exit', (code, signal) => { + messageBuffer.clear(); + reject(new Error(`Child process exited with code ${code} and signal ${signal}`)); + }); + }); +} + +function parseAiHighlighterMessage(messageString: string): IHighlighterMessage | string | null { + try { + if (messageString.includes(START_TOKEN) && messageString.includes(END_TOKEN)) { + const start = messageString.indexOf(START_TOKEN); + const end = messageString.indexOf(END_TOKEN); + const jsonString = messageString.substring(start, end).replace(START_TOKEN, ''); + // console.log('Json string:', jsonString); + + const aiHighlighterMessage = JSON.parse(jsonString) as IHighlighterMessage; + // console.log('Parsed ai highlighter message:', aiHighlighterMessage); + return aiHighlighterMessage; + } else { + return messageString; + } + } catch (error: unknown) { + console.log('Error parsing ai highlighter message:', error); + return null; + } +} + +export class ProgressTracker { + PRE_DURATION = 10; + POST_DURATION = 10; + progress = 0; + + onChangeCallback: (progress: number) => void; + + preInterval: NodeJS.Timeout; + postInterval: NodeJS.Timeout; + postStarted = false; + constructor(onChange = (progress: number) => {}) { + this.startPreTimer(); + this.onChangeCallback = onChange; + } + + startPreTimer() { + this.progress = 0; + this.preInterval = this.addOnePerSecond(this.PRE_DURATION); + } + + startPostTimer() { + if (!this.postStarted) { + this.postInterval = this.addOnePerSecond(this.POST_DURATION); + this.postStarted = true; + } + } + destroy() { + this.preInterval && clearInterval(this.preInterval); + this.postInterval && clearInterval(this.postInterval); + } + + updateProgressFromHighlighter(highlighterProgress: number) { + this.preInterval && clearInterval(this.preInterval); + const adjustedProgress = + highlighterProgress * ((100 - this.PRE_DURATION - this.POST_DURATION) / 100) + + this.PRE_DURATION; + + this.progress = adjustedProgress; + this.onChangeCallback(this.progress); + if (highlighterProgress === 100) { + this.startPostTimer(); + } + } + + addOnePerSecond(duration: number) { + let passedSeconds = 0; + const interval = setInterval(() => { + passedSeconds += 1; + this.progress += 1; + this.onChangeCallback(this.progress); + if (passedSeconds >= duration) { + clearInterval(interval); + } + }, 1000); + return interval; + } +} diff --git a/app/services/highlighter/ai-highlighter/updater.ts b/app/services/highlighter/ai-highlighter/updater.ts new file mode 100644 index 000000000000..f69fe0b2a217 --- /dev/null +++ b/app/services/highlighter/ai-highlighter/updater.ts @@ -0,0 +1,239 @@ +import { promises as fs, createReadStream, existsSync } from 'fs'; +import path from 'path'; +import { getSharedResource } from 'util/get-shared-resource'; +import { downloadFile, IDownloadProgress, jfetch } from 'util/requests'; +import crypto from 'crypto'; +import { pipeline } from 'stream/promises'; +import { importExtractZip } from 'util/slow-imports'; +import { spawn } from 'child_process'; +import { FFMPEG_EXE } from '../constants'; +import Utils from '../../utils'; + +interface IAIHighlighterManifest { + version: string; + platform: string; + url: string; + size: number; + checksum: string; + timestamp: number; +} + +/** + * Checks for updates to the AI Highlighter and updates the local installation + * if necessary. + * + * Responsible for storing the manifest and updating the highlighter binary, and maintains + * the paths to the highlighter binary and manifest. + */ +export class AiHighlighterUpdater { + private basepath: string; + private manifestPath: string; + private manifest: IAIHighlighterManifest | null; + private isCurrentlyUpdating: boolean = false; + private versionChecked: boolean = false; + + public currentUpdate: Promise | null = null; + + constructor() { + this.basepath = getSharedResource('ai-highlighter'); + this.manifestPath = path.resolve(this.basepath, 'manifest.json'); + } + + /** + * Spawn the AI Highlighter process that would process the video + */ + static startHighlighterProcess(videoUri: string, milestonesPath?: string) { + const isDev = Utils.isDevMode(); + if (isDev) { + const rootPath = '../highlighter-api/'; + const command = [ + 'run', + 'python', + `${rootPath}/highlighter_api/cli.py`, + videoUri, + '--ffmpeg_path', + FFMPEG_EXE, + '--loglevel', + 'debug', + ]; + + if (milestonesPath) { + command.push('--milestones_file'); + command.push(milestonesPath); + } + + return spawn('poetry', command, { + cwd: rootPath, + }); + } + + const highlighterBinaryPath = path.resolve( + getSharedResource('ai-highlighter'), + 'bin', + 'app.exe', + ); + + const command = [videoUri, '--ffmpeg_path', FFMPEG_EXE]; + if (milestonesPath) { + command.push('--milestones_file'); + command.push(milestonesPath); + } + + return spawn(highlighterBinaryPath, command); + } + + /** + * Check if an update is currently in progress + */ + public get updateInProgress(): boolean { + return this.isCurrentlyUpdating; + } + + /** + * Get version that is about to be installed + */ + public get version(): string | null { + return this.manifest?.version || null; + } + + /** + * Check if AI Highlighter requires an update + */ + public async isNewVersionAvailable(): Promise { + // check if updater checked version in current session already + if (this.versionChecked) { + return false; + } + + this.versionChecked = true; + console.log('checking for highlighter updates...'); + // fetch the latest version of the manifest for win x86_64 target + const newManifest = await jfetch( + new Request('https://cdn-highlighter-builds.streamlabs.com/manifest_win_x86_64.json'), + ); + this.manifest = newManifest; + + // if manifest.json does not exist, an initial download is required + if (!existsSync(this.manifestPath)) { + console.log('manifest.json not found, initial download required'); + return true; + } + + // read the current manifest + const currentManifest = JSON.parse( + await fs.readFile(this.manifestPath, 'utf-8'), + ) as IAIHighlighterManifest; + + if (newManifest.version !== currentManifest.version) { + console.log( + `new highlighter version available. ${currentManifest.version} -> ${newManifest.version}`, + ); + return true; + } + + console.log('highlighter is up to date'); + return false; + } + + /** + * Update highlighter to the latest version + */ + public async update(progressCallback?: (progress: IDownloadProgress) => void): Promise { + // if (Utils.isDevMode()) { + // console.log('skipping update in dev mode'); + // return; + // } + try { + this.isCurrentlyUpdating = true; + this.currentUpdate = this.performUpdate(progressCallback); + await this.currentUpdate; + } finally { + this.isCurrentlyUpdating = false; + } + } + + private async performUpdate(progressCallback: (progress: IDownloadProgress) => void) { + if (!this.manifest) { + throw new Error('Manifest not found, cannot update'); + } + + if (!existsSync(this.basepath)) { + await fs.mkdir(this.basepath); + } + + const zipPath = path.resolve(this.basepath, 'ai-highlighter.zip'); + console.log('downloading new version of AI Highlighter...'); + + // in case if some leftover zip file exists for incomplete update + if (existsSync(zipPath)) { + await fs.rm(zipPath); + } + + // download the new version + await downloadFile(this.manifest.url, zipPath, progressCallback); + console.log('download complete'); + + // verify the checksum + const checksum = await this.sha256(zipPath); + if (checksum !== this.manifest.checksum) { + throw new Error('Checksum verification failed'); + } + + console.log('unzipping archive...'); + const unzipPath = path.resolve(this.basepath, 'bin-' + this.manifest.version); + // delete leftover unzipped files in case something happened before + if (existsSync(unzipPath)) { + await fs.rm(unzipPath, { recursive: true }); + } + + // unzip archive and delete the zip after + await this.unzip(zipPath, unzipPath); + await fs.rm(zipPath); + console.log('unzip complete'); + + // swap with the new version + const binPath = path.resolve(this.basepath, 'bin'); + const outdateVersionPresent = existsSync(binPath); + + // backup the ouotdated version in case something goes bad + if (outdateVersionPresent) { + console.log('backing up outdated version...'); + await fs.rename(binPath, path.resolve(this.basepath, 'bin.bkp')); + } + console.log('swapping new version...'); + await fs.rename(unzipPath, binPath); + + // cleanup + console.log('cleaning up...'); + if (outdateVersionPresent) { + await fs.rm(path.resolve(this.basepath, 'bin.bkp'), { recursive: true }); + } + + console.log('updating manifest...'); + await fs.writeFile(this.manifestPath, JSON.stringify(this.manifest)); + console.log('update complete'); + } + + private async sha256(file: string): Promise { + const hash = crypto.createHash('sha256'); + const stream = createReadStream(file); + + await pipeline(stream, hash); + + return hash.digest('hex'); + } + + private async unzip(zipPath: string, unzipPath: string): Promise { + // extract the new version + const extractZip = (await importExtractZip()).default; + return new Promise((resolve, reject) => { + extractZip(zipPath, { dir: unzipPath }, err => { + if (err) { + reject(err); + } else { + resolve(); + } + }); + }); + } +} diff --git a/app/services/highlighter/audio-crossfader.ts b/app/services/highlighter/audio-crossfader.ts index 5943b8689e21..e9fb705a67be 100644 --- a/app/services/highlighter/audio-crossfader.ts +++ b/app/services/highlighter/audio-crossfader.ts @@ -1,13 +1,13 @@ import execa from 'execa'; import fs from 'fs'; import { FFMPEG_EXE } from './constants'; -import { Clip } from './clip'; +import { RenderingClip } from './clip'; import { AudioMixError } from './errors'; export class AudioCrossfader { constructor( public readonly outputPath: string, - public readonly clips: Clip[], + public readonly clips: RenderingClip[], public readonly transitionDuration: number, ) {} diff --git a/app/services/highlighter/clip.ts b/app/services/highlighter/clip.ts index b419e02c7739..abc77a7f428b 100644 --- a/app/services/highlighter/clip.ts +++ b/app/services/highlighter/clip.ts @@ -6,7 +6,7 @@ import fs from 'fs'; import { IExportOptions } from '.'; import path from 'path'; -export class Clip { +export class RenderingClip { frameSource: FrameSource; audioSource: AudioSource; diff --git a/app/services/highlighter/frame-writer.ts b/app/services/highlighter/frame-writer.ts index f540c4fc3292..0f62b2da13e3 100644 --- a/app/services/highlighter/frame-writer.ts +++ b/app/services/highlighter/frame-writer.ts @@ -19,39 +19,65 @@ export class FrameWriter { /* eslint-disable */ const args = [ // Video Input - '-f', 'rawvideo', - '-vcodec', 'rawvideo', - '-pix_fmt', 'rgba', - '-s', `${this.options.width}x${this.options.height}`, - '-r', `${this.options.fps}`, - '-i', '-', + '-f', + 'rawvideo', + '-vcodec', + 'rawvideo', + '-pix_fmt', + 'rgba', + '-s', + `${this.options.width}x${this.options.height}`, + '-r', + `${this.options.fps}`, + '-i', + '-', // Audio Input - '-i', this.audioInput, + '-i', + this.audioInput, // Input Mapping - '-map', '0:v:0', - '-map', '1:a:0', + '-map', + '0:v:0', + '-map', + '1:a:0', // Filters - '-af', `afade=type=out:duration=${FADE_OUT_DURATION}:start_time=${Math.max(this.duration - (FADE_OUT_DURATION + 0.2), 0)}`, - '-vf', `format=yuv420p,fade=type=out:duration=${FADE_OUT_DURATION}:start_time=${Math.max(this.duration - (FADE_OUT_DURATION + 0.2), 0)}`, - - // Video Output - '-vcodec', 'libx264', - '-profile:v', 'high', - '-preset:v', this.options.preset, - '-crf', '18', - '-movflags', 'faststart', + '-af', + `afade=type=out:duration=${FADE_OUT_DURATION}:start_time=${Math.max( + this.duration - (FADE_OUT_DURATION + 0.2), + 0, + )}`, + ]; - // Audio Output - '-acodec', 'aac', - '-b:a', '128k', + this.addVideoFilters(args); + + args.push( + ...[ + // Video Output + '-vcodec', + 'libx264', + '-profile:v', + 'high', + '-preset:v', + this.options.preset, + '-crf', + '18', + '-movflags', + 'faststart', + + // Audio Output + '-acodec', + 'aac', + '-b:a', + '128k', + + '-y', + this.outputPath, + ], + ); - '-y', this.outputPath, - ]; /* eslint-enable */ - this.ffmpeg = execa(FFMPEG_EXE, args, { encoding: null, buffer: false, @@ -76,6 +102,18 @@ export class FrameWriter { }); } + private addVideoFilters(args: string[]) { + const fadeFilter = `format=yuv420p,fade=type=out:duration=${FADE_OUT_DURATION}:start_time=${Math.max( + this.duration - (FADE_OUT_DURATION + 0.2), + 0, + )}`; + if (this.options.complexFilter) { + args.push('-vf', this.options.complexFilter + `[final]${fadeFilter}`); + } else { + args.push('-vf', fadeFilter); + } + } + async writeNextFrame(frameBuffer: Buffer) { if (!this.ffmpeg) this.startFfmpeg(); diff --git a/app/services/highlighter/index.ts b/app/services/highlighter/index.ts index c992a37e6b18..0856e5854232 100644 --- a/app/services/highlighter/index.ts +++ b/app/services/highlighter/index.ts @@ -20,9 +20,16 @@ import { } from 'services/platforms/youtube/uploader'; import { YoutubeService } from 'services/platforms/youtube'; import os from 'os'; -import { CLIP_DIR, SCRUB_SPRITE_DIRECTORY, SUPPORTED_FILE_TYPES, TEST_MODE } from './constants'; +import { + CLIP_DIR, + FFMPEG_EXE, + SCRUB_SPRITE_DIRECTORY, + SUPPORTED_FILE_TYPES, + TEST_MODE, + FFPROBE_EXE, +} from './constants'; import { pmap } from 'util/pmap'; -import { Clip } from './clip'; +import { RenderingClip } from './clip'; import { AudioCrossfader } from './audio-crossfader'; import { FrameWriter } from './frame-writer'; import { Transitioner } from './transitioner'; @@ -38,7 +45,23 @@ import { ENotificationType, NotificationsService } from 'services/notifications' import { JsonrpcService } from 'services/api/jsonrpc'; import { NavigationService } from 'services/navigation'; import { SharedStorageService } from 'services/integrations/shared-storage'; -import { EHighlighterInputTypes } from './ai-highlighter/ai-highlighter'; +import execa from 'execa'; +import moment from 'moment'; +import { + EHighlighterInputTypes, + getHighlightClips, + IHighlight, + IHighlighterInput, + IHighlighterMilestone, + ProgressTracker, +} from './ai-highlighter/ai-highlighter'; +import uuid from 'uuid'; +import { EMenuItemKey } from 'services/side-nav'; +import { AiHighlighterUpdater } from './ai-highlighter/updater'; +import { IDownloadProgress } from 'util/requests'; +import { IncrementalRolloutService } from 'app-services'; +import { EAvailableFeatures } from 'services/incremental-rollout'; +import { getSharedResource } from 'util/get-shared-resource'; export type TStreamInfo = | { orderPosition: number; @@ -49,6 +72,15 @@ export type TStreamInfo = const isAiClip = (clip: TClip): clip is IAiClip => clip.source === 'AiClip'; +// types for highlighter video operations +export type TOrientation = 'horizontal' | 'vertical'; +export interface ICoordinates { + x1: number; + y1: number; + x2: number; + y2: number; +} + interface IBaseClip { path: string; loaded: boolean; @@ -93,13 +125,17 @@ export interface IInput { export interface IAiClipInfo { inputs: IInput[]; score: number; - metadata: { round: number }; + metadata: { + round: number; + webcam_coordinates: ICoordinates; + }; } export type TClip = IAiClip | IReplayBufferClip | IManualClip; export enum EHighlighterView { CLIPS = 'clips', + STREAM = 'stream', SETTINGS = 'settings', } @@ -107,18 +143,27 @@ interface TClipsViewState { view: EHighlighterView.CLIPS; id: string | undefined; } +interface IStreamViewState { + view: EHighlighterView.STREAM; +} interface ISettingsViewState { view: EHighlighterView.SETTINGS; } -export type IViewState = TClipsViewState | ISettingsViewState; +export type IViewState = TClipsViewState | IStreamViewState | ISettingsViewState; + +export interface StreamMilestones { + streamId: string; + milestones: IHighlighterMilestone[]; +} // TODO: Need to clean up all of this export interface StreamInfoForAiHighlighter { id: string; game: string; title?: string; + milestonesPath?: string; } export interface INewClipData { @@ -129,18 +174,22 @@ export interface INewClipData { startTrim: number; endTrim: number; } + +export enum EAiDetectionState { + INITIALIZED = 'initialized', + IN_PROGRESS = 'detection-in-progress', + ERROR = 'error', + FINISHED = 'detection-finished', + CANCELED_BY_USER = 'detection-canceled-by-user', +} + export interface IHighlightedStream { id: string; game: string; title: string; date: string; state: { - type: - | 'initialized' - | 'detection-in-progress' - | 'error' - | 'detection-finished' - | 'detection-canceled-by-user'; + type: EAiDetectionState; progress: number; }; abortController?: AbortController; @@ -212,7 +261,7 @@ export interface IVideoInfo { outro: IOutroInfo; } -interface IHighligherState { +interface IHighlighterState { clips: Dictionary; transition: ITransitionInfo; video: IVideoInfo; @@ -221,7 +270,11 @@ interface IHighligherState { upload: IUploadInfo; dismissedTutorial: boolean; error: string; + useAiHighlighter: boolean; highlightedStreams: IHighlightedStream[]; + updaterProgress: number; + isUpdaterRunning: boolean; + highlighterVersion: string; } // Capitalization is not consistent because it matches with the @@ -338,9 +391,10 @@ export interface IExportOptions { width: number; height: number; preset: TPreset; + complexFilter?: string; } -class HighligherViews extends ViewHandler { +class HighlighterViews extends ViewHandler { /** * Returns an array of clips */ @@ -351,6 +405,16 @@ class HighligherViews extends ViewHandler { return this.state.clips; } + /** + * Returns wether or not the AiHighlighter should be used + */ + get useAiHighlighter() { + return this.state.useAiHighlighter; + } + + /** + * Returns wether or not the AiHighlighter should be used + */ get highlightedStreams() { return this.state.highlightedStreams; } @@ -408,6 +472,18 @@ class HighligherViews extends ViewHandler { return this.state.error; } + get highlighterVersion() { + return this.state.highlighterVersion; + } + + get isUpdaterRunning() { + return this.state.isUpdaterRunning; + } + + get updaterProgress() { + return this.state.updaterProgress; + } + /** * Takes a filepath to a video and returns a file:// url with a random * component to prevent the browser from caching it and missing changes. @@ -419,8 +495,18 @@ class HighligherViews extends ViewHandler { } @InitAfter('StreamingService') -export class HighlighterService extends PersistentStatefulService { - static defaultState: IHighligherState = { +export class HighlighterService extends PersistentStatefulService { + @Inject() streamingService: StreamingService; + @Inject() userService: UserService; + @Inject() usageStatisticsService: UsageStatisticsService; + @Inject() dismissablesService: DismissablesService; + @Inject() notificationsService: NotificationsService; + @Inject() jsonrpcService: JsonrpcService; + @Inject() navigationService: NavigationService; + @Inject() sharedStorageService: SharedStorageService; + @Inject() incrementalRolloutService: IncrementalRolloutService; + + static defaultState: IHighlighterState = { clips: {}, transition: { type: 'fade', @@ -459,10 +545,18 @@ export class HighlighterService extends PersistentStatefulService = {}; + renderingClips: Dictionary = {}; directoryCleared = false; @@ -576,12 +663,78 @@ export class HighlighterService extends PersistentStatefulService stream.id !== updatedStreamInfo.id, + ); + this.state.highlightedStreams = [...keepAsIs, updatedStreamInfo]; + } + + @mutation() + REMOVE_HIGHLIGHTED_STREAM(id: string) { + this.state.highlightedStreams = this.state.highlightedStreams.filter( + stream => stream.id !== id, + ); + } + + @mutation() + SET_UPDATER_PROGRESS(progress: number) { + this.state.updaterProgress = progress; + } + + @mutation() + SET_UPDATER_STATE(isRunning: boolean) { + this.state.isUpdaterRunning = isRunning; + } + + @mutation() + SET_HIGHLIGHTER_VERSION(version: string) { + this.state.highlighterVersion = version; + } + get views() { - return new HighligherViews(this.state); + return new HighlighterViews(this.state); } async init() { super.init(); + this.aiHighlighterEnabled = this.incrementalRolloutService.views.featureIsEnabled( + EAvailableFeatures.aiHighlighter, + ); + + if (this.aiHighlighterEnabled && !this.aiHighlighterUpdater) { + this.aiHighlighterUpdater = new AiHighlighterUpdater(); + } + + // check if ai highlighter is activated and we need to update it + if ( + this.aiHighlighterEnabled && + this.views.useAiHighlighter && + (await this.aiHighlighterUpdater.isNewVersionAvailable()) + ) { + await this.startUpdater(); + } + + // + this.views.clips.forEach(clip => { + if (isAiClip(clip) && (clip.aiInfo as any).moments) { + clip.aiInfo.inputs = (clip.aiInfo as any).moments; + delete (clip.aiInfo as any).moments; + } + }); //Check if files are existent, if not, delete this.views.clips.forEach(c => { @@ -598,6 +751,16 @@ export class HighlighterService extends PersistentStatefulService stream.state.type === 'detection-in-progress') + .forEach(stream => { + this.UPDATE_HIGHLIGHTED_STREAM({ + ...stream, + state: { type: EAiDetectionState.CANCELED_BY_USER, progress: 0 }, + }); + }); + this.views.clips.forEach(c => { this.UPDATE_CLIP({ path: c.path, @@ -617,50 +780,54 @@ export class HighlighterService extends PersistentStatefulService { - this.addClips([{ path: clipPath }], undefined, 'ReplayBuffer'); + const streamId = streamInfo?.id || undefined; + let endTime: number | undefined; + + if (streamId) { + endTime = moment().diff(aiRecordingStartTime, 'seconds'); + } else { + endTime = undefined; + } + + const REPLAY_BUFFER_DURATION = 20; // TODO M: Replace with settingsservice + const startTime = Math.max(0, endTime ? endTime - REPLAY_BUFFER_DURATION : 0); + + this.addClips([{ path: clipPath, startTime, endTime }], streamId, 'ReplayBuffer'); }); this.streamingService.streamingStatusChange.subscribe(async status => { if (status === EStreamingState.Live) { - streamStarted = true; + streamStarted = true; // console.log('live', this.streamingService.views.settings.platforms.twitch.title); + + if (this.views.useAiHighlighter === false) { + console.log('HighlighterService: Game:', this.streamingService.views.game); + // console.log('Highlighter not enabled or not Fortnite'); + return; + } + + // console.log('recording Alreadyt running?:', this.streamingService.views.isRecording); + + if (this.streamingService.views.isRecording) { + // console.log('Recording is already running'); + } else { + this.streamingService.toggleRecording(); + } + streamInfo = { + id: 'fromStreamRecording' + uuid(), + title: this.streamingService.views.settings.platforms.twitch?.title, + game: this.streamingService.views.game, + }; + aiRecordingInProgress = true; + aiRecordingStartTime = moment(); } if (status === EStreamingState.Offline) { @@ -681,13 +848,40 @@ export class HighlighterService extends PersistentStatefulService { + if (!aiRecordingInProgress) { + return; + } + + aiRecordingInProgress = false; + this.flow(path, streamInfo); + + this.navigationService.actions.navigate( + 'Highlighter', + { view: 'stream' }, + EMenuItemKey.Highlighter, + ); }); } } @@ -695,9 +889,12 @@ export class HighlighterService extends PersistentStatefulService { - const getHighestGlobalOrderPosition = this.getClips(this.views.clips, undefined).length; + const currentClips = this.getClips(this.views.clips, streamId); + const allClips = this.getClips(this.views.clips, undefined); + const getHighestGlobalOrderPosition = allClips.length; + + let newStreamInfo: { [key: string]: TStreamInfo } = {}; + if (source === 'Manual') { + if (streamId) { + currentClips.forEach(clip => { + if (clip?.streamInfo?.[streamId] === undefined) { + return; + } + + const updatedStreamInfo = { + ...clip.streamInfo, + [streamId]: { + ...clip.streamInfo[streamId], + orderPosition: clip.streamInfo[streamId]!.orderPosition + 1, + }, + }; + // update streaminfo position + this.UPDATE_CLIP({ + path: clip.path, + streamInfo: updatedStreamInfo, + }); + }); + + // Update globalOrderPosition of all other items as well + allClips.forEach(clip => { + this.UPDATE_CLIP({ + path: clip.path, + globalOrderPosition: clip.globalOrderPosition + 1, + }); + }); + + newStreamInfo = { + [streamId]: { + orderPosition: 0 + index, + }, + }; + } else { + // If no streamId currentCLips = allClips + currentClips.forEach(clip => { + this.UPDATE_CLIP({ + path: clip.path, + globalOrderPosition: clip.globalOrderPosition + 1, + }); + }); + } + } else { + if (streamId) { + newStreamInfo = { + [streamId]: { + orderPosition: index + currentClips.length + 1, + initialStartTime: clipData.startTime, + initialEndTime: clipData.endTime, + }, + }; + } + } if (this.state.clips[clipData.path]) { - // Clip exists already + //Add new newStreamInfo, wont be added if no streamId is available + const updatedStreamInfo = { + ...this.state.clips[clipData.path].streamInfo, + ...newStreamInfo, + }; + + this.UPDATE_CLIP({ + path: clipData.path, + streamInfo: updatedStreamInfo, + }); return; } else { this.ADD_CLIP({ @@ -722,14 +984,80 @@ export class HighlighterService extends PersistentStatefulService { + // Don't allow adding the same clip twice for ai clips + if (this.state.clips[clip.path]) return; + + const streamInfo: { [key: string]: TStreamInfo } = { + [newStreamInfo.id]: { + // Orderposition will get overwritten by sortStreamClipsByStartTime after creation + orderPosition: + index + currentHighestOrderPosition + (currentHighestOrderPosition === 0 ? 0 : 1), + initialStartTime: clip.startTime, + initialEndTime: clip.endTime, + }, + }; + + this.ADD_CLIP({ + path: clip.path, + loaded: false, + enabled: true, + startTrim: clip.startTrim, + endTrim: clip.endTrim, + deleted: false, + source: 'AiClip', + aiInfo: clip.aiClipInfo, + globalOrderPosition: + index + getHighestGlobalOrderPosition + (getHighestGlobalOrderPosition === 0 ? 0 : 1), + streamInfo, + }); + }); + this.sortStreamClipsByStartTime(this.views.clips, newStreamInfo); + await this.loadClips(newStreamInfo.id); + } + + // This sorts all clips (replayBuffer and aiClips) by initialStartTime + // That will assure that replayBuffer clips are also sorted in correctly in the stream + sortStreamClipsByStartTime(clips: TClip[], newStreamInfo: StreamInfoForAiHighlighter) { + const allClips = this.getClips(clips, newStreamInfo.id); + + const sortedClips = allClips.sort( + (a, b) => + (a.streamInfo?.[newStreamInfo.id]?.initialStartTime || 0) - + (b.streamInfo?.[newStreamInfo.id]?.initialStartTime || 0), + ); + + // Update order positions based on the sorted order + sortedClips.forEach((clip, index) => { + this.UPDATE_CLIP({ + path: clip.path, + streamInfo: { + [newStreamInfo.id]: { + ...(clip.streamInfo?.[newStreamInfo.id] ?? {}), + orderPosition: index, + }, + }, + }); + }); + return; + } + enableClip(path: string, enabled: boolean) { this.UPDATE_CLIP({ path, @@ -763,9 +1091,46 @@ export class HighlighterService extends PersistentStatefulService 1 + ) { + const updatedStreamInfo = { ...clip.streamInfo }; + delete updatedStreamInfo[streamId]; + + this.UPDATE_CLIP({ + path: clip.path, + streamInfo: updatedStreamInfo, + }); + } else { + this.REMOVE_CLIP(path); + this.removeScrubFile(clip.scrubSprite); + delete this.renderingClips[path]; + } + + if (clip.streamInfo !== undefined || streamId !== undefined) { + // if we are passing a streamId, only check if we need to remove the specific streamIds stream + // If we are not passing a streamId, check if we need to remove the streams the clip was part of + const ids: string[] = streamId ? [streamId] : Object.keys(clip.streamInfo ?? {}); + const length = this.views.clips.length; + + ids.forEach(id => { + let found = false; + if (length !== 0) { + for (let i = 0; i < length; i++) { + if (this.views.clips[i].streamInfo?.[id] !== undefined) { + found = true; + break; + } + } + } + if (!found) { + this.REMOVE_HIGHLIGHTED_STREAM(id); + } + }); + } } setTransition(transition: Partial) { @@ -780,6 +1145,9 @@ export class HighlighterService extends PersistentStatefulService(resolve => { + this.ADD_HIGHLIGHTED_STREAM(streamInfo); + setTimeout(() => { + resolve(); + }, 2000); + }); + } + + updateStream(streamInfo: IHighlightedStream) { + this.UPDATE_HIGHLIGHTED_STREAM(streamInfo); + } + + removeStream(streamId: string) { + this.REMOVE_HIGHLIGHTED_STREAM(streamId); + + //Remove clips from stream + const clipsToRemove = this.getClips(this.views.clips, streamId); + clipsToRemove.forEach(clip => { + this.removeClip(clip.path, streamId); + }); + } + async removeScrubFile(clipPath: string | undefined) { if (!clipPath) { console.warn('No scrub file path provided'); @@ -822,9 +1214,25 @@ export class HighlighterService extends PersistentStatefulService !c.loaded), - c => this.clips[c.path].init(), + c => this.renderingClips[c.path].init(), { concurrency: os.cpus().length, onProgress: completed => { - this.usageStatisticsService.recordAnalyticsEvent('Highlighter', { - type: 'ClipImport', - source: completed.source, - }); + this.usageStatisticsService.recordAnalyticsEvent( + this.views.useAiHighlighter ? 'AIHighlighter' : 'Highlighter', + { + type: 'ClipImport', + source: completed.source, + }, + ); this.UPDATE_CLIP({ path: completed.path, loaded: true, - scrubSprite: this.clips[completed.path].frameSource?.scrubJpg, - duration: this.clips[completed.path].duration, - deleted: this.clips[completed.path].deleted, + scrubSprite: this.renderingClips[completed.path].frameSource?.scrubJpg, + duration: this.renderingClips[completed.path].duration, + deleted: this.renderingClips[completed.path].deleted, }); }, }, @@ -869,6 +1281,10 @@ export class HighlighterService extends PersistentStatefulService clip.enabled && clip.streamInfo && clip.streamInfo[streamId] !== undefined) + renderingClips = this.getClips(this.views.clips, streamId) + .filter( + clip => + !!clip && clip.enabled && clip.streamInfo && clip.streamInfo[streamId] !== undefined, + ) .sort( (a: TClip, b: TClip) => (a.streamInfo?.[streamId]?.orderPosition ?? 0) - (b.streamInfo?.[streamId]?.orderPosition ?? 0), ) .map(c => { - const clip = this.clips[c.path]; + const clip = this.renderingClips[c.path]; clip.startTrim = c.startTrim; clip.endTrim = c.endTrim; @@ -929,11 +1352,11 @@ export class HighlighterService extends PersistentStatefulService c.enabled) .sort((a: TClip, b: TClip) => a.globalOrderPosition - b.globalOrderPosition) .map(c => { - const clip = this.clips[c.path]; + const clip = this.renderingClips[c.path]; clip.startTrim = c.startTrim; clip.endTrim = c.endTrim; @@ -942,19 +1365,19 @@ export class HighlighterService extends PersistentStatefulService c.reset(exportOptions), { + await pmap(renderingClips, c => c.reset(exportOptions), { onProgress: c => { if (c.deleted) { this.UPDATE_CLIP({ path: c.sourcePath, deleted: true }); @@ -977,18 +1405,18 @@ export class HighlighterService extends PersistentStatefulService !c.deleted); + renderingClips = renderingClips.filter(c => !c.deleted); - if (!clips.length) { + if (!renderingClips.length) { console.error('Highlighter: Export called without any clips!'); return; } // Estimate the total number of frames to set up export info - const totalFrames = clips.reduce((count: number, clip) => { + const totalFrames = renderingClips.reduce((count: number, clip) => { return count + clip.frameSource.nFrames; }, 0); - const numTransitions = clips.length - 1; + const numTransitions = renderingClips.length - 1; const transitionFrames = this.views.transitionDuration * exportOptions.fps; const totalFramesAfterTransitions = totalFrames - numTransitions * transitionFrames; @@ -1008,11 +1436,13 @@ export class HighlighterService extends PersistentStatefulService c.hasAudio).map(clip => clip.audioSource.extract())); + await Promise.all( + renderingClips.filter(c => c.hasAudio).map(clip => clip.audioSource.extract()), + ); const parsed = path.parse(this.views.exportInfo.file); const audioConcat = path.join(parsed.dir, `${parsed.name}-concat.flac`); let audioMix = path.join(parsed.dir, `${parsed.name}-mix.flac`); - fader = new AudioCrossfader(audioConcat, clips, this.views.transitionDuration); + fader = new AudioCrossfader(audioConcat, renderingClips, this.views.transitionDuration); await fader.export(); if (this.views.audio.musicEnabled && this.views.audio.musicPath) { @@ -1032,14 +1462,14 @@ export class HighlighterService extends PersistentStatefulService clip.audioSource.cleanup())); - const nClips = clips.length; + await Promise.all(renderingClips.map(clip => clip.audioSource.cleanup())); + const nClips = renderingClips.length; this.SET_EXPORT_INFO({ step: EExportStep.FrameRender }); // Cannot be null because we already checked there is at least 1 element in the array - let fromClip = clips.shift()!; - let toClip = clips.shift(); + let fromClip = renderingClips.shift()!; + let toClip = renderingClips.shift(); let transitioner: Transitioner | null = null; const exportPath = preview ? this.views.exportInfo.previewFile : this.views.exportInfo.file; @@ -1123,7 +1553,7 @@ export class HighlighterService extends PersistentStatefulService + isAiClip(clip) && + !!clip?.aiInfo?.metadata?.webcam_coordinates && + this.renderingClips[clip.path], + ) as IAiClip; + return clipWithWebcam?.aiInfo?.metadata?.webcam_coordinates || undefined; + } + /** + * + * @param webcamCoordinates + * @param outputWidth + * @param outputHeight + * @returns properly formatted complex filter for ffmpeg to move webcam to top in vertical video + */ + private getWebcamComplexFilterForFfmpeg( + webcamCoordinates: ICoordinates | null, + outputWidth: number, + outputHeight: number, + ) { + if (!webcamCoordinates) { + return ` + [0:v]crop=ih*${outputWidth}/${outputHeight}:ih,scale=${outputWidth}:-1:force_original_aspect_ratio=increase[final]; + `; + } + + const webcamTopX = webcamCoordinates?.x1; + const webcamTopY = webcamCoordinates?.y1; + const webcamWidth = webcamCoordinates?.x2 - webcamCoordinates?.x1; + const webcamHeight = webcamCoordinates?.y2 - webcamCoordinates?.y1; + + const oneThirdHeight = outputHeight / 3; + const twoThirdsHeight = (outputHeight * 2) / 3; + + return ` + [0:v]split=3[webcam][vid][blur_source]; + color=c=black:s=${outputWidth}x${outputHeight}:d=1[base]; + [webcam]crop=w=${webcamWidth}:h=${webcamHeight}:x=${webcamTopX}:y=${webcamTopY},scale=-1:${oneThirdHeight}[webcam_final]; + [vid]crop=ih*${outputWidth}/${twoThirdsHeight}:ih,scale=${outputWidth}:${twoThirdsHeight}[vid_cropped]; + [blur_source]crop=ih*${outputWidth}/${twoThirdsHeight}:ih,scale=${outputWidth}:${oneThirdHeight},gblur=sigma=50[blur]; + [base][blur]overlay=x=0:y=0[blur_base]; + [blur_base][webcam_final]overlay='(${outputWidth}-overlay_w)/2:(${oneThirdHeight}-overlay_h)/2'[base_webcam]; + [base_webcam][vid_cropped]overlay=x=0:y=${oneThirdHeight}[final]; + `; + } + // We throttle because this can go extremely fast, especially on previews @throttle(100) private setCurrentFrame(frame: number) { @@ -1232,9 +1744,12 @@ export class HighlighterService extends PersistentStatefulService { + if (this.aiHighlighterEnabled === false) { + console.log('HighlighterService: Not enabled'); + return; + } + + // if update is already in progress, need to wait until it's done + if (this.aiHighlighterUpdater.updateInProgress) { + await this.aiHighlighterUpdater.currentUpdate; + } else if (await this.aiHighlighterUpdater.isNewVersionAvailable()) { + await this.startUpdater(); + } + + const fallbackTitle = 'awesome-stream'; + const sanitizedTitle = streamInfo.title + ? streamInfo.title.replace(/[\\/:"*?<>|]+/g, ' ') + : this.extractDateTimeFromPath(filePath) || fallbackTitle; + + const setStreamInfo: IHighlightedStream = { + state: { + type: EAiDetectionState.IN_PROGRESS, + progress: 0, + }, + date: moment().toISOString(), + id: streamInfo.id || 'noId', + title: sanitizedTitle, + game: streamInfo.game || 'no title', + abortController: new AbortController(), + path: filePath, + }; + + this.streamMilestones = { + streamId: setStreamInfo.id, + milestones: [], + }; + + await this.addStream(setStreamInfo); + + const progressTracker = new ProgressTracker(progress => { + setStreamInfo.state.progress = progress; + this.updateStream(setStreamInfo); + }); + + const renderHighlights = async (partialHighlights: IHighlight[]) => { + console.log('🔄 cutHighlightClips'); + this.updateStream(setStreamInfo); + const clipData = await this.cutHighlightClips(filePath, partialHighlights, setStreamInfo); + console.log('✅ cutHighlightClips'); + // 6. add highlight clips + progressTracker.destroy(); + setStreamInfo.state.type = EAiDetectionState.FINISHED; + this.updateStream(setStreamInfo); + + console.log('🔄 addClips', clipData); + this.addAiClips(clipData, streamInfo); + console.log('✅ addClips'); + }; + + console.log('🔄 HighlighterData'); + try { + const highlighterResponse = await getHighlightClips( + filePath, + renderHighlights, + setStreamInfo.abortController!.signal, + (progress: number) => { + progressTracker.updateProgressFromHighlighter(progress); + }, + streamInfo.milestonesPath, + (milestone: IHighlighterMilestone) => { + this.streamMilestones?.milestones?.push(milestone); + }, + ); + + this.usageStatisticsService.recordAnalyticsEvent('AIHighlighter', { + type: 'Detection', + clips: highlighterResponse.length, + game: 'Fortnite', // hardcode for now + }); + console.log('✅ Final HighlighterData', highlighterResponse); + } catch (error: unknown) { + if (error instanceof Error && error.message === 'Highlight generation canceled') { + setStreamInfo.state.type = EAiDetectionState.CANCELED_BY_USER; + } else { + console.error('Error in highlight generation:', error); + setStreamInfo.state.type = EAiDetectionState.ERROR; + } + } finally { + setStreamInfo.abortController = undefined; + this.updateStream(setStreamInfo); + // stopProgressUpdates(); + } + + return; + } + + cancelHighlightGeneration(streamId: string): void { + const stream = this.views.highlightedStreams.find(s => s.id === streamId); + if (stream && stream.abortController) { + console.log('cancelHighlightGeneration', streamId); + stream.abortController.abort(); + } + } + + async getHighlightClipsRest( + type: string, + video_uri: string, + trim: { start_time: number; start_end: number } | undefined, + ) { + // Call highlighter code - replace with function + try { + const body = { + video_uri, + url, + trim, + }; + + const controller = new AbortController(); + const signal = controller.signal; + const timeout = 1000 * 60 * 30; // 30 minutes + console.time('requestDuration'); + const fetchTimeout = setTimeout(() => { + controller.abort(); + }, timeout); + + const response = await fetch(`http://127.0.0.1:8000${type}`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Access-Control-Allow-Origin': '*', + }, + body: JSON.stringify(body), + signal, + }); + + clearTimeout(fetchTimeout); + console.timeEnd('requestDuration'); + + if (!response.ok) { + throw new Error(`HTTP error! status: ${response.status}`); + } + + const data = await response.json(); + return data; + } catch (error: unknown) { + console.timeEnd('requestDuration'); + + if ((error as any).name === 'AbortError') { + console.error('Fetch request timed out'); + } else { + console.error('Fetch error:', error); + } + + throw new Error('Error while fetching'); + } + } + + async cutHighlightClips( + videoUri: string, + highlighterData: IHighlight[], + streamInfo: IHighlightedStream, + ): Promise { + const id = streamInfo.id; + const fallbackTitle = 'awesome-stream'; + const videoDir = path.dirname(videoUri); + const filename = path.basename(videoUri); + const sanitizedTitle = streamInfo.title + ? streamInfo.title.replace(/[\\/:"*?<>|]+/g, ' ') + : fallbackTitle; + const folderName = `${filename}-Clips-${sanitizedTitle}-${id.slice(id.length - 4, id.length)}`; + const outputDir = path.join(videoDir, folderName); + + // Check if directory for clips exists, if not create it + try { + try { + await fs.readdir(outputDir); + } catch (error: unknown) { + await fs.mkdir(outputDir); + } + } catch (error: unknown) { + console.error('Error creating file directory'); + return []; + } + + const sortedHighlights = highlighterData.sort((a, b) => a.start_time - b.start_time); + const results: INewClipData[] = []; + const processedFiles = new Set(); + + const duration = await this.getVideoDuration(videoUri); + + // First check the codec + const probeArgs = [ + '-v', + 'error', + '-select_streams', + 'v:0', + '-show_entries', + 'stream=codec_name,format=duration', + '-of', + 'default=nokey=1:noprint_wrappers=1', + videoUri, + ]; + let codec = ''; + try { + const codecResult = await execa(FFPROBE_EXE, probeArgs); + codec = codecResult.stdout.trim(); + console.log(`Codec for ${videoUri}: ${codec}`); + } catch (error: unknown) { + console.error(`Error checking codec for ${videoUri}:`, error); + } + console.time('export'); + const BATCH_SIZE = 1; + const DEFAULT_START_TRIM = 10; + const DEFAULT_END_TRIM = 10; + + for (let i = 0; i < sortedHighlights.length; i += BATCH_SIZE) { + const highlightBatch = sortedHighlights.slice(i, i + BATCH_SIZE); + const batchTasks = highlightBatch.map((highlight: IHighlight) => { + return async () => { + const formattedStart = highlight.start_time.toString().padStart(6, '0'); + const formattedEnd = highlight.end_time.toString().padStart(6, '0'); + const outputFilename = `${folderName}-${formattedStart}-${formattedEnd}.mp4`; + const outputUri = path.join(outputDir, outputFilename); + + if (processedFiles.has(outputUri)) { + console.log('File already exists'); + return null; + } + processedFiles.add(outputUri); + + // Check if the file with that name already exists and delete it if it does + try { + await fs.access(outputUri); + await fs.unlink(outputUri); + } catch (err: unknown) { + if ((err as any).code !== 'ENOENT') { + console.error(`Error checking existence of ${outputUri}:`, err); + } + } + + // Calculate new start and end times + new clip duration + const newClipStartTime = Math.max(0, highlight.start_time - DEFAULT_START_TRIM); + const actualStartTrim = highlight.start_time - newClipStartTime; + const newClipEndTime = Math.min(duration, highlight.end_time + DEFAULT_END_TRIM); + const actualEndTrim = newClipEndTime - highlight.end_time; + + const args = [ + '-ss', + newClipStartTime.toString(), + '-to', + newClipEndTime.toString(), + '-i', + videoUri, + '-c:v', + codec === 'h264' ? 'copy' : 'libx264', + '-c:a', + 'aac', + '-strict', + 'experimental', + '-b:a', + '192k', + '-movflags', + 'faststart', + outputUri, + ]; + + try { + const subprocess = execa(FFMPEG_EXE, args); + const timeoutDuration = 1000 * 60 * 5; + const timeoutId = setTimeout(() => { + console.warn(`FFMPEG process timed out for ${outputUri}`); + subprocess.kill('SIGTERM', { forceKillAfterTimeout: 2000 }); + }, timeoutDuration); + + try { + await subprocess; + console.log(`Created segment: ${outputUri}`); + const newClipData: INewClipData = { + path: outputUri, + aiClipInfo: { + inputs: highlight.inputs, + score: highlight.score, + metadata: highlight.metadata, + }, + startTime: highlight.start_time, + endTime: highlight.end_time, + startTrim: actualStartTrim, + endTrim: actualEndTrim, + }; + return newClipData; + } catch (error: unknown) { + console.warn(`Error during FFMPEG execution for ${outputUri}:`, error); + return null; + } finally { + clearTimeout(timeoutId); + } + } catch (error: unknown) { + console.error(`Error creating segment: ${outputUri}`, error); + return null; + } + }; + }); + + const batchResults = await Promise.allSettled(batchTasks.map(task => task())); + results.push( + ...batchResults + .filter((result): result is PromiseFulfilledResult => result.status === 'fulfilled') + .map(result => result.value) + .filter(value => value !== null), + ); + + const failedResults = batchResults.filter(result => result.status === 'rejected'); + + if (failedResults.length > 0) { + console.error('Failed exports:', failedResults); + } + } + + console.timeEnd('export'); + return results; + } getClips(clips: TClip[], streamId?: string): TClip[] { - const inputClips = clips.filter(clip => clip.path !== 'add'); - let wantedClips; + return clips.filter(clip => { + if (clip.path === 'add') { + return false; + } + const exists = this.fileExists(clip.path); + if (!exists) { + this.removeClip(clip.path, streamId); + return false; + } + if (streamId) { + return clip.streamInfo?.[streamId]; + } + return true; + }); + } - if (streamId) { - wantedClips = inputClips.filter(clip => clip.streamInfo?.[streamId]); - } else { - wantedClips = inputClips; + getClipsLoaded(clips: TClip[], streamId?: string): boolean { + return this.getClips(clips, streamId).every(clip => clip.loaded); + } + + getRoundDetails( + clips: TClip[], + ): { round: number; inputs: IInput[]; duration: number; hypeScore: number }[] { + const roundsMap: { + [key: number]: { inputs: IInput[]; duration: number; hypeScore: number; count: number }; + } = {}; + clips.forEach(clip => { + const aiClip = isAiClip(clip) ? clip : undefined; + const round = aiClip?.aiInfo?.metadata?.round ?? undefined; + if (aiClip?.aiInfo?.inputs && round) { + if (!roundsMap[round]) { + roundsMap[round] = { inputs: [], duration: 0, hypeScore: 0, count: 0 }; + } + roundsMap[round].inputs.push(...aiClip.aiInfo.inputs); + roundsMap[round].duration += aiClip.duration + ? aiClip.duration - aiClip.startTrim - aiClip.endTrim + : 0; + roundsMap[round].hypeScore += aiClip.aiInfo.score; + roundsMap[round].count += 1; + } + }); + + return Object.keys(roundsMap).map(round => { + const averageScore = + roundsMap[parseInt(round, 10)].hypeScore / roundsMap[parseInt(round, 10)].count; + const hypeScore = Math.ceil(Math.min(1, Math.max(0, averageScore)) * 5); + + return { + round: parseInt(round, 10), + inputs: roundsMap[parseInt(round, 10)].inputs, + duration: roundsMap[parseInt(round, 10)].duration, + hypeScore, + }; + }); + } + + async getVideoDuration(filePath: string): Promise { + const { stdout } = await execa(FFPROBE_EXE, [ + '-v', + 'error', + '-show_entries', + 'format=duration', + '-of', + 'default=noprint_wrappers=1:nokey=1', + filePath, + ]); + const duration = parseFloat(stdout); + return duration; + } + + enableOnlySpecificClips(clips: TClip[], streamId?: string) { + clips.forEach(clip => { + this.UPDATE_CLIP({ + path: clip.path, + enabled: false, + }); + }); + + // Enable specific clips + const clipsToEnable = this.getClips(clips, streamId); + clipsToEnable.forEach(clip => { + this.UPDATE_CLIP({ + path: clip.path, + enabled: true, + }); + }); + } + + private updateProgress(progress: IDownloadProgress) { + // this is a lie and its not a percent, its float from 0 and 1 + this.SET_UPDATER_PROGRESS(progress.percent * 100); + } + + /** + * Start updater process + */ + private async startUpdater() { + try { + this.SET_UPDATER_STATE(true); + this.SET_HIGHLIGHTER_VERSION(this.aiHighlighterUpdater.version || ''); + await this.aiHighlighterUpdater.update(progress => this.updateProgress(progress)); + } finally { + this.SET_UPDATER_STATE(false); } + } - const outputClips = wantedClips.filter(c => this.fileExists(c.path)); - if (outputClips.length !== wantedClips.length) { - wantedClips - .filter(c => !this.fileExists(c.path)) - .forEach(clip => { - this.removeClip(clip.path, streamId); - }); + /** + * Create milestones file if ids match and return path + */ + private async prepareMilestonesFile(streamId: string): Promise { + if ( + !this.streamMilestones || + this.streamMilestones.streamId !== streamId || + this.streamMilestones.milestones.length === 0 + ) { + return; } - return outputClips; + + const basepath = getSharedResource('ai-highlighter'); + const milestonesPath = path.join(basepath, 'milestones', 'milestones.json'); + + const milestonesData = JSON.stringify(this.streamMilestones.milestones); + await fs.outputFile(milestonesPath, milestonesData); + + return milestonesPath; } } diff --git a/app/services/incremental-rollout.ts b/app/services/incremental-rollout.ts index f8279f00a604..0ce94ce19ced 100644 --- a/app/services/incremental-rollout.ts +++ b/app/services/incremental-rollout.ts @@ -6,6 +6,7 @@ import { HostsService } from './hosts'; import Utils from 'services/utils'; import { InitAfter } from './core'; import { AppService } from './app'; +import { getOS, OS } from 'util/operating-systems'; export enum EAvailableFeatures { platform = 'slobs--platform', @@ -15,6 +16,7 @@ export enum EAvailableFeatures { restream = 'slobs--restream', tiktok = 'slobs--tiktok', highlighter = 'slobs--highlighter', + aiHighlighter = 'slobs--ai-highlighter', growTab = 'slobs--grow-tab', themeAudit = 'slobs--theme-audit', reactWidgets = 'slobs--react-widgets', @@ -114,6 +116,10 @@ class IncrementalRolloutView extends ViewHandler -1; } } diff --git a/app/services/streaming/streaming.ts b/app/services/streaming/streaming.ts index 2cdcd9a55a0c..d57266367d84 100644 --- a/app/services/streaming/streaming.ts +++ b/app/services/streaming/streaming.ts @@ -108,6 +108,7 @@ export class StreamingService replayBufferFileWrite = new Subject(); streamInfoChanged = new Subject>(); signalInfoChanged = new Subject(); + latestRecordingPath = new Subject(); streamErrorCreated = new Subject(); // Dummy subscription for stream deck @@ -1374,6 +1375,7 @@ export class StreamingService this.recordingModeService.actions.addRecordingEntry(parsedFilename); this.markersService.actions.exportCsv(parsedFilename); this.recordingModeService.addRecordingEntry(parsedFilename); + this.latestRecordingPath.next(filename); // Wrote signals come after Offline, so we return early here // to not falsely set our state out of Offline return; diff --git a/app/services/usage-statistics.ts b/app/services/usage-statistics.ts index 690159e6f54b..3c92d43abe92 100644 --- a/app/services/usage-statistics.ts +++ b/app/services/usage-statistics.ts @@ -35,6 +35,7 @@ type TAnalyticsEvent = | 'Shown' | 'AppStart' | 'Highlighter' + | 'AIHighlighter' | 'Hardware' | 'WebcamUse' | 'MicrophoneUse' diff --git a/package.json b/package.json index ece99634c75b..009cfdd2d851 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "package:mac-arm64": "yarn generate-agreement:mac && rimraf dist && electron-builder build -m --arm64 --config electron-builder/base.config.js", "package:preview": "yarn generate-agreement && rimraf dist && electron-builder build -w --x64 --config electron-builder/preview.config.js", "package:beta": "cross-env SLD_COMPILE_FOR_BETA=1 yarn compile && yarn generate-agreement && rimraf dist && electron-builder build -w --x64 --config electron-builder/beta.config.js", + "package:highlighter": "cross-env NODE_ENV=production yarn compile && cross-env SLOBS_NO_SIGN=true yarn package", "eslint": "eslint \"{app,guest-api,obs-api,updater}/**/*.ts\" main.js", "test": "tsc -p test && ava -v --timeout=3m ./test-dist/test/regular/**/*.js", "test:file": "tsc -p test && ava -v --timeout=60m", @@ -249,4 +250,4 @@ "got@^9.6.0": "11.8.5" }, "packageManager": "yarn@3.1.1" -} +} \ No newline at end of file diff --git a/test/regular/highlighter.ts b/test/regular/highlighter.ts index 6786638b61f3..43048b7f9418 100644 --- a/test/regular/highlighter.ts +++ b/test/regular/highlighter.ts @@ -33,13 +33,12 @@ test('Highlighter save and export', async t => { await stopStream(); await focusMain(); - await clickButton('All clips'); + await clickButton('All Clips'); await clickButton('Export'); const fileName = 'MyTestVideo.mp4'; const exportLocation = path.resolve(recordingDir, fileName); await fillForm({ exportLocation }); - const $exportBtn = await (await select('.ant-modal-content')).$('span=Export'); - await click($exportBtn); + await clickButton('Export Horizontal'); await waitForDisplayed('h1=Upload To', { timeout: 60000 }); t.true(fs.existsSync(exportLocation), 'The video file should exist'); });