Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[MM-57019] Calls: Live captions support for mobile #7854

Merged
merged 9 commits into from
Mar 20, 2024
Merged

[MM-57019] Calls: Live captions support for mobile #7854

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
0b36822
mobile support for live captions
cpoile Mar 5, 2024
2f3ac45
add the 'Live captions turned on' ephemeral notice
cpoile Mar 5, 2024
30bfb31
PR comments
cpoile Mar 6, 2024
7dfede4
message should be translatable
cpoile Mar 6, 2024
2c3d73f
run i18n-extract
enahum Mar 7, 2024
ad008c7
call_recording_state -> call_job_state; ccAvailable is now dynamic
cpoile Mar 14, 2024
0336a93
Merge remote-tracking branch 'origin/MM-57019-live-captions' into MM-…
cpoile Mar 14, 2024
dde966d
backwards compatibility with pre calls v0.26.0
cpoile Mar 15, 2024
e0f0e68
correct mobile version in deprecation comments
cpoile Mar 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions app/actions/websocket/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,12 @@ import {openAllUnreadChannels} from '@actions/remote/preference';
import {autoUpdateTimezone} from '@actions/remote/user';
import {loadConfigAndCalls} from '@calls/actions/calls';
import {
handleCallCaption,
handleCallChannelDisabled,
handleCallChannelEnabled,
handleCallEnded,
handleCallHostChanged,
handleCallJobState,
handleCallRecordingState,
handleCallScreenOff,
handleCallScreenOn,
Expand Down Expand Up @@ -432,15 +434,23 @@ export async function handleEvent(serverUrl: string, msg: WebSocketMessage) {
case WebsocketEvents.CALLS_USER_REACTED:
handleCallUserReacted(serverUrl, msg);
break;

// DEPRECATED in favour of CALLS_JOB_STATE (since v2.15.0)
case WebsocketEvents.CALLS_RECORDING_STATE:
handleCallRecordingState(serverUrl, msg);
break;
case WebsocketEvents.CALLS_JOB_STATE:
handleCallJobState(serverUrl, msg);
break;
case WebsocketEvents.CALLS_HOST_CHANGED:
handleCallHostChanged(serverUrl, msg);
break;
case WebsocketEvents.CALLS_USER_DISMISSED_NOTIFICATION:
handleUserDismissedNotification(serverUrl, msg);
break;
case WebsocketEvents.CALLS_CAPTION:
handleCallCaption(serverUrl, msg);
break;

case WebsocketEvents.GROUP_RECEIVED:
handleGroupReceivedEvent(serverUrl, msg);
Expand Down
10 changes: 10 additions & 0 deletions app/constants/calls.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,18 @@ const PluginId = 'com.mattermost.calls';
const REACTION_TIMEOUT = 10000;
const REACTION_LIMIT = 20;
const CALL_QUALITY_RESET_MS = toMilliseconds({minutes: 1});
const CAPTION_TIMEOUT = 5000;

export enum MessageBarType {
Microphone,
CallQuality,
}

// The JobTypes from calls plugin's server/public/job.go
const JOB_TYPE_RECORDING = 'recording';
const JOB_TYPE_TRANSCRIBING = 'transcribing';
const JOB_TYPE_CAPTIONING = 'captioning';

export default {
RefreshConfigMillis,
RequiredServer,
Expand All @@ -39,4 +45,8 @@ export default {
REACTION_LIMIT,
MessageBarType,
CALL_QUALITY_RESET_MS,
CAPTION_TIMEOUT,
JOB_TYPE_RECORDING,
JOB_TYPE_TRANSCRIBING,
JOB_TYPE_CAPTIONING,
};
5 changes: 5 additions & 0 deletions app/constants/websocket.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,9 +81,14 @@ const WebsocketEvents = {
CALLS_USER_RAISE_HAND: `custom_${Calls.PluginId}_user_raise_hand`,
CALLS_USER_UNRAISE_HAND: `custom_${Calls.PluginId}_user_unraise_hand`,
CALLS_USER_REACTED: `custom_${Calls.PluginId}_user_reacted`,

// DEPRECATED in favour of CALLS_JOB_STATE (since v2.15.0)
CALLS_RECORDING_STATE: `custom_${Calls.PluginId}_call_recording_state`,
CALLS_JOB_STATE: `custom_${Calls.PluginId}_call_job_state`,
CALLS_HOST_CHANGED: `custom_${Calls.PluginId}_call_host_changed`,
CALLS_USER_DISMISSED_NOTIFICATION: `custom_${Calls.PluginId}_user_dismissed_notification`,
CALLS_CAPTION: `custom_${Calls.PluginId}_caption`,

GROUP_RECEIVED: 'received_group',
GROUP_MEMBER_ADD: 'group_member_add',
GROUP_MEMBER_DELETE: 'group_member_delete',
Expand Down
6 changes: 3 additions & 3 deletions app/products/calls/client/rest.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
// See LICENSE.txt for license information.

import type {ApiResp, CallsVersion} from '@calls/types/calls';
import type {CallChannelState, CallRecordingState, CallsConfig} from '@mattermost/calls/lib/types';
import type {CallChannelState, CallJobState, CallsConfig} from '@mattermost/calls/lib/types';
import type {RTCIceServer} from 'react-native-webrtc';

export interface ClientCallsMix {
Expand All @@ -14,8 +14,8 @@ export interface ClientCallsMix {
enableChannelCalls: (channelId: string, enable: boolean) => Promise<CallChannelState>;
endCall: (channelId: string) => Promise<ApiResp>;
genTURNCredentials: () => Promise<RTCIceServer[]>;
startCallRecording: (callId: string) => Promise<ApiResp | CallRecordingState>;
stopCallRecording: (callId: string) => Promise<ApiResp | CallRecordingState>;
startCallRecording: (callId: string) => Promise<ApiResp | CallJobState>;
stopCallRecording: (callId: string) => Promise<ApiResp | CallJobState>;
dismissCall: (channelId: string) => Promise<ApiResp>;
}

Expand Down
119 changes: 119 additions & 0 deletions app/products/calls/components/captions.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.

import React, {useEffect, useState} from 'react';
import {useIntl} from 'react-intl';
import {StyleSheet, Text, View} from 'react-native';

import CompassIcon from '@components/compass_icon';
import FormattedText from '@components/formatted_text';
import {changeOpacity} from '@utils/theme';
import {displayUsername} from '@utils/user';

import type {CallSession, LiveCaptionMobile} from '@calls/types/calls';

const styles = StyleSheet.create({
spacingContainer: {
position: 'relative',
width: '90%',
height: 48,
},
captionContainer: {
display: 'flex',
height: 400,
bottom: -352, // 48-400, to place the bottoms at the same place
gap: 8,
alignItems: 'center',
flexDirection: 'column-reverse',
overflow: 'hidden',
},
caption: {
paddingTop: 1,
paddingRight: 8,
paddingBottom: 3,
paddingLeft: 8,
borderRadius: 4,
backgroundColor: changeOpacity('#000', 0.64),
},
captionNotice: {
display: 'flex',
flexDirection: 'row',
gap: 8,
},
text: {
color: '#FFF',
fontFamily: 'Open Sans',
fontSize: 16,
fontStyle: 'normal',
fontWeight: '400',
lineHeight: 22,
textAlign: 'center',
},
});

type Props = {
captionsDict: Dictionary<LiveCaptionMobile>;
sessionsDict: Dictionary<CallSession>;
teammateNameDisplay: string;
}

const Captions = ({captionsDict, sessionsDict, teammateNameDisplay}: Props) => {
const intl = useIntl();
const [showCCNotice, setShowCCNotice] = useState(true);

useEffect(() => {
const timeoutID = setTimeout(() => {
setShowCCNotice(false);
}, 2000);
return () => clearTimeout(timeoutID);
}, []);

const captionsArr = Object.values(captionsDict).reverse();
cpoile marked this conversation as resolved.
Show resolved Hide resolved

if (showCCNotice && captionsArr.length > 0) {
setShowCCNotice(false);
}
if (showCCNotice) {
return (
<View style={styles.spacingContainer}>
<View style={styles.captionContainer}>
<View style={[styles.caption, styles.captionNotice]}>
<CompassIcon
name='closed-caption-outline'
color={'#FFF'}
size={18}
style={{alignSelf: 'center'}}
/>
<FormattedText
id={'mobile.calls_captions_turned_on'}
defaultMessage={'Live captions turned on'}
style={styles.text}
/>
</View>
</View>
</View>
);
}

return (
<View style={styles.spacingContainer}>
<View style={styles.captionContainer}>
{captionsArr.map((cap) => (
cpoile marked this conversation as resolved.
Show resolved Hide resolved
<View
key={cap.captionId}
style={styles.caption}
>
<Text
style={styles.text}
numberOfLines={0}
>
{`(${displayUsername(sessionsDict[cap.sessionId]?.userModel, intl.locale, teammateNameDisplay)}) ${cap.text}`}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Again more of a question for my own understanding. Is there any downside in using the whole sessions dictionary in terms of re-renders? After all what we really need is just the participant's id, not even the session as technically the user id would be sufficient right?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not sure what you mean here about 'all what we really need is just the participant's id' -- it's the userModel that we need. If that's what you meant, then I guess we could create a userModels dict in the parent component and only pass that in, but that wouldn't save any renders because it would be a new object also on every parent render, and we would then be creating another new object that would need to be collected later.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

My point here is that the sessions and participants state is data that can mutate quite often whereas the users (participants) models would only change when joining/leaving. So I was wondering whether there could be value in making this component depend on the latter.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Normally yes, that would be the right thing to do. But the way that the userModels are stored, we would need to extract it out on any change anyway. That would potentially be a good optimization to do (make an independent useModel dict), so I'll put that on my list. On the other hand, the amount of speedup in this case would be very little I imagine, given how it's only a few lines of captions. But the principle is good for maybe the participants grid or something more heavy.

</Text>
</View>
))}
</View>
</View>
);
};

export default Captions;
33 changes: 30 additions & 3 deletions app/products/calls/connection/websocket_event_handlers.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,13 @@ import {DeviceEventEmitter} from 'react-native';
import {fetchUsersByIds} from '@actions/remote/user';
import {
callEnded,
callStarted, getCallsConfig,
callStarted,
getCallsConfig,
receivedCaption,
removeIncomingCall,
setCallScreenOff,
setCallScreenOn,
setCaptioningState,
setChannelEnabled,
setHost,
setRaisedHand,
Expand All @@ -22,14 +25,18 @@ import {
} from '@calls/state';
import {isMultiSessionSupported} from '@calls/utils';
import {WebsocketEvents} from '@constants';
import Calls from '@constants/calls';
import DatabaseManager from '@database/manager';
import {getCurrentUserId} from '@queries/servers/system';

import type {CallRecordingStateData} from '@calls/types/calls';
import type {
CallHostChangedData,
CallRecordingStateData,
CallJobState,
CallJobStateData,
CallStartData,
EmptyData,
LiveCaptionData,
UserConnectedData,
UserDisconnectedData,
UserDismissedNotification,
Expand Down Expand Up @@ -155,8 +162,24 @@ export const handleCallUserReacted = (serverUrl: string, msg: WebSocketMessage<U
userReacted(serverUrl, msg.broadcast.channel_id, msg.data);
};

// DEPRECATED in favour of handleCallJobState (since v2.15.0)
export const handleCallRecordingState = (serverUrl: string, msg: WebSocketMessage<CallRecordingStateData>) => {
setRecordingState(serverUrl, msg.broadcast.channel_id, msg.data.recState);
const jobState: CallJobState = {
type: Calls.JOB_TYPE_RECORDING,
...msg.data.recState,
};
setRecordingState(serverUrl, msg.broadcast.channel_id, jobState);
};

export const handleCallJobState = (serverUrl: string, msg: WebSocketMessage<CallJobStateData>) => {
switch (msg.data.jobState.type) {
case Calls.JOB_TYPE_RECORDING:
setRecordingState(serverUrl, msg.broadcast.channel_id, msg.data.jobState);
break;
case Calls.JOB_TYPE_CAPTIONING:
setCaptioningState(serverUrl, msg.broadcast.channel_id, msg.data.jobState);
break;
}
};

export const handleCallHostChanged = (serverUrl: string, msg: WebSocketMessage<CallHostChangedData>) => {
Expand All @@ -177,3 +200,7 @@ export const handleUserDismissedNotification = async (serverUrl: string, msg: We

removeIncomingCall(serverUrl, msg.data.callID);
};

export const handleCallCaption = (serverUrl: string, msg: WebSocketMessage<LiveCaptionData>) => {
receivedCaption(serverUrl, msg.data);
};
Loading
Loading