FaceAPI、React Hooks、TypeScriptを使用し、Twilioビデオアプリケーションにフィルターを追加する方法

// start/src/App.tsx
...
import { connect, Room as RoomType } from 'twilio-video';
import * as faceapi from '@vladmandic/face-api';
...
function App() {
  ...
  return (
    ...
          <button
            ...
            onClick={async () => {
                ...
                const room = await connect(data.accessToken, {
                  name: 'cool-room',
                  audio: true,
                  video: { width: 640, height: 480 }
                });

                await faceapi.nets.ssdMobilenetv1.loadFromUri('/model');
                setRoom(room);

                ...
  )
}
...

// start/src/Track.tsx
...
function Track(props: { track: AudioTrack | VideoTrack }) {
  let divRef = useRef<HTMLDivElement>(null);
  // adding additional refs
  let canvasRef = useRef<HTMLCanvasElement>(null);
  let localAudioRef = useRef<HTMLAudioElement | null>(null);
  let localVideoRef = useRef<HTMLVideoElement | null>(null);
  let requestRef = useRef<number>();

  useEffect(() => {
    // refactoring a bit
    if (props.track) {
      divRef.current?.classList.add(props.track.kind);
      switch (props.track.kind) {
        case 'audio':
          localAudioRef.current = props.track.attach();
          break;
        case 'video':
          localVideoRef.current = props.track.attach();
          break;
      }
    }
  }, []);

  return (
    <div className="track" ref={divRef}>
      {props.track.kind === 'audio' &&
        <audio autoPlay={true} ref={localAudioRef} />
      }
      {props.track.kind === 'video' &&
        <>
          <video autoPlay={true} ref={localVideoRef} />
          <canvas width="640" height="480" ref={canvasRef} />
        </>
      }
    </div>
  );

// start/src/Track.tsx

import * as faceapi from '@vladmandic/face-api';

function Track(props: { track: AudioTrack | VideoTrack }) {
  ...
  useEffect(() => {
    function drawFilter() {
      let ctx = canvasRef.current?.getContext('2d');
      let image = new Image();
      image.src = 'sunglasses.png';

      async function step() {
        const results = await faceapi.detectAllFaces(localVideoRef.current);
        ctx?.drawImage(localVideoRef.current!, 0, 0);
        // eslint-disable-next-line array-callback-return
        results.map((result) => {
          ctx?.drawImage(
            image,
            result.box.x + 15,
            result.box.y + 30,
            result.box.width,
            result.box.width * (image.height / image.width)
          );
        });
        requestRef.current = requestAnimationFrame(step);
      }

      requestRef.current = requestAnimationFrame(step);
    }

   ...
  }, [])
  ... 
}
...

// start/src/Track.tsx
...
function Track(props: { track: AudioTrack | VideoTrack }) {
  ...
  useEffect(() => {
    ...
    if (props.track) {
      ...
      case 'video':
          localVideoRef.current = props.track.attach();
          localVideoRef.current?.addEventListener('playing', drawFilter);
          break;
       ...
    }
  }
}
...

// start/src/Track.tsx
...
function Track(props: { track: AudioTrack | VideoTrack }) {
  ...
  useEffect(() => {
    ...
    if (props.track) {
      divRef.current?.classList.add(props.track.kind);
      switch (props.track.kind) {
        case 'audio':
          localAudioRef.current = props.track.attach();
          break;
        case 'video':
          localVideoRef.current = props.track.attach();
          localVideoRef.current?.addEventListener('playing', drawFilter);
          break;
      }
    }

    return () => {
      if (props.track && props.track.kind === 'video') {
        localVideoRef.current?.removeEventListener('playing', drawFilter);
        cancelAnimationFrame(requestRef.current!);
      }
    }
  }, []);

  ...
}

// start/src/FilterMenu.tsx
import React from 'react';

function FilterMenu(props: { changeFilter: (filter: string) => void }) {
  const filters = ['Sunglasses', 'CoolerSunglasses'];

  return (
    <div className="filterMenu">
      {
        filters.map(filter => 
          <div className={`icon icon-${filter}`} 
            onClick={() => props.changeFilter(filter)}>
              {filter}
          </div>  
        )
      }
    </div>
  );
}

export default FilterMenu;

// start/src/Participant.tsx
...
import FilterMenu from './FilterMenu';

function Participant(props: { localParticipant: boolean, participant: LocalParticipant | RemoteParticipant }) {
  ...
  const [tracks, setTracks] = useState(nonNullTracks);
  const [filter, setFilter] = useState('Sunglasses');
  ...
  return (
    <div className="participant" id={props.participant.identity}>
      <div className="identity">{props.participant.identity}</div>
      {
        props.localParticipant
        ? <FilterMenu changeFilter={(filter) => {
            setFilter(filter);
          }} />
        : ''
      }

      {
        tracks.map((track) =>
          <Track key={track!.name} track={(track as VideoTrack | AudioTrack)} filter={filter} />)
      }
    </div>
  )
}

// start/src/Track.tsx
...
function Track(props: { track: AudioTrack | VideoTrack, filter: string }) {

// replace this
image.src = 'sunglasses.png';

// with this
image.src = props.filter === 'Sunglasses' ? 'sunglasses.png' : 'sunglasses-style.png';

// change this
}, []);

// to this
}, [props.filter]);

...
import { connect, Room as RoomType, LocalDataTrack } from 'twilio-video';
...
function App() {
  ...
         const room = await connect(data.accessToken, {
           name: 'cool-room',
           audio: true,
           video: { width: 640, height: 480 }
         });

         const localDataTrack = new LocalDataTrack();
         await room.localParticipant.publishTrack(localDataTrack);
         await faceapi.nets.ssdMobilenetv1.loadFromUri('/model');
}

// start/src/Participant.tsx
...
import { LocalParticipant, RemoteParticipant, LocalTrackPublication, RemoteTrackPublication, VideoTrack, AudioTrack, LocalDataTrack, DataTrack } from 'twilio-video';
...
function Participant(props: { localParticipant: boolean, participant: LocalParticipant | RemoteParticipant }) {
  ...
  useEffect(() => {
    if (!props.localParticipant) {
      ...
      // here the user adds the data track to the list of local tracks
      props.participant.on('trackPublished', track => {
        setTracks(prevState => ([...prevState, track]));
      });
    }
  // eslint-disable-next-line react-hooks/exhaustive-deps
  }, []);

  return (
    ...
      {
        props.localParticipant
        ? <FilterMenu changeFilter={(filter) => {
            // when the user changes the filter, notify all other users
            // retrieve the dataTrack from the list of tracks
            const dataTrack = tracks.find(track => track!.kind === 'data') as LocalDataTrack;
            // send filter information
            dataTrack!.send(filter);
            setFilter(filter);
          }} />
        : ''
      }

      {
        tracks.map((track) =>
          <Track key={track!.name} track={(track as VideoTrack | AudioTrack | DataTrack)} filter={filter} setFilter={setFilter}/>)
      }
    ...
  )
}

// start/src/Track.tsx
...
import { AudioTrack, VideoTrack, DataTrack } from 'twilio-video';
...
function Track(props: { track: AudioTrack | VideoTrack | DataTrack, filter: string, setFilter: (filter: string) => void }) {
  ...
  useEffect(() => {
    ...
    if (props.track) {
      ...
      switch (props.track.kind) {
        ...
        case 'data':
          // when receiving a message, update the filter
          props.track.on('message', props.setFilter);
          break;
      }
    }
    ...
  }, [props.filter]);
   
}
...