Adding screen recording capabilities.

.gitignore

 NuGetPackages
+*.webm
 # Created by https://www.gitignore.io/api/unity
 # Edit at https://www.gitignore.io/?templates=unity
 

Assets/DLU/Scenes/Controller.unity

@@ -133,7 +133,7 @@ GameObject:
   - component: {fileID: 44631462}
   m_Layer: 0
   m_Name: Camera 1
-  m_TagString: MainCamera
+  m_TagString: Untagged
   m_Icon: {fileID: 0}
   m_NavMeshLayer: 0
   m_StaticEditorFlags: 0
@@ -457,7 +457,8 @@ Transform:
   m_LocalRotation: {x: -0, y: -0, z: -0, w: 1}
   m_LocalPosition: {x: 0, y: 0, z: 0}
   m_LocalScale: {x: 1, y: 1, z: 1}
-  m_Children: []
+  m_Children:
+  - {fileID: 1385312691}
   m_Father: {fileID: 44631460}
   m_RootOrder: 0
   m_LocalEulerAnglesHint: {x: 26.049002, y: 32.257, z: 0}
@@ -493,6 +494,7 @@ MonoBehaviour:
   m_Name: 
   m_EditorClassIdentifier: 
   contextScene: Kitchen
+  captureVideo: 1
 --- !u!4 &1309245142
 Transform:
   m_ObjectHideFlags: 0
@@ -533,6 +535,100 @@ MonoBehaviour:
   m_Script: {fileID: 11500000, guid: d0bf19aac958d45fba9fe660e211e56a, type: 3}
   m_Name: 
   m_EditorClassIdentifier: 
+  cameraCapture: {fileID: 1385312689}
+  useScreenSize: 0
+--- !u!1 &1385312688
+GameObject:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  serializedVersion: 6
+  m_Component:
+  - component: {fileID: 1385312691}
+  - component: {fileID: 1385312690}
+  - component: {fileID: 1385312689}
+  m_Layer: 0
+  m_Name: CaptureCam
+  m_TagString: Untagged
+  m_Icon: {fileID: 0}
+  m_NavMeshLayer: 0
+  m_StaticEditorFlags: 0
+  m_IsActive: 1
+--- !u!114 &1385312689
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 1385312688}
+  m_Enabled: 0
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: a66fbd5ffe9b9d64da304996f1919f40, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+  _width: 640
+  _height: 400
+  _preset: 8
+  _frameRate: 10
+  _videoPath: 
+--- !u!20 &1385312690
+Camera:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 1385312688}
+  m_Enabled: 1
+  serializedVersion: 2
+  m_ClearFlags: 1
+  m_BackGroundColor: {r: 0.19215687, g: 0.3019608, b: 0.4745098, a: 0}
+  m_projectionMatrixMode: 1
+  m_GateFitMode: 2
+  m_FOVAxisMode: 0
+  m_SensorSize: {x: 36, y: 24}
+  m_LensShift: {x: 0, y: 0}
+  m_FocalLength: 50
+  m_NormalizedViewPortRect:
+    serializedVersion: 2
+    x: 0
+    y: 0
+    width: 1
+    height: 1
+  near clip plane: 0.3
+  far clip plane: 1000
+  field of view: 60
+  orthographic: 0
+  orthographic size: 5
+  m_Depth: 0
+  m_CullingMask:
+    serializedVersion: 2
+    m_Bits: 4294967295
+  m_RenderingPath: -1
+  m_TargetTexture: {fileID: 0}
+  m_TargetDisplay: 0
+  m_TargetEye: 3
+  m_HDR: 1
+  m_AllowMSAA: 1
+  m_AllowDynamicResolution: 0
+  m_ForceIntoRT: 0
+  m_OcclusionCulling: 1
+  m_StereoConvergence: 10
+  m_StereoSeparation: 0.022
+--- !u!4 &1385312691
+Transform:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 1385312688}
+  m_LocalRotation: {x: 0, y: 0, z: 0, w: 1}
+  m_LocalPosition: {x: 0, y: 0, z: 0}
+  m_LocalScale: {x: 1, y: 1, z: 1}
+  m_Children: []
+  m_Father: {fileID: 963194228}
+  m_RootOrder: 0
+  m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
 --- !u!1 &1396513430
 GameObject:
   m_ObjectHideFlags: 0

Assets/DLU/Scripts/FFmpegCapture.cs

-using System.Collections;
-using System.Collections.Generic;
-using UnityEngine;
+using UnityEngine;
+using FFmpegOut;
 
 public class FFmpegCapture : MonoBehaviour
 {
-    // Start is called before the first frame update
+    public CameraCapture cameraCapture;
+    public bool useScreenSize = true;
+
     void Start()
     {
-        
+        if (cameraCapture == null)
+        {
+            cameraCapture = Camera.main.gameObject.GetComponent<CameraCapture>();
+        }
+        if (useScreenSize)
+        {
+            cameraCapture.height = Screen.height;
+            cameraCapture.width = Screen.width;
+        }
+        cameraCapture.enabled = false;
     }
 
-    // Update is called once per frame
-    void Update()
+    public void StartCapture()
     {
+        cameraCapture.enabled = true;
+    }
 
+    public string StopCapture()
+    {
+        cameraCapture.enabled = false;
+        return cameraCapture.videoPath;
     }
 }

Assets/DLU/Scripts/SceneController.cs

@@ -2,20 +2,24 @@
 using UnityEngine.SceneManagement;
 using SimpleJSON;
 using System;
+using System.IO;
 using System.Collections;
 using System.Collections.Generic;
 
 namespace dlu
 {
 
+[RequireComponent(typeof(FFmpegCapture))]
 public class SceneController : MonoBehaviour
 {
     private JSONArray instruction;
     public string contextScene = "Kitchen";
-    private Func<string, IEnumerator> replyRequest;
-
+    private Func<string, IEnumerator> replyToRequest;
     private Dictionary<string, bool> actionParticipants;
 
+    public bool captureVideo = true;
+    private FFmpegCapture videoCapture;
+
     public void Start()
     {
         if (!SceneManager.GetSceneByName(contextScene).isLoaded)
@@ -27,6 +31,10 @@ public class SceneController : MonoBehaviour
         {
             SceneManager.LoadSceneAsync("UI", LoadSceneMode.Additive);
         }
+        if (videoCapture == null)
+        {
+            videoCapture = GetComponent<FFmpegCapture>();
+        }
     }
 
     public void SimpleSceneLoaded(Scene scene, LoadSceneMode mode)
@@ -40,10 +48,10 @@ public class SceneController : MonoBehaviour
         yield return RunSimulation(null, replyRequest);
     }
 
-    public IEnumerator RunSimulation(JSONArray instruction, Func<string, IEnumerator> replyRequest)
+    public IEnumerator RunSimulation(JSONArray instruction, Func<string, IEnumerator> replyToRequest)
     {
         this.instruction = instruction;
-        this.replyRequest = replyRequest;
+        this.replyToRequest = replyToRequest;
         SceneManager.sceneUnloaded += OnSceneUnloaded;
         SceneManager.UnloadSceneAsync(contextScene);
         yield return null;
@@ -60,13 +68,17 @@ public class SceneController : MonoBehaviour
     {
         JSONHandler.instance.RefreshSemanticLabels();
         SceneManager.sceneLoaded -= OnSceneLoaded;
+        if (captureVideo)
+        {
+            videoCapture.StartCapture();
+        }
         if (this.instruction != null)
         {
             StartCoroutine(EvaluateInstruction());
         }
         else
         {
-            StartCoroutine(replyRequest(JSONHandler.instance.GenerateContextJSON().ToString()));
+            StartCoroutine(replyToRequest(JSONHandler.instance.GenerateContextJSON().ToString()));
         }
     }
 
@@ -178,9 +190,25 @@ public class SceneController : MonoBehaviour
         else
         {
             yield return new WaitForSeconds(5);
-            // TODO: record trajectories of objects etc.
-            yield return replyRequest("{\"result\": \"Success!\"}");
+            string videoPath = null;
+            if (captureVideo)
+            {
+                videoPath = videoCapture.StopCapture();
+                string dir = Directory.GetCurrentDirectory();
+                videoPath = Path.Combine(dir, videoPath);
+            }
+            // TODO: record trajectories of objects etc. and append them
+            string response = PrepareResponse(videoPath).ToString();
+            yield return replyToRequest(response);
+        }
     }
+
+    private JSONObject PrepareResponse(string videoPath)
+    {
+        JSONObject response = new JSONObject();
+        response.Add("video_file", videoPath);
+        response.Add("result", "Success!");
+        return response;
     }
 }
 

Assets/DLU/Scripts/ZMQServer.cs

@@ -60,11 +60,11 @@ public class ZMQServer : MonoBehaviour
 
         if (request == "GET_CONTEXT")
         {
-            StartCoroutine(sceneController.GetContext(ReplyRequest));
+            StartCoroutine(sceneController.GetContext(ReplyToRequest));
         }
         else
         {
-            StartCoroutine(sceneController.RunSimulation(JSON.Parse(request) as JSONArray, ReplyRequest));
+            StartCoroutine(sceneController.RunSimulation(JSON.Parse(request) as JSONArray, ReplyToRequest));
         }
     }
 
@@ -73,7 +73,7 @@ public class ZMQServer : MonoBehaviour
         server.Close();
     }
 
-    public IEnumerator ReplyRequest(string response)
+    public IEnumerator ReplyToRequest(string response)
     {
         Debug.Log($"[ZMQServer] Sending message:\n{response}");
         server.SendFrame(response);

Assets/DLU/Textures/Camera 1.renderTexture

@@ -13,8 +13,8 @@ RenderTexture:
   m_ForcedFallbackFormat: 4
   m_DownscaleFallback: 0
   serializedVersion: 3
-  m_Width: 1280
-  m_Height: 800
+  m_Width: 640
+  m_Height: 400
   m_AntiAliasing: 1
   m_MipCount: -1
   m_DepthFormat: 2

Assets/DLU/Textures/Camera 2.renderTexture

@@ -13,8 +13,8 @@ RenderTexture:
   m_ForcedFallbackFormat: 4
   m_DownscaleFallback: 0
   serializedVersion: 3
-  m_Width: 1280
-  m_Height: 800
+  m_Width: 640
+  m_Height: 400
   m_AntiAliasing: 1
   m_MipCount: -1
   m_DepthFormat: 2

Assets/DLU/Textures/Camera 3.renderTexture

@@ -13,8 +13,8 @@ RenderTexture:
   m_ForcedFallbackFormat: 4
   m_DownscaleFallback: 0
   serializedVersion: 3
-  m_Width: 1280
-  m_Height: 800
+  m_Width: 640
+  m_Height: 400
   m_AntiAliasing: 1
   m_MipCount: -1
   m_DepthFormat: 2

Assets/DLU/Textures/Camera 4.renderTexture

@@ -13,8 +13,8 @@ RenderTexture:
   m_ForcedFallbackFormat: 4
   m_DownscaleFallback: 0
   serializedVersion: 3
-  m_Width: 1280
-  m_Height: 800
+  m_Width: 640
+  m_Height: 400
   m_AntiAliasing: 1
   m_MipCount: -1
   m_DepthFormat: 2