Verified Commit 0dc3687b authored by Sebastian Höffner's avatar Sebastian Höffner
Browse files

Adding screen recording capabilities.

parent 40e781b0
NuGetPackages NuGetPackages
*.webm
# Created by https://www.gitignore.io/api/unity # Created by https://www.gitignore.io/api/unity
# Edit at https://www.gitignore.io/?templates=unity # Edit at https://www.gitignore.io/?templates=unity
......
...@@ -133,7 +133,7 @@ GameObject: ...@@ -133,7 +133,7 @@ GameObject:
- component: {fileID: 44631462} - component: {fileID: 44631462}
m_Layer: 0 m_Layer: 0
m_Name: Camera 1 m_Name: Camera 1
m_TagString: MainCamera m_TagString: Untagged
m_Icon: {fileID: 0} m_Icon: {fileID: 0}
m_NavMeshLayer: 0 m_NavMeshLayer: 0
m_StaticEditorFlags: 0 m_StaticEditorFlags: 0
...@@ -457,7 +457,8 @@ Transform: ...@@ -457,7 +457,8 @@ Transform:
m_LocalRotation: {x: -0, y: -0, z: -0, w: 1} m_LocalRotation: {x: -0, y: -0, z: -0, w: 1}
m_LocalPosition: {x: 0, y: 0, z: 0} m_LocalPosition: {x: 0, y: 0, z: 0}
m_LocalScale: {x: 1, y: 1, z: 1} m_LocalScale: {x: 1, y: 1, z: 1}
m_Children: [] m_Children:
- {fileID: 1385312691}
m_Father: {fileID: 44631460} m_Father: {fileID: 44631460}
m_RootOrder: 0 m_RootOrder: 0
m_LocalEulerAnglesHint: {x: 26.049002, y: 32.257, z: 0} m_LocalEulerAnglesHint: {x: 26.049002, y: 32.257, z: 0}
...@@ -493,6 +494,7 @@ MonoBehaviour: ...@@ -493,6 +494,7 @@ MonoBehaviour:
m_Name: m_Name:
m_EditorClassIdentifier: m_EditorClassIdentifier:
contextScene: Kitchen contextScene: Kitchen
captureVideo: 1
--- !u!4 &1309245142 --- !u!4 &1309245142
Transform: Transform:
m_ObjectHideFlags: 0 m_ObjectHideFlags: 0
...@@ -533,6 +535,100 @@ MonoBehaviour: ...@@ -533,6 +535,100 @@ MonoBehaviour:
m_Script: {fileID: 11500000, guid: d0bf19aac958d45fba9fe660e211e56a, type: 3} m_Script: {fileID: 11500000, guid: d0bf19aac958d45fba9fe660e211e56a, type: 3}
m_Name: m_Name:
m_EditorClassIdentifier: m_EditorClassIdentifier:
cameraCapture: {fileID: 1385312689}
useScreenSize: 0
--- !u!1 &1385312688
GameObject:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
serializedVersion: 6
m_Component:
- component: {fileID: 1385312691}
- component: {fileID: 1385312690}
- component: {fileID: 1385312689}
m_Layer: 0
m_Name: CaptureCam
m_TagString: Untagged
m_Icon: {fileID: 0}
m_NavMeshLayer: 0
m_StaticEditorFlags: 0
m_IsActive: 1
--- !u!114 &1385312689
MonoBehaviour:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1385312688}
m_Enabled: 0
m_EditorHideFlags: 0
m_Script: {fileID: 11500000, guid: a66fbd5ffe9b9d64da304996f1919f40, type: 3}
m_Name:
m_EditorClassIdentifier:
_width: 640
_height: 400
_preset: 8
_frameRate: 10
_videoPath:
--- !u!20 &1385312690
Camera:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1385312688}
m_Enabled: 1
serializedVersion: 2
m_ClearFlags: 1
m_BackGroundColor: {r: 0.19215687, g: 0.3019608, b: 0.4745098, a: 0}
m_projectionMatrixMode: 1
m_GateFitMode: 2
m_FOVAxisMode: 0
m_SensorSize: {x: 36, y: 24}
m_LensShift: {x: 0, y: 0}
m_FocalLength: 50
m_NormalizedViewPortRect:
serializedVersion: 2
x: 0
y: 0
width: 1
height: 1
near clip plane: 0.3
far clip plane: 1000
field of view: 60
orthographic: 0
orthographic size: 5
m_Depth: 0
m_CullingMask:
serializedVersion: 2
m_Bits: 4294967295
m_RenderingPath: -1
m_TargetTexture: {fileID: 0}
m_TargetDisplay: 0
m_TargetEye: 3
m_HDR: 1
m_AllowMSAA: 1
m_AllowDynamicResolution: 0
m_ForceIntoRT: 0
m_OcclusionCulling: 1
m_StereoConvergence: 10
m_StereoSeparation: 0.022
--- !u!4 &1385312691
Transform:
m_ObjectHideFlags: 0
m_CorrespondingSourceObject: {fileID: 0}
m_PrefabInstance: {fileID: 0}
m_PrefabAsset: {fileID: 0}
m_GameObject: {fileID: 1385312688}
m_LocalRotation: {x: 0, y: 0, z: 0, w: 1}
m_LocalPosition: {x: 0, y: 0, z: 0}
m_LocalScale: {x: 1, y: 1, z: 1}
m_Children: []
m_Father: {fileID: 963194228}
m_RootOrder: 0
m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
--- !u!1 &1396513430 --- !u!1 &1396513430
GameObject: GameObject:
m_ObjectHideFlags: 0 m_ObjectHideFlags: 0
......
using System.Collections; using UnityEngine;
using System.Collections.Generic; using FFmpegOut;
using UnityEngine;
public class FFmpegCapture : MonoBehaviour public class FFmpegCapture : MonoBehaviour
{ {
// Start is called before the first frame update public CameraCapture cameraCapture;
public bool useScreenSize = true;
void Start() void Start()
{ {
if (cameraCapture == null)
{
cameraCapture = Camera.main.gameObject.GetComponent<CameraCapture>();
}
if (useScreenSize)
{
cameraCapture.height = Screen.height;
cameraCapture.width = Screen.width;
}
cameraCapture.enabled = false;
} }
// Update is called once per frame public void StartCapture()
void Update()
{ {
cameraCapture.enabled = true;
}
public string StopCapture()
{
cameraCapture.enabled = false;
return cameraCapture.videoPath;
} }
} }
...@@ -2,20 +2,24 @@ ...@@ -2,20 +2,24 @@
using UnityEngine.SceneManagement; using UnityEngine.SceneManagement;
using SimpleJSON; using SimpleJSON;
using System; using System;
using System.IO;
using System.Collections; using System.Collections;
using System.Collections.Generic; using System.Collections.Generic;
namespace dlu namespace dlu
{ {
[RequireComponent(typeof(FFmpegCapture))]
public class SceneController : MonoBehaviour public class SceneController : MonoBehaviour
{ {
private JSONArray instruction; private JSONArray instruction;
public string contextScene = "Kitchen"; public string contextScene = "Kitchen";
private Func<string, IEnumerator> replyRequest; private Func<string, IEnumerator> replyToRequest;
private Dictionary<string, bool> actionParticipants; private Dictionary<string, bool> actionParticipants;
public bool captureVideo = true;
private FFmpegCapture videoCapture;
public void Start() public void Start()
{ {
if (!SceneManager.GetSceneByName(contextScene).isLoaded) if (!SceneManager.GetSceneByName(contextScene).isLoaded)
...@@ -27,6 +31,10 @@ public class SceneController : MonoBehaviour ...@@ -27,6 +31,10 @@ public class SceneController : MonoBehaviour
{ {
SceneManager.LoadSceneAsync("UI", LoadSceneMode.Additive); SceneManager.LoadSceneAsync("UI", LoadSceneMode.Additive);
} }
if (videoCapture == null)
{
videoCapture = GetComponent<FFmpegCapture>();
}
} }
public void SimpleSceneLoaded(Scene scene, LoadSceneMode mode) public void SimpleSceneLoaded(Scene scene, LoadSceneMode mode)
...@@ -40,10 +48,10 @@ public class SceneController : MonoBehaviour ...@@ -40,10 +48,10 @@ public class SceneController : MonoBehaviour
yield return RunSimulation(null, replyRequest); yield return RunSimulation(null, replyRequest);
} }
public IEnumerator RunSimulation(JSONArray instruction, Func<string, IEnumerator> replyRequest) public IEnumerator RunSimulation(JSONArray instruction, Func<string, IEnumerator> replyToRequest)
{ {
this.instruction = instruction; this.instruction = instruction;
this.replyRequest = replyRequest; this.replyToRequest = replyToRequest;
SceneManager.sceneUnloaded += OnSceneUnloaded; SceneManager.sceneUnloaded += OnSceneUnloaded;
SceneManager.UnloadSceneAsync(contextScene); SceneManager.UnloadSceneAsync(contextScene);
yield return null; yield return null;
...@@ -60,13 +68,17 @@ public class SceneController : MonoBehaviour ...@@ -60,13 +68,17 @@ public class SceneController : MonoBehaviour
{ {
JSONHandler.instance.RefreshSemanticLabels(); JSONHandler.instance.RefreshSemanticLabels();
SceneManager.sceneLoaded -= OnSceneLoaded; SceneManager.sceneLoaded -= OnSceneLoaded;
if (captureVideo)
{
videoCapture.StartCapture();
}
if (this.instruction != null) if (this.instruction != null)
{ {
StartCoroutine(EvaluateInstruction()); StartCoroutine(EvaluateInstruction());
} }
else else
{ {
StartCoroutine(replyRequest(JSONHandler.instance.GenerateContextJSON().ToString())); StartCoroutine(replyToRequest(JSONHandler.instance.GenerateContextJSON().ToString()));
} }
} }
...@@ -178,9 +190,25 @@ public class SceneController : MonoBehaviour ...@@ -178,9 +190,25 @@ public class SceneController : MonoBehaviour
else else
{ {
yield return new WaitForSeconds(5); yield return new WaitForSeconds(5);
// TODO: record trajectories of objects etc. string videoPath = null;
yield return replyRequest("{\"result\": \"Success!\"}"); if (captureVideo)
{
videoPath = videoCapture.StopCapture();
string dir = Directory.GetCurrentDirectory();
videoPath = Path.Combine(dir, videoPath);
}
// TODO: record trajectories of objects etc. and append them
string response = PrepareResponse(videoPath).ToString();
yield return replyToRequest(response);
}
} }
private JSONObject PrepareResponse(string videoPath)
{
JSONObject response = new JSONObject();
response.Add("video_file", videoPath);
response.Add("result", "Success!");
return response;
} }
} }
......
...@@ -60,11 +60,11 @@ public class ZMQServer : MonoBehaviour ...@@ -60,11 +60,11 @@ public class ZMQServer : MonoBehaviour
if (request == "GET_CONTEXT") if (request == "GET_CONTEXT")
{ {
StartCoroutine(sceneController.GetContext(ReplyRequest)); StartCoroutine(sceneController.GetContext(ReplyToRequest));
} }
else else
{ {
StartCoroutine(sceneController.RunSimulation(JSON.Parse(request) as JSONArray, ReplyRequest)); StartCoroutine(sceneController.RunSimulation(JSON.Parse(request) as JSONArray, ReplyToRequest));
} }
} }
...@@ -73,7 +73,7 @@ public class ZMQServer : MonoBehaviour ...@@ -73,7 +73,7 @@ public class ZMQServer : MonoBehaviour
server.Close(); server.Close();
} }
public IEnumerator ReplyRequest(string response) public IEnumerator ReplyToRequest(string response)
{ {
Debug.Log($"[ZMQServer] Sending message:\n{response}"); Debug.Log($"[ZMQServer] Sending message:\n{response}");
server.SendFrame(response); server.SendFrame(response);
......
...@@ -13,8 +13,8 @@ RenderTexture: ...@@ -13,8 +13,8 @@ RenderTexture:
m_ForcedFallbackFormat: 4 m_ForcedFallbackFormat: 4
m_DownscaleFallback: 0 m_DownscaleFallback: 0
serializedVersion: 3 serializedVersion: 3
m_Width: 1280 m_Width: 640
m_Height: 800 m_Height: 400
m_AntiAliasing: 1 m_AntiAliasing: 1
m_MipCount: -1 m_MipCount: -1
m_DepthFormat: 2 m_DepthFormat: 2
......
...@@ -13,8 +13,8 @@ RenderTexture: ...@@ -13,8 +13,8 @@ RenderTexture:
m_ForcedFallbackFormat: 4 m_ForcedFallbackFormat: 4
m_DownscaleFallback: 0 m_DownscaleFallback: 0
serializedVersion: 3 serializedVersion: 3
m_Width: 1280 m_Width: 640
m_Height: 800 m_Height: 400
m_AntiAliasing: 1 m_AntiAliasing: 1
m_MipCount: -1 m_MipCount: -1
m_DepthFormat: 2 m_DepthFormat: 2
......
...@@ -13,8 +13,8 @@ RenderTexture: ...@@ -13,8 +13,8 @@ RenderTexture:
m_ForcedFallbackFormat: 4 m_ForcedFallbackFormat: 4
m_DownscaleFallback: 0 m_DownscaleFallback: 0
serializedVersion: 3 serializedVersion: 3
m_Width: 1280 m_Width: 640
m_Height: 800 m_Height: 400
m_AntiAliasing: 1 m_AntiAliasing: 1
m_MipCount: -1 m_MipCount: -1
m_DepthFormat: 2 m_DepthFormat: 2
......
...@@ -13,8 +13,8 @@ RenderTexture: ...@@ -13,8 +13,8 @@ RenderTexture:
m_ForcedFallbackFormat: 4 m_ForcedFallbackFormat: 4
m_DownscaleFallback: 0 m_DownscaleFallback: 0
serializedVersion: 3 serializedVersion: 3
m_Width: 1280 m_Width: 640
m_Height: 800 m_Height: 400
m_AntiAliasing: 1 m_AntiAliasing: 1
m_MipCount: -1 m_MipCount: -1
m_DepthFormat: 2 m_DepthFormat: 2
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment