using UnityEngine; using UnityEngine.SceneManagement; using SimpleJSON; using System; using System.IO; using System.Collections; using System.Collections.Generic; namespace dlu { [RequireComponent(typeof(FFmpegCapture))] public class SceneController : MonoBehaviour { private JSONArray instruction; public string contextScene = "Kitchen"; private Func replyToRequest; private Dictionary actionParticipants; public bool captureVideo = true; private FFmpegCapture videoCapture; public void Start() { if (!SceneManager.GetSceneByName(contextScene).isLoaded) { SceneManager.sceneLoaded += SimpleSceneLoaded; SceneManager.LoadSceneAsync(contextScene, LoadSceneMode.Additive); } if (!Application.isEditor && !SceneManager.GetSceneByName("UI").isLoaded) { SceneManager.LoadSceneAsync("UI", LoadSceneMode.Additive); } if (videoCapture == null) { videoCapture = GetComponent(); } } public void SimpleSceneLoaded(Scene scene, LoadSceneMode mode) { JSONHandler.instance.RefreshSemanticLabels(); SceneManager.sceneLoaded -= SimpleSceneLoaded; } public IEnumerator GetContext(Func replyRequest) { yield return RunSimulation(null, replyRequest); } public IEnumerator RunSimulation(JSONArray instruction, Func replyToRequest) { this.instruction = instruction; this.replyToRequest = replyToRequest; SceneManager.sceneUnloaded += OnSceneUnloaded; SceneManager.UnloadSceneAsync(contextScene); yield return null; } public void OnSceneUnloaded(Scene scene) { SceneManager.sceneUnloaded -= OnSceneUnloaded; SceneManager.sceneLoaded += OnSceneLoaded; SceneManager.LoadSceneAsync(scene.buildIndex, LoadSceneMode.Additive); } public void OnSceneLoaded(Scene scene, LoadSceneMode mode) { JSONHandler.instance.RefreshSemanticLabels(); SceneManager.sceneLoaded -= OnSceneLoaded; if (captureVideo) { videoCapture.StartCapture(); } if (this.instruction != null) { StartCoroutine(EvaluateInstruction()); } else { StartCoroutine(replyToRequest(JSONHandler.instance.GenerateContextJSON().ToString())); } } public IEnumerator EvaluateInstruction() { Dictionary taskDict = new Dictionary(); JSONObject task = null; // Populate quick access dict and search for task foreach (JSONObject jo in this.instruction) { taskDict[jo["name"]] = jo; if (task == null && jo.HasKey("classifies")) { task = jo; Debug.Log($"Task identified: {task}"); } } // If no task is found, stop if (task == null) { Debug.LogError("No task found!"); yield return null; } else // if task is found, execute actions { foreach (JSONNode action in task["classifies"]) { yield return ExecuteAction(action, taskDict); } } } public IEnumerator ExecuteAction(string actionName, Dictionary taskDict) { actionParticipants = new Dictionary(); JSONObject action = taskDict[actionName]; Debug.Log($"Performing action {actionName}: {action}"); List participants = new List(); foreach (JSONNode participant in action["hasParticipant"]) { Debug.Log($"Participant: {participant}"); foreach(JSONNode is_a in taskDict[participant]["is_a"]) { // TODO(@shoeffner): This is not good: we should query the ontology and don't use is_a which are also PhysicalAgents etc., also just add participants once if (is_a != "http://www.ontologydesignpatterns.org/ont/dul/DUL.owl#PhysicalAgent") { participants.Add(participant); break; } } } foreach (string participant in participants) { string referenceName = $"{actionName}/{participant}"; actionParticipants.Add(referenceName, false); GameObject participantGO = JSONHandler.instance.sceneDict[participant]; GameObject targetGO = new GameObject($"Target of {participant}"); SceneManager.MoveGameObjectToScene(targetGO, SceneManager.GetSceneByName(contextScene)); // Setup visuals GameObject renderer = new GameObject("RenderComponents"); renderer.transform.SetParent(targetGO.transform); renderer.transform.localScale = participantGO.transform.lossyScale; MeshFilter mesh = renderer.AddComponent(); mesh.mesh = participantGO.GetComponent().mesh; MeshRenderer mr = renderer.AddComponent(); mr.material = Resources.Load("Target"); Vector3 position = taskDict[participant]["components"]["transform"]["position"]; Quaternion orientation = taskDict[participant]["components"]["transform"]["orientation"]; targetGO.transform.SetPositionAndRotation(position, orientation); double speed = taskDict[participant]["components"].HasKey("speed") ? taskDict[participant]["components"]["speed"].AsDouble : 1.0; Hashtable arguments = new Hashtable(); arguments["name"] = actionName; arguments["position"] = targetGO.transform; arguments["speed"] = speed; arguments["oncomplete"] = "ActionParticipantDone"; arguments["oncompleteparams"] = referenceName; arguments["oncompletetarget"] = gameObject; iTween.MoveTo(participantGO, arguments); } yield return WaitForActionsOrTimeout(); } public void ActionParticipantDone(string referenceName) { Debug.Log($"{referenceName} is done."); actionParticipants[referenceName] = true; } public IEnumerator WaitForActionsOrTimeout() { bool allDone = true; foreach(bool status in actionParticipants.Values) { allDone &= status; } if (!allDone) { yield return new WaitForSeconds(1); yield return WaitForActionsOrTimeout(); } else { yield return new WaitForSeconds(5); string videoPath = null; if (captureVideo) { videoPath = videoCapture.StopCapture(); string dir = Directory.GetCurrentDirectory(); videoPath = Path.Combine(dir, videoPath); } // TODO: record trajectories of objects etc. and append them string response = PrepareResponse(videoPath).ToString(); yield return replyToRequest(response); } } private JSONObject PrepareResponse(string videoPath) { JSONObject response = new JSONObject(); response.Add("video_file", videoPath); response.Add("result", "Success!"); return response; } } }