SceneController.cs 9.11 KB
Newer Older
1
using UnityEngine;
2
using UnityEngine.SceneManagement;
3
using SimpleJSON;
4
using System;
5
using System.IO;
6
7
using System.Collections;
using System.Collections.Generic;
8
9
10
11

namespace dlu
{

12
[RequireComponent(typeof(FFmpegCapture))]
13
14
public class SceneController : MonoBehaviour
{
15
16
    private JSONArray instruction;
    public string contextScene = "Kitchen";
17
    private Func<string, IEnumerator> replyToRequest;
18
    private Dictionary<string, bool> actionParticipants;
19

20
21
22
    public bool captureVideo = true;
    private FFmpegCapture videoCapture;

23
24
    public void Start()
    {
25
        if (!SceneManager.GetSceneByName(contextScene).isLoaded)
26
        {
27
28
            SceneManager.sceneLoaded += SimpleSceneLoaded;
            SceneManager.LoadSceneAsync(contextScene, LoadSceneMode.Additive);
29
        }
30
        if (!Application.isEditor && !SceneManager.GetSceneByName("UI").isLoaded)
31
        {
32
            SceneManager.LoadSceneAsync("UI", LoadSceneMode.Additive);
33
        }
34
35
36
37
        if (videoCapture == null)
        {
            videoCapture = GetComponent<FFmpegCapture>();
        }
38
    }
39

40
    public void SimpleSceneLoaded(Scene scene, LoadSceneMode mode)
41
    {
42
43
        JSONHandler.instance.RefreshSemanticLabels();
        SceneManager.sceneLoaded -= SimpleSceneLoaded;
44
    }
45

46
    public IEnumerator GetContext(Func<string, IEnumerator> replyRequest)
47
    {
48
49
50
        yield return RunSimulation(null, replyRequest);
    }

51
    public IEnumerator RunSimulation(JSONArray instruction, Func<string, IEnumerator> replyToRequest)
52
53
    {
        this.instruction = instruction;
54
        this.replyToRequest = replyToRequest;
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
        SceneManager.sceneUnloaded += OnSceneUnloaded;
        SceneManager.UnloadSceneAsync(contextScene);
        yield return null;
    }

    public void OnSceneUnloaded(Scene scene)
    {
        SceneManager.sceneUnloaded -= OnSceneUnloaded;
        SceneManager.sceneLoaded += OnSceneLoaded;
        SceneManager.LoadSceneAsync(scene.buildIndex, LoadSceneMode.Additive);
    }

    public void OnSceneLoaded(Scene scene, LoadSceneMode mode)
    {
        JSONHandler.instance.RefreshSemanticLabels();
        SceneManager.sceneLoaded -= OnSceneLoaded;
71
72
73
74
        if (captureVideo)
        {
            videoCapture.StartCapture();
        }
75
76
77
78
79
80
        if (this.instruction != null)
        {
            StartCoroutine(EvaluateInstruction());
        }
        else
        {
81
            StartCoroutine(replyToRequest(JSONHandler.instance.GenerateContextJSON().ToString()));
82
        }
83
84
85
86
87
88
89
    }

    public IEnumerator EvaluateInstruction()
    {
        Dictionary<string, JSONObject> taskDict = new Dictionary<string, JSONObject>();
        JSONObject task = null;
        // Populate quick access dict and search for task
90
        foreach (JSONObject jo in this.instruction)
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
        {
            taskDict[jo["name"]] = jo;
            if (task == null && jo.HasKey("classifies"))
            {
                task = jo;
                Debug.Log($"Task identified: {task}");
            }
        }

        // If no task is found, stop
        if (task == null)
        {
            Debug.LogError("No task found!");
            yield return null;
        }
        else  // if task is found, execute actions
        {
            foreach (JSONNode action in task["classifies"])
            {
                yield return ExecuteAction(action, taskDict);
            }
        }
    }

    public IEnumerator ExecuteAction(string actionName, Dictionary<string, JSONObject> taskDict)
    {
117
118
        actionParticipants = new Dictionary<string, bool>();

119
120
121
122
123
124
125
126
        JSONObject action = taskDict[actionName];
        Debug.Log($"Performing action {actionName}: {action}");
        List<string> participants = new List<string>();
        foreach (JSONNode participant in action["hasParticipant"])
        {
            Debug.Log($"Participant: {participant}");
            foreach(JSONNode is_a in taskDict[participant]["is_a"])
            {
127
                // TODO(@shoeffner): This is not good: we should query the ontology and don't use is_a which are also PhysicalAgents etc., also just add participants once
128
129
130
                if (is_a != "http://www.ontologydesignpatterns.org/ont/dul/DUL.owl#PhysicalAgent")
                {
                    participants.Add(participant);
131
                    break;
132
133
134
135
136
                }
            }
        }
        foreach (string participant in participants)
        {
137
138
139
140
            string referenceName = $"{actionName}/{participant}";
            actionParticipants.Add(referenceName, false);

            GameObject participantGO = JSONHandler.instance.sceneDict[participant];
141
            GameObject targetGO = new GameObject($"Target of {participant}");
142
            SceneManager.MoveGameObjectToScene(targetGO, SceneManager.GetSceneByName(contextScene));
143
144
145
146
147
148
149
150
151
152

            // Setup visuals
            GameObject renderer = new GameObject("RenderComponents");
            renderer.transform.SetParent(targetGO.transform);
            renderer.transform.localScale = participantGO.transform.lossyScale;
            MeshFilter mesh = renderer.AddComponent<MeshFilter>();
            mesh.mesh = participantGO.GetComponent<MeshFilter>().mesh;
            MeshRenderer mr = renderer.AddComponent<MeshRenderer>();
            mr.material = Resources.Load<Material>("Target");

153
154
155
156
157
158
159
160
161
            Vector3 position = taskDict[participant]["components"]["transform"]["position"];
            Quaternion orientation = taskDict[participant]["components"]["transform"]["orientation"];
            targetGO.transform.SetPositionAndRotation(position, orientation);
            double speed = taskDict[participant]["components"].HasKey("speed") ? taskDict[participant]["components"]["speed"].AsDouble : 1.0;

            Hashtable arguments = new Hashtable();
            arguments["name"] = actionName;
            arguments["position"] = targetGO.transform;
            arguments["speed"] = speed;
162
163
164
            arguments["oncomplete"] = "ActionParticipantDone";
            arguments["oncompleteparams"] = referenceName;
            arguments["oncompletetarget"] = gameObject;
165

166
            iTween.MoveTo(participantGO, arguments);
167
        }
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
        yield return WaitForActionsOrTimeout();
    }

    public void ActionParticipantDone(string referenceName)
    {
        Debug.Log($"{referenceName} is done.");
        actionParticipants[referenceName] = true;
    }

    public IEnumerator WaitForActionsOrTimeout()
    {
        bool allDone = true;
        foreach(bool status in actionParticipants.Values)
        {
            allDone &= status;
        }

        if (!allDone)
        {
            yield return new WaitForSeconds(1);
            yield return WaitForActionsOrTimeout();
        }
        else
        {
192
            yield return new WaitForSeconds(5);
193
194
195
196
197
198
199
200
201
202
            string videoPath = null;
            if (captureVideo)
            {
                videoPath = videoCapture.StopCapture();
                string dir = Directory.GetCurrentDirectory();
                videoPath = Path.Combine(dir, videoPath);
            }
            // TODO: record trajectories of objects etc. and append them
            string response = PrepareResponse(videoPath).ToString();
            yield return replyToRequest(response);
203
        }
204
    }
205
206
207
208
209

    private JSONObject PrepareResponse(string videoPath)
    {
        JSONObject response = new JSONObject();
        response.Add("video_file", videoPath);
210
211
        response.Add("trajectories.log", GetSchemasimLog());
        response.Add("context", JSONHandler.instance.GenerateContextJSON());
212
213
214
        response.Add("result", "Success!");
        return response;
    }
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252

    private JSONArray GetSchemasimLog()
    {
        Dictionary<string, List<JSONObject>> logs = new Dictionary<string, List<JSONObject>>();

        int count = 0;
        foreach (SemanticBehaviour sb in JSONHandler.instance.semanticBehaviours)
        {
            (List<Vector3> positions, List<Quaternion> rotations) = sb.GetLog();
            count = positions.Count;
            List<JSONObject> logData = new List<JSONObject>();
            for (int i = 0; i < positions.Count; ++i)
            {
                JSONObject jsonObject = new JSONObject();
                jsonObject.Add("tx", positions[i].x);
                jsonObject.Add("ty", positions[i].y);
                jsonObject.Add("tz", positions[i].z);
                jsonObject.Add("rx", rotations[i].x);
                jsonObject.Add("ry", rotations[i].y);
                jsonObject.Add("rz", rotations[i].z);
                jsonObject.Add("rw", rotations[i].w);
                logData.Add(jsonObject);
            }
            logs.Add(sb.GetComponent<InstanceIdentifier>().name, logData);
        }

        JSONArray logRows = new JSONArray();
        for (int t = 0; t < count; ++t)
        {
            JSONObject row = new JSONObject();
            foreach (var log in logs)
            {
                row.Add(log.Key, log.Value[t]);
            }
            logRows.Add(row);
        }
        return logRows;
    }
253
254
255
}

}