Fixups for Mac, and a bunch for PC (#1654)

* fix mac nui

* fix cereal

* builds on mac without warnings

* ui swap shouldn't be instant on pc

* camerad builds on mac

* modeld, and now everything, builds on mac

* ui draws pictures on mac now

* fix camerad shutdown

* support ui looping on pc

* python3 on that

* SCNu64 should be cross platform

* bump cereal and new apk

* clean up

Co-authored-by: Adeeb Shihadeh <adeebshihadeh@gmail.com>

SConstruct

@@ -62,6 +62,9 @@ if arch == "aarch64" or arch == "larch64":
       cxxflags += ["-DQCOM_REPLAY"]
 
 else:
+  cflags = []
+  cxxflags = []
+
   lenv = {
     "PATH": "#external/bin:" + os.environ['PATH'],
   }
@@ -77,6 +80,8 @@ else:
       "/usr/local/lib",
       "/System/Library/Frameworks/OpenGL.framework/Libraries",
     ]
+    cflags.append("-DGL_SILENCE_DEPRECATION")
+    cxxflags.append("-DGL_SILENCE_DEPRECATION")
   else:
     libpath = [
       "#phonelibs/snpe/x86_64-linux-clang",
@@ -96,9 +101,6 @@ else:
   # allows shared libraries to work globally
   rpath = [os.path.join(os.getcwd(), x) for x in rpath]
 
-  cflags = []
-  cxxflags = []
-
 ccflags_asan = ["-fsanitize=address", "-fno-omit-frame-pointer"] if GetOption('asan') else []
 ldflags_asan = ["-fsanitize=address"] if GetOption('asan') else []
 
@@ -215,9 +217,8 @@ SConscript(['common/kalman/SConscript'])
 SConscript(['common/transformations/SConscript'])
 SConscript(['phonelibs/SConscript'])
 
-if arch != "Darwin":
-  SConscript(['selfdrive/camerad/SConscript'])
-  SConscript(['selfdrive/modeld/SConscript'])
+SConscript(['selfdrive/camerad/SConscript'])
+SConscript(['selfdrive/modeld/SConscript'])
 
 SConscript(['selfdrive/controls/lib/cluster/SConscript'])
 SConscript(['selfdrive/controls/lib/lateral_mpc/SConscript'])

cereal

@@ -1 +1 @@
-Subproject commit 68fdf28670b48a16a60498b4e5cc1984e7e66651
+Subproject commit 9915b2086a4205d9a28eead6139d5d7cbb73b00b

selfdrive/camerad/SConscript

@@ -22,6 +22,10 @@ else:
   else:
     libs += []
     cameras = ['cameras/camera_frame_stream.cc']
+  if arch == "Darwin":
+    del libs[libs.index('OpenCL')]
+    env = env.Clone()
+    env['FRAMEWORKS'] = ['OpenCL']
 
 env.SharedLibrary('snapshot/visionipc',
   ["#selfdrive/common/visionipc.c", "#selfdrive/common/ipc.c"])

selfdrive/camerad/cameras/camera_frame_stream.cc

@@ -70,18 +70,8 @@ void run_frame_stream(DualCameraState *s) {
 
     cl_command_queue q = rear_camera->camera_bufs[buf_idx].copy_q;
     cl_mem yuv_cl = rear_camera->camera_bufs[buf_idx].buf_cl;
-    cl_event map_event;
-    void *yuv_buf = (void *)clEnqueueMapBuffer(q, yuv_cl, CL_TRUE,
-                                                CL_MAP_WRITE, 0, frame.getImage().size(),
-                                                0, NULL, &map_event, &err);
-    assert(err == 0);
-    clWaitForEvents(1, &map_event);
-    clReleaseEvent(map_event);
-    memcpy(yuv_buf, frame.getImage().begin(), frame.getImage().size());
 
-    clEnqueueUnmapMemObject(q, yuv_cl, yuv_buf, 0, NULL, &map_event);
-    clWaitForEvents(1, &map_event);
-    clReleaseEvent(map_event);
+    clEnqueueWriteBuffer(q, yuv_cl, CL_TRUE, 0, frame.getImage().size(), frame.getImage().begin(), 0, NULL, NULL);
     tbuffer_dispatch(tb, buf_idx);
   }
 }

selfdrive/camerad/main.cc

@@ -15,6 +15,7 @@
 #include "common/util.h"
 #include "common/swaglog.h"
 
+#include "common/ipc.h"
 #include "common/visionipc.h"
 #include "common/visionbuf.h"
 #include "common/visionimg.h"
@@ -346,8 +347,12 @@ void* processing_thread(void *arg) {
   LOG("setpriority returns %d", err);
 
   // init cl stuff
+#ifdef __APPLE__
+  cl_command_queue q = clCreateCommandQueue(s->context, s->device_id, 0, &err);
+#else
   const cl_queue_properties props[] = {0}; //CL_QUEUE_PRIORITY_KHR, CL_QUEUE_PRIORITY_HIGH_KHR, 0};
   cl_command_queue q = clCreateCommandQueueWithProperties(s->context, s->device_id, props, &err);
+#endif
   assert(err == 0);
 
   // init the net
@@ -542,7 +547,7 @@ void* processing_thread(void *arg) {
     // one thumbnail per 5 seconds (instead of %5 == 0 posenet)
     if (cnt % 100 == 3) {
       uint8_t* thumbnail_buffer = NULL;
-      uint64_t thumbnail_len = 0;
+      unsigned long thumbnail_len = 0;
 
       unsigned char *row = (unsigned char *)malloc(s->rgb_width/4*3);
 
@@ -852,21 +857,7 @@ void* visionserver_thread(void* arg) {
   assert(terminate);
   void* terminate_raw = zsock_resolve(terminate);
 
-  unlink(VIPC_SOCKET_PATH);
-
-  int sock = socket(AF_UNIX, SOCK_SEQPACKET, 0);
-  struct sockaddr_un addr = {
-    .sun_family = AF_UNIX,
-    .sun_path = VIPC_SOCKET_PATH,
-  };
-  err = bind(sock, (struct sockaddr *)&addr, sizeof(addr));
-  assert(err == 0);
-
-  err = listen(sock, 3);
-  assert(err == 0);
-
-  // printf("waiting\n");
-
+  int sock = ipc_bind(VIPC_SOCKET_PATH);
   while (!do_exit) {
     zmq_pollitem_t polls[2] = {{0}};
     polls[0].socket = terminate_raw;
@@ -1134,9 +1125,10 @@ void init_buffers(VisionState *s) {
                                             3);
   s->krnl_rgb_laplacian = clCreateKernel(s->prg_rgb_laplacian, "rgb2gray_conv2d", &err);
   assert(err == 0);
-  s->rgb_conv_roi_cl = clCreateBuffer(s->context, CL_MEM_READ_WRITE | CL_MEM_SVM_FINE_GRAIN_BUFFER,
+  // TODO: Removed CL_MEM_SVM_FINE_GRAIN_BUFFER, confirm it doesn't matter
+  s->rgb_conv_roi_cl = clCreateBuffer(s->context, CL_MEM_READ_WRITE,
       s->rgb_width/NUM_SEGMENTS_X * s->rgb_height/NUM_SEGMENTS_Y * 3 * sizeof(uint8_t), NULL, NULL);
-  s->rgb_conv_result_cl = clCreateBuffer(s->context, CL_MEM_READ_WRITE | CL_MEM_SVM_FINE_GRAIN_BUFFER,
+  s->rgb_conv_result_cl = clCreateBuffer(s->context, CL_MEM_READ_WRITE,
       s->rgb_width/NUM_SEGMENTS_X * s->rgb_height/NUM_SEGMENTS_Y * sizeof(int16_t), NULL, NULL);
   s->rgb_conv_filter_cl = clCreateBuffer(s->context, CL_MEM_READ_ONLY | CL_MEM_COPY_HOST_PTR,
       9 * sizeof(int16_t), (void*)&lapl_conv_krnl, NULL);
@@ -1194,7 +1186,7 @@ void party(VisionState *s) {
                        processing_thread, s);
   assert(err == 0);
 
-#ifndef QCOM2
+#if !defined(QCOM2) && !defined(__APPLE__)
   // TODO: fix front camera on qcom2
   pthread_t frontview_thread_handle;
   err = pthread_create(&frontview_thread_handle, NULL,
@@ -1215,7 +1207,7 @@ void party(VisionState *s) {
 
   zsock_signal(s->terminate_pub, 0);
 
-#if !defined(QCOM2) && !defined(QCOM_REPLAY)
+#if !defined(QCOM2) && !defined(QCOM_REPLAY) && !defined(__APPLE__)
   LOG("joining frontview_thread");
   err = pthread_join(frontview_thread_handle, NULL);
   assert(err == 0);

selfdrive/common/ipc.c

@@ -15,7 +15,11 @@
 int ipc_connect(const char* socket_path) {
   int err;
 
+#ifdef __APPLE__
+  int sock = socket(AF_UNIX, SOCK_STREAM, 0);
+#else
   int sock = socket(AF_UNIX, SOCK_SEQPACKET, 0);
+#endif
   if (sock < 0) return -1;
   struct sockaddr_un addr = {
     .sun_family = AF_UNIX,
@@ -35,7 +39,11 @@ int ipc_bind(const char* socket_path) {
 
   unlink(socket_path);
 
+#ifdef __APPLE__
+  int sock = socket(AF_UNIX, SOCK_STREAM, 0);
+#else
   int sock = socket(AF_UNIX, SOCK_SEQPACKET, 0);
+#endif
   struct sockaddr_un addr = {
     .sun_family = AF_UNIX,
   };

selfdrive/common/util.h

@@ -29,6 +29,7 @@ typedef void (*sighandler_t)(int sig);
 
 #define ARRAYSIZE(x) (sizeof(x)/sizeof(x[0]))
 
+#undef ALIGN
 #define ALIGN(x, align) (((x) + (align)-1) & ~((align)-1))
 
 #ifdef __cplusplus

selfdrive/common/visionbuf_cl.c

@@ -18,11 +18,17 @@
 int offset = 0;
 void *malloc_with_fd(size_t len, int *fd) {
   char full_path[0x100];
+#ifdef __APPLE__
+  snprintf(full_path, sizeof(full_path)-1, "/tmp/visionbuf_%d_%d", getpid(), offset++);
+#else
   snprintf(full_path, sizeof(full_path)-1, "/dev/shm/visionbuf_%d_%d", getpid(), offset++);
+#endif
   *fd = open(full_path, O_RDWR | O_CREAT, 0777);
+  assert(*fd >= 0);
   unlink(full_path);
   ftruncate(*fd, len);
   void *addr = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED, *fd, 0);
+  assert(addr != MAP_FAILED);
   return addr;
 }
 

selfdrive/common/visionipc.c

@@ -35,7 +35,7 @@ int vipc_recv(int fd, VisionPacket *out_p) {
     p2.d = p.d;
     *out_p = p2;
   }
-  //printf("%d = vipc_recv(%d, %d): %d %d %d %u\n", ret, fd, p2.num_fds, out_p->d.stream_bufs.type, out_p->d.stream_bufs.width, out_p->d.stream_bufs.height, out_p->d.stream_bufs.buf_len);
+  //printf("%d = vipc_recv(%d, %d): %d %d %d %zu\n", ret, fd, p2.num_fds, out_p->d.stream_bufs.type, out_p->d.stream_bufs.width, out_p->d.stream_bufs.height, out_p->d.stream_bufs.buf_len);
   return ret;
 }
 
@@ -47,7 +47,7 @@ int vipc_send(int fd, const VisionPacket *p2) {
     .d = p2->d,
   };
   int ret = ipc_sendrecv_with_fds(true, fd, (void*)&p, sizeof(p), (int*)p2->fds, p2->num_fds, NULL);
-  //printf("%d = vipc_send(%d, %d): %d %d %d %u\n", ret, fd, p2->num_fds, p2->d.stream_bufs.type, p2->d.stream_bufs.width, p2->d.stream_bufs.height, p2->d.stream_bufs.buf_len);
+  //printf("%d = vipc_send(%d, %d): %d %d %d %zu\n", ret, fd, p2->num_fds, p2->d.stream_bufs.type, p2->d.stream_bufs.width, p2->d.stream_bufs.height, p2->d.stream_bufs.buf_len);
   return ret;
 }
 

selfdrive/modeld/SConscript

@@ -27,6 +27,15 @@ else:
   # tell runners to use it
   lenv['CFLAGS'].append("-DUSE_TF_MODEL")
   lenv['CXXFLAGS'].append("-DUSE_TF_MODEL")
+  if arch == "Darwin":
+    # fix OpenCL
+    del libs[libs.index('OpenCL')]
+    lenv['FRAMEWORKS'] = ['OpenCL']
+
+    # no SNPE on Mac
+    del libs[libs.index('SNPE')]
+    del libs[libs.index('symphony-cpu')]
+    del common_src[common_src.index('runners/snpemodel.cc')]
 
 common = lenv.Object(common_src)
 

selfdrive/modeld/models/commonmodel.h

@@ -2,7 +2,11 @@
 #define COMMONMODEL_H
 
 #define CL_USE_DEPRECATED_OPENCL_1_2_APIS
+#ifdef __APPLE__
+#include <OpenCL/cl.h>
+#else
 #include <CL/cl.h>
+#endif
 
 #include "common/mat.h"
 #include "transforms/transform.h"

selfdrive/proclogd/proclogd.cc

@@ -99,14 +99,14 @@ int main() {
       uint64_t mem_cached = 0, mem_active = 0, mem_inactive = 0, mem_shared = 0;
 
       while (std::getline(smem, mem_line)) {
-        if (util::starts_with(mem_line, "MemTotal:")) sscanf(mem_line.data(), "MemTotal: %lu kB", &mem_total);
-        else if (util::starts_with(mem_line, "MemFree:")) sscanf(mem_line.data(), "MemFree: %lu kB", &mem_free);
-        else if (util::starts_with(mem_line, "MemAvailable:")) sscanf(mem_line.data(), "MemAvailable: %lu kB", &mem_available);
-        else if (util::starts_with(mem_line, "Buffers:")) sscanf(mem_line.data(), "Buffers: %lu kB", &mem_buffers);
-        else if (util::starts_with(mem_line, "Cached:")) sscanf(mem_line.data(), "Cached: %lu kB", &mem_cached);
-        else if (util::starts_with(mem_line, "Active:")) sscanf(mem_line.data(), "Active: %lu kB", &mem_active);
-        else if (util::starts_with(mem_line, "Inactive:")) sscanf(mem_line.data(), "Inactive: %lu kB", &mem_inactive);
-        else if (util::starts_with(mem_line, "Shmem:")) sscanf(mem_line.data(), "Shmem: %lu kB", &mem_shared);
+        if (util::starts_with(mem_line, "MemTotal:")) sscanf(mem_line.data(), "MemTotal: %" SCNu64 " kB", &mem_total);
+        else if (util::starts_with(mem_line, "MemFree:")) sscanf(mem_line.data(), "MemFree: %" SCNu64 " kB", &mem_free);
+        else if (util::starts_with(mem_line, "MemAvailable:")) sscanf(mem_line.data(), "MemAvailable: %" SCNu64 " kB", &mem_available);
+        else if (util::starts_with(mem_line, "Buffers:")) sscanf(mem_line.data(), "Buffers: %" SCNu64 " kB", &mem_buffers);
+        else if (util::starts_with(mem_line, "Cached:")) sscanf(mem_line.data(), "Cached: %" SCNu64 " kB", &mem_cached);
+        else if (util::starts_with(mem_line, "Active:")) sscanf(mem_line.data(), "Active: %" SCNu64 " kB", &mem_active);
+        else if (util::starts_with(mem_line, "Inactive:")) sscanf(mem_line.data(), "Inactive: %" SCNu64 " kB", &mem_inactive);
+        else if (util::starts_with(mem_line, "Shmem:")) sscanf(mem_line.data(), "Shmem: %" SCNu64 " kB", &mem_shared);
       }
 
       mem.setTotal(mem_total * 1024);

selfdrive/ui/linux.cc

@@ -43,7 +43,7 @@ FramebufferState* framebuffer_init(
   }
 
   glfwMakeContextCurrent(window);
-  glfwSwapInterval(0);
+  glfwSwapInterval(1);
 
   // clear screen
   glClearColor(0.2f, 0.2f, 0.2f, 1.0f);

selfdrive/ui/paint.cc

@@ -269,6 +269,7 @@ static void draw_frame(UIState *s) {
     glBindTexture(GL_TEXTURE_2D, s->frame_texs[s->cur_vision_idx]);
     #ifndef QCOM
       // TODO: a better way to do this?
+      //printf("%d\n", ((int*)s->priv_hnds[s->cur_vision_idx])[0]);
       glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, 1164, 874, 0, GL_RGB, GL_UNSIGNED_BYTE, s->priv_hnds[s->cur_vision_idx]);
     #endif
   }
@@ -807,10 +808,10 @@ static const char frame_fragment_shader[] =
   "#version 150 core\n"
   "precision mediump float;\n"
   "uniform sampler2D uTexture;\n"
-  "out vec4 vTexCoord;\n"
-  "out vec4 outColor;\n"
+  "in vec4 vTexCoord;\n"
+  "out vec4 colorOut;\n"
   "void main() {\n"
-  "  outColor = texture(uTexture, vTexCoord.xy);\n"
+  "  colorOut = texture(uTexture, vTexCoord.xy);\n"
   "}\n";
 #else
 static const char frame_vertex_shader[] =

selfdrive/ui/ui.cc

@@ -406,7 +406,7 @@ void handle_message(UIState *s, SubMaster &sm) {
     s->preview_started = data.getIsPreview();
   }
 
-  s->started = scene.thermal.getStarted() || s->preview_started ;
+  s->started = scene.thermal.getStarted() || s->preview_started;
   // Handle onroad/offroad transition
   if (!s->started) {
     if (s->status != STATUS_STOPPED) {
@@ -415,6 +415,13 @@ void handle_message(UIState *s, SubMaster &sm) {
       s->vision_seen = false;
       s->controls_seen = false;
       s->active_app = cereal::UiLayoutState::App::HOME;
+
+      #ifndef QCOM
+      // disconnect from visionipc on PC
+      close(s->ipc_fd);
+      s->ipc_fd = -1;
+      #endif
+
       update_offroad_layout_state(s);
     }
   } else if (s->status == STATUS_STOPPED) {
@@ -514,8 +521,14 @@ static void ui_update(UIState *s) {
 
   zmq_pollitem_t polls[1] = {{0}};
   // Take an rgb image from visiond if there is one
+  assert(s->ipc_fd >= 0);
   while(true) {
-    assert(s->ipc_fd >= 0);
+    if (s->ipc_fd < 0) {
+      // TODO: rethink this, for now it should only trigger on PC
+      LOGW("vision disconnected by other thread");
+      s->vision_connected = false;
+      return;
+    }
     polls[0].fd = s->ipc_fd;
     polls[0].events = ZMQ_POLLIN;
     #ifdef UI_60FPS

tools/nui/get_files_comma_api.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 import json
 import sys
 

tools/nui/nui

@@ -1,11 +1,18 @@
-#!/bin/sh
+#!/bin/bash -e
 
 if [ $# -gt 0 ]; then
-    if [ "$INTERNAL" = 1 ]; then
-        ./_nui "$1"
+  if [ "$INTERNAL" = 1 ]; then
+    ./_nui "$1"
+  else
+    ./get_files_comma_api.py $1 
+    if [ -f ./_nui ]; then
+      ./_nui use_api
+    elif [ -f _nui.app/Contents/MacOS/_nui ]; then
+      ./_nui.app/Contents/MacOS/_nui use_api
     else
-        python get_files_comma_api.py $1 && ./_nui use_api
+      echo "nui not found, please build it"
     fi
+  fi
 else
-    echo "Please Enter a Route"
+  echo "Please Enter a Route"
 fi

tools/replay/camera.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 import os
 
 from common.basedir import BASEDIR