From 0e21bc4336c6f9856ed39ebd9b18323b7119227b Mon Sep 17 00:00:00 2001 From: interplanetarychris Date: Thu, 7 Nov 2019 20:31:28 -0800 Subject: [PATCH 1/8] Update requirements.txt git+https://github.com/haavee/ppgplot.git@master Has problems finding arrayobject.h, but works with a manual build --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5bf8a36..cd71262 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,4 +12,5 @@ six spacetrack==0.13.0 termcolor zwoasi -git+https://github.com/haavee/ppgplot.git@master +# Has problems finding arrayobject.h, but works with a manual build +# git+https://github.com/haavee/ppgplot.git@master From 62a3f7b0d028c2590926168d999b623db3930f92 Mon Sep 17 00:00:00 2001 From: interplanetarychris Date: Sat, 9 Nov 2019 20:17:40 -0800 Subject: [PATCH 2/8] Implement queue to accomodate slow CPUs Also: * Introduces more error handling and caught exceptions * Allow user to specify a test duration with -t * Provide information when multiple ZWOASI cameras are present * Document image variables in compress() --- acquire.py | 603 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 352 insertions(+), 251 deletions(-) diff --git a/acquire.py b/acquire.py index 363be0c..78bcd25 100755 --- a/acquire.py +++ b/acquire.py @@ -19,77 +19,125 @@ import zwoasi as asi # Capture images from cv2 -def capture_cv2(buf, z1, t1, z2, t2, nx, ny, nz, tend, device_id, live): +def capture_cv2(image_queue, z1, t1, z2, t2, nx, ny, nz, tend, device_id, live): # Array flag first = True - # Initialize device + # Initialize cv2 device device = cv2.VideoCapture(device_id) # Set properties device.set(3, nx) device.set(4, ny) - # Loop until reaching end time - while float(time.time()) < tend: - # Get frames - for i in range(nz): - # Store start time - t0 = float(time.time()) + try: + # Loop until reaching end time + while float(time.time()) < tend: + # Wait for available capture buffer to become available + if (image_queue.qsize() > 1): + logger.warning("Acquiring data faster than your CPU can process") + slow_CPU = True + while (image_queue.qsize() > 1): + time.sleep(0.1) + if slow_CPU: + lost_video = time.time() - t + logger.info("Waited %.3fs for available capture buffer" % lost_video) + slow_CPU = False - # Get frame - res, frame = device.read() + # Get frames + for i in range(nz): + # Store start time + t0 = float(time.time()) - # Compute mid time - t = (float(time.time())+t0)/2.0 + # Get frame + res, frame = device.read() - # Skip lost frames - if res is True: - # Convert image to grayscale - z = np.asarray(cv2.cvtColor( - frame, cv2.COLOR_BGR2GRAY)).astype(np.uint8) + # Compute mid time + t = (float(time.time())+t0)/2.0 - # Display Frame - if live is True: - cv2.imshow("Capture", z) - cv2.waitKey(1) + # Skip lost frames + if res is True: + # Convert image to grayscale + z = np.asarray(cv2.cvtColor( + frame, cv2.COLOR_BGR2GRAY)).astype(np.uint8) - # Store results - if first: - z1[i] = z - t1[i] = t - else: - z2[i] = z - t2[i] = t + # Display Frame + if live is True: + cv2.imshow("Capture", z) + cv2.waitKey(1) - # Assign buffer ready - if first: - buf.value = 1 - else: - buf.value = 2 + # Store results + if first: + z1[i] = z + t1[i] = t + else: + z2[i] = z + t2[i] = t - # Swap flag - first = not first + if first: + buf = 1 + else: + buf = 2 + image_queue.put(buf) + logger.debug("Captured z%d" % buf) - # End capture - logger.info("Exiting capture") - device.release() + # Swap flag + first = not first + reason = "Session complete" + except KeyboardInterrupt: + print() + reason = "Keyboard interrupt" + except ValueError as e: + logger.error("%s" % e) + reason = "Wrong image dimensions? Fix nx, ny in config." + finally: + # End capture + logger.info("Capture: %s - Exiting" % reason) + device.release() # Capture images -def capture_asi(buf, z1, t1, z2, t2, nx, ny, nz, tend, device_id, live, gain, - maxgain, autogain, exposure, bins, brightness, bandwidth, - high_speed, hardware_bin, sdk): - # Array flag - first = True +def capture_asi(image_queue, z1, t1, z2, t2, nx, ny, nz, tend, device_id, live, cfg): + first = True # Array flag + slow_CPU = False # Performance issue flag + + camera_type = "ASI" + gain = cfg.getint(camera_type, 'gain') + maxgain = cfg.getint(camera_type, 'maxgain') + autogain = cfg.getboolean(camera_type, 'autogain') + exposure = cfg.getint(camera_type, 'exposure') + binning = cfg.getint(camera_type, 'bin') + brightness = cfg.getint(camera_type, 'brightness') + bandwidth = cfg.getint(camera_type, 'bandwidth') + high_speed = cfg.getint(camera_type, 'high_speed') + hardware_bin = cfg.getint(camera_type, 'hardware_bin') + sdk = cfg.get(camera_type, 'sdk') # Initialize device asi.init(sdk) + num_cameras = asi.get_num_cameras() + if num_cameras == 0: + logger.error("No ZWOASI cameras found") + raise ValueError + sys.exit() + + cameras_found = asi.list_cameras() # Models names of the connected cameras + + if num_cameras == 1: + device_id = 0 + logger.info("Found one camera: %s" % cameras_found[0]) + else: + logger.info("Found %d ZWOASI cameras" % num_cameras) + for n in range(num_cameras): + logger.info(" %d: %s" % (n, cameras_found[n])) + logger.info("Using #%d: %s" % (device_id, cameras_found[device_id])) + camera = asi.Camera(device_id) camera_info = camera.get_camera_property() - logger.info('ASI Camera info: %s' % camera_info) - logger.info(camera_info['MaxHeight']) + logger.debug("ASI Camera info:") + for (key, value) in camera_info.items(): + logger.debug(" %s : %s" % (key,value)) camera.set_control_value(asi.ASI_BANDWIDTHOVERLOAD, bandwidth) camera.disable_dark_subtract() @@ -108,183 +156,230 @@ def capture_asi(buf, z1, t1, z2, t2, nx, ny, nz, tend, device_id, live, gain, camera.start_video_capture() camera.set_image_type(asi.ASI_IMG_RAW8) - # Fix autogain - if autogain: - while True: - # Get frame - z = camera.capture_video_frame() - - # Break on no change in gain - settings = camera.get_control_values() - if gain == settings["Gain"]: - break - gain = settings["Gain"] - camera.set_control_value(asi.ASI_GAIN, gain, auto=autogain) - - # Loop until reaching end time - while float(time.time()) < tend: - # Get settings - settings = camera.get_control_values() - gain = settings["Gain"] - temp = settings["Temperature"]/10.0 - logging.info("Capturing frame with gain %d, temperature %.1f" % (gain, temp)) - - # Set gain + try: + # Fix autogain if autogain: - camera.set_control_value(asi.ASI_GAIN, gain, auto=autogain) + while True: + # Get frame + z = camera.capture_video_frame() - # Get frames - for i in range(nz): - # Store start time - t0 = float(time.time()) + # Break on no change in gain + settings = camera.get_control_values() + if gain == settings["Gain"]: + break + gain = settings["Gain"] + camera.set_control_value(asi.ASI_GAIN, gain, auto=autogain) - # Get frame - z = camera.capture_video_frame() + # Loop until reaching end time + while float(time.time()) < tend: + # Wait for available capture buffer to become available + if (image_queue.qsize() > 1): + logger.warning("Acquiring data faster than your CPU can process") + slow_CPU = True + while (image_queue.qsize() > 1): + time.sleep(0.1) + if slow_CPU: + lost_video = time.time() - t + logger.info("Waited %.3fs for available capture buffer" % lost_video) + slow_CPU = False - # Compute mid time - t = (float(time.time())+t0)/2.0 + # Get settings + settings = camera.get_control_values() + gain = settings["Gain"] + temp = settings["Temperature"]/10.0 + logger.info("Capturing frame with gain %d, temperature %.1f" % (gain, temp)) - # Display Frame - if live is True: - cv2.imshow("Capture", z) - cv2.waitKey(1) + # Set gain + if autogain: + camera.set_control_value(asi.ASI_GAIN, gain, auto=autogain) - # Store results - if first: - z1[i] = z - t1[i] = t + # Get frames + for i in range(nz): + # Store start time + t0 = float(time.time()) + + # Get frame + z = camera.capture_video_frame() + + # Compute mid time + t = (float(time.time())+t0)/2.0 + + # Display Frame + if live is True: + cv2.imshow("Capture", z) + cv2.waitKey(1) + + # Store results + if first: + z1[i] = z + t1[i] = t + else: + z2[i] = z + t2[i] = t + + if first: + buf = 1 else: - z2[i] = z - t2[i] = t + buf = 2 + image_queue.put(buf) + logger.debug("Captured z%d" % buf) - # Assign buffer ready - if first: - buf.value = 1 - else: - buf.value = 2 - - # Swap flag - first = not first - - # End capture - logger.info("Exiting capture") - camera.stop_video_capture() + # Swap flag + first = not first + reason = "Session complete" + except KeyboardInterrupt: + print() + reason = "Keyboard interrupt" + except ValueError as e: + logger.error("%s" % e) + reason = "Wrong image dimensions? Fix nx, ny in config." + except MemoryError as e: + logger.error("Capture: Memory error %s" % e) + finally: + # End capture + logger.info("Capture: %s - Exiting" % reason) + camera.stop_video_capture() + camera.close() -def compress(buf, z1, t1, z2, t2, nx, ny, nz, tend, path, device_id): - # Flag to keep track of processed buffer - process_buf = 1 +def compress(image_queue, z1, t1, z2, t2, nx, ny, nz, tend, path, device_id): + """ compress: Aggregate nframes of observations into a single FITS file, with statistics. + ImageHDU[0]: mean pixel value nframes (zmax) + ImageHDU[1]: standard deviation of nframes (zstd) + ImageHDU[2]: maximum pixel value of nframes (zmax) + ImageHDU[3]: maximum pixel value frame number (znum) + + Also updates a [observations_path]/control/state.txt for interfacing with satttools/runsched and sattools/slewto + """ # Force a restart controlpath = os.path.join(path, "control") if not os.path.exists(controlpath): - os.makedirs(controlpath) + try: + os.makedirs(controlpath) + except PermissionError: + logger.error("Can not create control path directory: %s" % controlpath) + raise with open(os.path.join(controlpath, "state.txt"), "w") as fp: fp.write("restart\n") - # Start processing - while True: - # Check mount state - restart = False - with open(os.path.join(controlpath, "state.txt"), "r") as fp: - line = fp.readline().rstrip() - if line == "restart": - restart = True + try: + # Start processing + while True: + # Check mount state + restart = False + with open(os.path.join(controlpath, "state.txt"), "r") as fp: + line = fp.readline().rstrip() + if line == "restart": + restart = True - # Restart - if restart: - # Log state - with open(os.path.join(controlpath, "state.txt"), "w") as fp: - fp.write("observing\n") + # Restart + if restart: + # Log state + with open(os.path.join(controlpath, "state.txt"), "w") as fp: + fp.write("observing\n") - # Get obsid - t = time.gmtime() - obsid = "%s_%d/%s" % (time.strftime("%Y%m%d", t), device_id, time.strftime("%H%M%S", t)) - filepath = os.path.join(path, obsid) - logger.info("Storing files in %s" % filepath) + # Get obsid + t = time.gmtime() + obsid = "%s_%d/%s" % (time.strftime("%Y%m%d", t), device_id, time.strftime("%H%M%S", t)) + filepath = os.path.join(path, obsid) + logger.info("Storing files in %s" % filepath) - # Wait for buffer to become available - while buf.value != process_buf: - time.sleep(1.0) + # Wait for completed capture buffer to become available + while (image_queue.qsize == 0): + time.sleep(0.1) - # Process first buffer - if buf.value == 1: - t = t1 - z = z1.astype('float32') - process_buf = 2 - elif buf.value == 2: - t = t2 - z = z2.astype('float32') - process_buf = 1 + # Get next buffer # from the work queue + proc_buffer = image_queue.get() + logger.debug("Processing z%d" % proc_buffer) - # Compute statistics - zmax = np.max(z, axis=0) - znum = np.argmax(z, axis=0) - zs1 = np.sum(z, axis=0)-zmax - zs2 = np.sum(z*z, axis=0)-zmax*zmax - zavg = zs1/float(nz-1) - zstd = np.sqrt((zs2-zs1*zavg)/float(nz-2)) + # Process first buffer + if proc_buffer == 1: + t = t1 + z = z1.astype('float32') + elif proc_buffer == 2: + t = t2 + z = z2.astype('float32') - # Convert to float and flip - zmax = np.flipud(zmax.astype('float32')) - znum = np.flipud(znum.astype('float32')) - zavg = np.flipud(zavg.astype('float32')) - zstd = np.flipud(zstd.astype('float32')) + # Compute statistics + zmax = np.max(z, axis=0) + znum = np.argmax(z, axis=0) + zs1 = np.sum(z, axis=0)-zmax + zs2 = np.sum(z*z, axis=0)-zmax*zmax + zavg = zs1/float(nz-1) + zstd = np.sqrt((zs2-zs1*zavg)/float(nz-2)) - # Format time - nfd = "%s.%03d" % (time.strftime("%Y-%m-%dT%T", - time.gmtime(t[0])), int((t[0]-np.floor(t[0]))*1000)) - t0 = Time(nfd, format='isot') - dt = t-t[0] + # Convert to float and flip + zmax = np.flipud(zmax.astype('float32')) + znum = np.flipud(znum.astype('float32')) + zavg = np.flipud(zavg.astype('float32')) + zstd = np.flipud(zstd.astype('float32')) - # Generate fits - fname = "%s.fits" % nfd + # Format time + nfd = "%s.%03d" % (time.strftime("%Y-%m-%dT%T", + time.gmtime(t[0])), int((t[0]-np.floor(t[0]))*1000)) + t0 = Time(nfd, format='isot') + dt = t-t[0] - # Format header - hdr = fits.Header() - hdr['DATE-OBS'] = "%s" % nfd - hdr['MJD-OBS'] = t0.mjd - hdr['EXPTIME'] = dt[-1]-dt[0] - hdr['NFRAMES'] = nz - hdr['CRPIX1'] = float(nx)/2.0 - hdr['CRPIX2'] = float(ny)/2.0 - hdr['CRVAL1'] = 0.0 - hdr['CRVAL2'] = 0.0 - hdr['CD1_1'] = 1.0/3600.0 - hdr['CD1_2'] = 0.0 - hdr['CD2_1'] = 0.0 - hdr['CD2_2'] = 1.0/3600.0 - hdr['CTYPE1'] = "RA---TAN" - hdr['CTYPE2'] = "DEC--TAN" - hdr['CUNIT1'] = "deg" - hdr['CUNIT2'] = "deg" - hdr['CRRES1'] = 0.0 - hdr['CRRES2'] = 0.0 - hdr['EQUINOX'] = 2000.0 - hdr['RADECSYS'] = "ICRS" - hdr['COSPAR'] = cfg.getint('Common', 'observer_cospar') - hdr['OBSERVER'] = cfg.get('Common', 'observer_name') - for i in range(nz): - hdr['DT%04d' % i] = dt[i] - for i in range(10): - hdr['DUMY%03d' % i] = 0.0 + # Generate fits + fname = "%s.fits" % nfd - # Create output directory - if not os.path.exists(filepath): - os.makedirs(filepath) + # Format header + hdr = fits.Header() + hdr['DATE-OBS'] = "%s" % nfd + hdr['MJD-OBS'] = t0.mjd + hdr['EXPTIME'] = dt[-1]-dt[0] + hdr['NFRAMES'] = nz + hdr['CRPIX1'] = float(nx)/2.0 + hdr['CRPIX2'] = float(ny)/2.0 + hdr['CRVAL1'] = 0.0 + hdr['CRVAL2'] = 0.0 + hdr['CD1_1'] = 1.0/3600.0 + hdr['CD1_2'] = 0.0 + hdr['CD2_1'] = 0.0 + hdr['CD2_2'] = 1.0/3600.0 + hdr['CTYPE1'] = "RA---TAN" + hdr['CTYPE2'] = "DEC--TAN" + hdr['CUNIT1'] = "deg" + hdr['CUNIT2'] = "deg" + hdr['CRRES1'] = 0.0 + hdr['CRRES2'] = 0.0 + hdr['EQUINOX'] = 2000.0 + hdr['RADECSYS'] = "ICRS" + hdr['COSPAR'] = cfg.getint('Common', 'observer_cospar') + hdr['OBSERVER'] = cfg.get('Common', 'observer_name') + for i in range(nz): + hdr['DT%04d' % i] = dt[i] + for i in range(10): + hdr['DUMY%03d' % i] = 0.0 - # Write fits file - hdu = fits.PrimaryHDU(data=np.array([zavg, zstd, zmax, znum]), - header=hdr) - hdu.writeto(os.path.join(filepath, fname)) - logger.info("Compressed %s" % fname) + # Create output directory + if not os.path.exists(filepath): + try: + os.makedirs(filepath) + except PermissionError: + logger.error("Can not create output directory: %s" % filepath) + raise - # Exit on end of capture - if t[-1] > tend: - break + # Write fits file + hdu = fits.PrimaryHDU(data=np.array([zavg, zstd, zmax, znum]), + header=hdr) + hdu.writeto(os.path.join(filepath, fname)) + logger.info("Compressed %s" % fname) - # Exiting - logger.info("Exiting compress") + # Exit on end of capture + if t[-1] > tend: + break + logger.debug("Processed z%d" % proc_buffer) + + + except KeyboardInterrupt: + pass + except MemoryError as e: + logger.error("Compress: Memory error %s" % e) + finally: + # Exiting + logger.info("Exiting compress") # Main function @@ -297,8 +392,12 @@ if __name__ == '__main__': help="Specify configuration file. If no file" + " is specified 'configuration.ini' is used.", metavar="FILE") - conf_parser.add_argument('-t', '--test', action='store_true', - help='Testing mode - Start capturing immediately') + conf_parser.add_argument('-t', '--test', + nargs='?', + action='store', + default=False, + help='Testing mode - Start capturing immediately for (optional) seconds', + metavar="s") conf_parser.add_argument('-l', '--live', action='store_true', help='Display live image while capturing') @@ -306,45 +405,28 @@ if __name__ == '__main__': # Process commandline options and parse configuration cfg = configparser.ConfigParser(inline_comment_prefixes=('#', ';')) - if args.conf_file: - cfg.read([args.conf_file]) - else: - cfg.read('configuration.ini') + + conf_file = args.conf_file if args.conf_file else "configuration.ini" + result = cfg.read([conf_file]) - # Testing mode - if args.test: - testing = True - else: - testing = False - - # Live mode - if args.live: - live = True - else: - live = False - - # Get camera type - camera_type = cfg.get('Camera', 'camera_type') - - # Get device id - device_id = cfg.getint(camera_type, 'device_id') - - # Get camera type - camera_type = cfg.get('Camera', 'camera_type') - - # Current time - tnow = Time.now() - - # Generate directory - path = os.path.abspath(cfg.get('Common', 'observations_path')) - if not os.path.exists(path): - os.makedirs(path) + if not result: + print("Could not read config file: %s\nExiting..." % conf_file) + sys.exit() # Setup logging logFormatter = logging.Formatter("%(asctime)s [%(threadName)-12.12s] " + "[%(levelname)-5.5s] %(message)s") logger = logging.getLogger() + # Generate directory + path = os.path.abspath(cfg.get('Common', 'observations_path')) + if not os.path.exists(path): + try: + os.makedirs(path) + except PermissionError: + logger.error("Can not create observations_path: %s" % path) + sys.exit() + fileHandler = logging.FileHandler(os.path.join(path, "acquire.log")) fileHandler.setFormatter(logFormatter) logger.addHandler(fileHandler) @@ -354,6 +436,32 @@ if __name__ == '__main__': logger.addHandler(consoleHandler) logger.setLevel(logging.DEBUG) + logger.info("Using config: %s" % conf_file) + + # Testing mode + if args.test is None: + test_duration = 31 + testing = True + elif args.test is not False: + test_duration = int(args.test) + testing = True + logger.info("Test mode: %s" % testing) + if (testing): + logger.info("Test duration: %ds" % test_duration) + + # Live mode + live = True if args.live else False + logger.info("Live mode: %s" % live) + + # Get camera type + camera_type = cfg.get('Camera', 'camera_type') + + # Get device id + device_id = cfg.getint(camera_type, 'device_id') + + # Current time + tnow = Time.now() + # Set location loc = EarthLocation(lat=cfg.getfloat('Common', 'observer_lat')*u.deg, lon=cfg.getfloat('Common', 'observer_lon')*u.deg, @@ -361,9 +469,11 @@ if __name__ == '__main__': if not testing: # Reference altitudes - refalt_set = cfg.getfloat('Control', 'alt_sunset')*u.deg + refalt_set = cfg.getfloat('Control', 'alt_sunset')*u.deg refalt_rise = cfg.getfloat('Control', 'alt_sunrise')*u.deg + # FIXME: The following will fail without internet access + # due to failure to download finals2000A.all # Get sunrise and sunset times state, tset, trise = get_sunset_and_sunrise(tnow, loc, refalt_set, refalt_rise) @@ -388,26 +498,15 @@ if __name__ == '__main__': except KeyboardInterrupt: sys.exit() else: - tend = tnow + 31.0*u.s + tend = tnow + test_duration*u.s - logger.info("Starting data acquisition.") - logger.info("Acquisition will end at "+tend.isot) + logger.info("Starting data acquisition") + logger.info("Acquisition will end after "+tend.isot) # Get settings nx = cfg.getint(camera_type, 'nx') ny = cfg.getint(camera_type, 'ny') nz = cfg.getint(camera_type, 'nframes') - if camera_type == "ASI": - gain = cfg.getint(camera_type, 'gain') - maxgain = cfg.getint(camera_type, 'maxgain') - autogain = cfg.getboolean(camera_type, 'autogain') - exposure = cfg.getint(camera_type, 'exposure') - binning = cfg.getint(camera_type, 'bin') - brightness = cfg.getint(camera_type, 'brightness') - bandwidth = cfg.getint(camera_type, 'bandwidth') - sdk = cfg.get(camera_type, 'sdk') - high_speed = cfg.getint(camera_type, 'high_speed') - hardware_bin = cfg.getint(camera_type, 'hardware_bin') # Initialize arrays z1base = multiprocessing.Array(ctypes.c_uint8, nx*ny*nz) @@ -418,23 +517,21 @@ if __name__ == '__main__': z2 = np.ctypeslib.as_array(z2base.get_obj()).reshape(nz, ny, nx) t2base = multiprocessing.Array(ctypes.c_double, nz) t2 = np.ctypeslib.as_array(t2base.get_obj()) - buf = multiprocessing.Value('i', 0) + + image_queue = multiprocessing.Queue() # Set processes pcompress = multiprocessing.Process(target=compress, - args=(buf, z1, t1, z2, t2, nx, ny, + args=(image_queue, z1, t1, z2, t2, nx, ny, nz, tend.unix, path, device_id)) if camera_type == "CV2": pcapture = multiprocessing.Process(target=capture_cv2, - args=(buf, z1, t1, z2, t2, + args=(image_queue, z1, t1, z2, t2, nx, ny, nz, tend.unix, device_id, live)) elif camera_type == "ASI": pcapture = multiprocessing.Process(target=capture_asi, - args=(buf, z1, t1, z2, t2, - nx, ny, nz, tend.unix, device_id, live, gain, - maxgain, autogain, exposure, binning, - brightness, bandwidth, high_speed, hardware_bin, - sdk)) + args=(image_queue, z1, t1, z2, t2, + nx, ny, nz, tend.unix, device_id, live, cfg)) # Start pcapture.start() @@ -444,10 +541,14 @@ if __name__ == '__main__': try: pcapture.join() pcompress.join() - except KeyboardInterrupt: + except (KeyboardInterrupt, ValueError): + time.sleep(0.1) # Allow a little time for a graceful exit + except MemoryError as e: + logger.error("Memory error %s" % e) + finally: pcapture.terminate() pcompress.terminate() # Release device if live is True: - cv2.destroyAllWindows() + cv2.destroyAllWindows() \ No newline at end of file From b611d4bbf1129a5c2b57b901d33f919d75d242ad Mon Sep 17 00:00:00 2001 From: interplanetarychris Date: Sat, 9 Nov 2019 20:38:15 -0800 Subject: [PATCH 3/8] Add missing houghrmin and nhoughmin cfg parms --- configuration.ini-dist | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/configuration.ini-dist b/configuration.ini-dist index 649334b..a9fc013 100644 --- a/configuration.ini-dist +++ b/configuration.ini-dist @@ -50,8 +50,10 @@ low_app = 18 # Arcsec per pixel low scale - Resolution of camera high_app = 20 # Arcsec per pixel high scale - Resolution of camera [Processing] +nstarsmin = 10 drdtmin = 5.0 # Minimum predicted velocity (pixels/s) -trkrmin = 10.0 # Track selection region around prediction (pixels) trksig = 5.0 # Track selection sigma ntrkmin = 10 # Minimum track points -nstarsmin = 10 +trkrmin = 10.0 # Track selection region around prediction (pixels) +houghrmin = 10.0 # Hough3dLines: Track selection region around prediction (pixels) +nhoughmin = 10 # Hough3dLines: Minimum track points \ No newline at end of file From 8b641a9e9383e6e57a2ed26b0c1de48ecd19db65 Mon Sep 17 00:00:00 2001 From: interplanetarychris Date: Sat, 9 Nov 2019 20:51:09 -0800 Subject: [PATCH 4/8] Update requirements.txt --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index cd71262..5d197f5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,5 +12,5 @@ six spacetrack==0.13.0 termcolor zwoasi -# Has problems finding arrayobject.h, but works with a manual build -# git+https://github.com/haavee/ppgplot.git@master +# Has problems finding arrayobject.h, but works with a manual build, or by calling pip -r requirements.txt twice +git+https://github.com/haavee/ppgplot.git@master From f71e889a6b71a19e262d687d12938727e8099ae2 Mon Sep 17 00:00:00 2001 From: interplanetarychris Date: Sat, 9 Nov 2019 22:18:34 -0800 Subject: [PATCH 5/8] Implements option to use only astrometry.net --- calibrate.py | 32 +++++++++++++++++++------------- configuration.ini-dist | 1 + 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/calibrate.py b/calibrate.py index 5c45aad..47faa04 100755 --- a/calibrate.py +++ b/calibrate.py @@ -22,10 +22,12 @@ if __name__ == '__main__': # Process commandline options and parse configuration cfg = configparser.ConfigParser(inline_comment_prefixes=('#', ';')) - if args.conf_file: - cfg.read([args.conf_file]) - else: - cfg.read('configuration.ini') + conf_file = args.conf_file if args.conf_file else "configuration.ini" + result = cfg.read([conf_file]) + + if not result: + print("Could not read config file: %s\nExiting..." % conf_file) + sys.exit() path = args.file_dir extension = 'fits' @@ -37,17 +39,21 @@ if __name__ == '__main__': print("Found " + file_for_astrometry + " for astrometric solving.") sex_config = cfg.get('Astrometry', 'sex_config') - low_app = cfg.get('Astrometry', 'low_app') - high_app = cfg.get('Astrometry', 'high_app') + no_sex = cfg.get('Astrometry', 'no_sex') + low_app = cfg.get('Astrometry', 'low_app') + high_app = cfg.get('Astrometry', 'high_app') - # Format command - command = "solve-field -O -y -u app -L %s -H %s --downsample 2 " % (low_app, high_app) + \ - "--use-sextractor --sextractor-config %s --x-column X_IMAGE " % sex_config + \ - "--y-column Y_IMAGE --sort-column MAG_AUTO --sort-ascending " + \ - "--no-plots -T -N %s/test.fits %s" % (path, file_for_astrometry) + # Format solve-field command + command = "solve-field -O -y -u app -L %s -H %s --downsample 2 " % (low_app, high_app) + if (not no_sex): + command = command + \ + "--use-sextractor --sextractor-config %s --x-column X_IMAGE " % sex_config + \ + "--y-column Y_IMAGE --sort-column MAG_AUTO --sort-ascending " + command = command + \ + "--no-plots -T -N %s/test.fits %s" % (path, file_for_astrometry) - # Run sextractor + # Run solve-field subprocess.run(command, shell=True, stderr=subprocess.STDOUT) else: - print("No fits file found for astrometric solving.") + print("No fits file found for astrometric solving.") \ No newline at end of file diff --git a/configuration.ini-dist b/configuration.ini-dist index 649334b..628bc86 100644 --- a/configuration.ini-dist +++ b/configuration.ini-dist @@ -46,6 +46,7 @@ sdk = /path/to/libASICamera2.so # path to the SDK library [Astrometry] sex_config = /path/to/solve.sex +no_sex = False # Set to True to use only astrometry.net low_app = 18 # Arcsec per pixel low scale - Resolution of camera high_app = 20 # Arcsec per pixel high scale - Resolution of camera From b5cba659e4add1d53663cde9a2b9377083935ded Mon Sep 17 00:00:00 2001 From: interplanetarychris Date: Sat, 9 Nov 2019 22:43:06 -0800 Subject: [PATCH 6/8] Fallback to copyfile if copy2 has permission errs --- stvid/extract.py | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/stvid/extract.py b/stvid/extract.py index 1b377ba..3c0fbd6 100644 --- a/stvid/extract.py +++ b/stvid/extract.py @@ -166,11 +166,18 @@ def store_results(ident, fname, path, iod_line): # Copy files pngfile = fname.replace(".fits", "_%05d.png" % ident.norad) - shutil.copy2(fname, dest) - shutil.copy2(fname + ".cat", dest) - shutil.copy2(fname + ".cal", dest) - shutil.copy2(fname + ".id", dest) - shutil.copy2(fname + ".png", dest) + try: + shutil.copy2(fname, dest) + shutil.copy2(fname + ".cat", dest) + shutil.copy2(fname + ".cal", dest) + shutil.copy2(fname + ".id", dest) + shutil.copy2(fname + ".png", dest) + except PermissionError: + shutil.copyfile(fname, os.path.join(dest,fname)) + shutil.copyfile(fname + ".cat", os.path.join(dest, fname + ".cat")) + shutil.copyfile(fname + ".cal", os.path.join(dest, fname + ".cal")) + shutil.copyfile(fname + ".id", os.path.join(dest, fname + ".id")) + shutil.copyfile(fname + ".png", os.path.join(dest, fname + ".png")) if os.path.exists(pngfile): shutil.move(pngfile, os.path.join(dest, pngfile)) @@ -187,12 +194,18 @@ def store_not_seen(ident, fname, path): dest = os.path.join(path, "not_seen") # Copy files - shutil.copy2(fname, dest) - shutil.copy2(fname + ".cat", dest) - shutil.copy2(fname + ".cal", dest) - shutil.copy2(fname + ".id", dest) - shutil.copy2(fname + ".png", dest) - + try: + shutil.copy2(fname, dest) + shutil.copy2(fname + ".cat", dest) + shutil.copy2(fname + ".cal", dest) + shutil.copy2(fname + ".id", dest) + shutil.copy2(fname + ".png", dest) + except PermissionError: + shutil.copyfile(fname, os.path.join(dest,fname)) + shutil.copyfile(fname + ".cat", os.path.join(dest, fname + ".cat")) + shutil.copyfile(fname + ".cal", os.path.join(dest, fname + ".cal")) + shutil.copyfile(fname + ".id", os.path.join(dest, fname + ".id")) + shutil.copyfile(fname + ".png", os.path.join(dest, fname + ".png")) return From 965438089b74a38efca752903963684c686b5121 Mon Sep 17 00:00:00 2001 From: Cees Bassa Date: Sun, 10 Nov 2019 12:33:42 +0100 Subject: [PATCH 7/8] Fix operation if not in testing mode --- acquire.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/acquire.py b/acquire.py index 78bcd25..c8a02ee 100755 --- a/acquire.py +++ b/acquire.py @@ -222,7 +222,7 @@ def capture_asi(image_queue, z1, t1, z2, t2, nx, ny, nz, tend, device_id, live, else: buf = 2 image_queue.put(buf) - logger.debug("Captured z%d" % buf) + logger.debug("Captured buffer %d" % buf) # Swap flag first = not first @@ -291,7 +291,7 @@ def compress(image_queue, z1, t1, z2, t2, nx, ny, nz, tend, path, device_id): # Get next buffer # from the work queue proc_buffer = image_queue.get() - logger.debug("Processing z%d" % proc_buffer) + logger.debug("Processing buffer %d" % proc_buffer) # Process first buffer if proc_buffer == 1: @@ -370,7 +370,7 @@ def compress(image_queue, z1, t1, z2, t2, nx, ny, nz, tend, path, device_id): # Exit on end of capture if t[-1] > tend: break - logger.debug("Processed z%d" % proc_buffer) + logger.debug("Processed buffer %d" % proc_buffer) except KeyboardInterrupt: @@ -445,6 +445,8 @@ if __name__ == '__main__': elif args.test is not False: test_duration = int(args.test) testing = True + else: + testing = False logger.info("Test mode: %s" % testing) if (testing): logger.info("Test duration: %ds" % test_duration) @@ -551,4 +553,4 @@ if __name__ == '__main__': # Release device if live is True: - cv2.destroyAllWindows() \ No newline at end of file + cv2.destroyAllWindows() From 648c54b0a1a37fead63838e71f069d9b47ba3c77 Mon Sep 17 00:00:00 2001 From: Cees Bassa Date: Sun, 10 Nov 2019 12:42:55 +0100 Subject: [PATCH 8/8] Update processing description --- configuration.ini-dist | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/configuration.ini-dist b/configuration.ini-dist index a9fc013..51945d5 100644 --- a/configuration.ini-dist +++ b/configuration.ini-dist @@ -50,10 +50,10 @@ low_app = 18 # Arcsec per pixel low scale - Resolution of camera high_app = 20 # Arcsec per pixel high scale - Resolution of camera [Processing] -nstarsmin = 10 -drdtmin = 5.0 # Minimum predicted velocity (pixels/s) -trksig = 5.0 # Track selection sigma -ntrkmin = 10 # Minimum track points +drdtmin = 5.0 # Minimum predicted satellite velocity (pixels/s) trkrmin = 10.0 # Track selection region around prediction (pixels) +ntrkmin = 10 # Minimum number of points making up a track +trksig = 5.0 # Track selection sigma +nstarsmin = 10 # Minimum number of stars to attempt astrometric calibration houghrmin = 10.0 # Hough3dLines: Track selection region around prediction (pixels) nhoughmin = 10 # Hough3dLines: Minimum track points \ No newline at end of file