Skip to content

Feature/simulation #5

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
blankey1337 merged 3 commits into from
Nov 28, 2025
Merged

Feature/simulation #5

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
1a7002d
Add URDF model and visualization script
blankey1337 Nov 28, 2025
e1bc3ec
Add Web Dashboard and update ZMQ to PUB/SUB
blankey1337 Nov 28, 2025
4bd1896
Add Kinematic Simulator
blankey1337 Nov 28, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 84 additions & 0 deletions software/dashboard/app.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
import json
import threading
import time
import base64
import zmq
import cv2
import numpy as np
from flask import Flask, render_template, Response, jsonify

app = Flask(__name__)

# Global state
latest_observation = {}
lock = threading.Lock()
connected = False

def zmq_worker(ip='127.0.0.1', port=5556):
global latest_observation, connected
context = zmq.Context()
socket = context.socket(zmq.SUB)
socket.setsockopt(zmq.SUBSCRIBE, b"")
socket.connect(f"tcp://{ip}:{port}")
socket.setsockopt(zmq.CONFLATE, 1)

print(f"Connecting to ZMQ Stream at {ip}:{port}...")

while True:
try:
msg = socket.recv_string()
data = json.loads(msg)

with lock:
latest_observation = data
connected = True
except Exception as e:
print(f"Error in ZMQ worker: {e}")
connected = False
time.sleep(1)

def generate_frames(camera_name):
while True:
frame_data = None
with lock:
if camera_name in latest_observation:
b64_str = latest_observation[camera_name]
if b64_str:
try:
# It is already a base64 encoded JPG from the host
frame_data = base64.b64decode(b64_str)
except Exception:
pass

if frame_data:
yield (b'--frame\r\n'
b'Content-Type: image/jpeg\r\n\r\n' + frame_data + b'\r\n')
else:
# Return a blank or placeholder image if no data
pass

time.sleep(0.05) # Limit FPS for browser

@app.route('/')
def index():
return render_template('index.html')

@app.route('/video_feed/<camera_name>')
def video_feed(camera_name):
return Response(generate_frames(camera_name),
mimetype='multipart/x-mixed-replace; boundary=frame')

@app.route('/api/status')
def get_status():
with lock:
# Filter out large image data for status endpoint
status = {k: v for k, v in latest_observation.items() if not (isinstance(v, str) and len(v) > 1000)}
status['connected'] = connected
return jsonify(status)

if __name__ == '__main__':
# Start ZMQ thread
t = threading.Thread(target=zmq_worker, daemon=True)
t.start()

app.run(host='0.0.0.0', port=5000, debug=False)
88 changes: 88 additions & 0 deletions software/dashboard/templates/index.html
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>AlohaMini Dashboard</title>
<style>
body { font-family: sans-serif; background: #222; color: #eee; margin: 0; padding: 20px; }
.container { display: flex; flex-wrap: wrap; gap: 20px; }
.camera-grid { display: flex; flex-wrap: wrap; gap: 10px; flex: 2; }
.camera-box { background: #333; padding: 5px; border-radius: 5px; }
.camera-box img { max-width: 100%; height: auto; display: block; }
.camera-title { text-align: center; font-size: 0.9em; margin-bottom: 5px; }
.status-panel { flex: 1; background: #333; padding: 15px; border-radius: 5px; min-width: 300px; }
table { width: 100%; border-collapse: collapse; }
td, th { padding: 5px; border-bottom: 1px solid #444; font-size: 0.9em; }
th { text-align: left; color: #aaa; }
.value { font-family: monospace; color: #4f4; }
</style>
</head>
<body>
<h1>AlohaMini Dashboard</h1>

<div class="container">
<div class="camera-grid" id="cameraGrid">
<!-- Cameras will be injected here -->
</div>

<div class="status-panel">
<h3>Robot State</h3>
<table id="statusTable">
<!-- Status rows injected here -->
</table>
</div>
</div>

<script>
const knownCameras = ['head_top', 'head_back', 'head_front', 'wrist_left', 'wrist_right', 'cam_high', 'cam_low', 'cam_left_wrist', 'cam_right_wrist'];
const activeCameras = new Set();

function updateStatus() {
fetch('/api/status')
.then(response => response.json())
.then(data => {
const table = document.getElementById('statusTable');
let rows = '';

// Update Status Table
for (const [key, value] of Object.entries(data)) {
// Skip camera data (if any leaks into status) and long strings
if (typeof value === 'string' && value.length > 100) continue;

// Check if this key might be a camera
// If it's a camera key and not in activeCameras, add it
if (knownCameras.includes(key) || key.includes('cam') || key.includes('wrist')) {
if (!activeCameras.has(key) && !document.getElementById('cam-' + key)) {
addCamera(key);
activeCameras.add(key);
}
continue; // Don't show camera keys in text table
}

let displayValue = value;
if (typeof value === 'number') displayValue = value.toFixed(2);

rows += `<tr><th>${key}</th><td class="value">${displayValue}</td></tr>`;
}
table.innerHTML = rows;
})
.catch(err => console.error("Error fetching status:", err));
}

function addCamera(name) {
const grid = document.getElementById('cameraGrid');
const div = document.createElement('div');
div.className = 'camera-box';
div.id = 'cam-' + name;
div.innerHTML = `
<div class="camera-title">${name}</div>
<img src="/video_feed/${name}" alt="${name}" width="320" height="240">
`;
grid.appendChild(div);
}

setInterval(updateStatus, 1000);
updateStatus();
</script>
</body>
</html>
31 changes: 15 additions & 16 deletions software/examples/alohamini/teleoperate_bi.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,15 @@
import os
import time

from lerobot.robots.alohamini import LeKiwiClient, LeKiwiClientConfig
from lerobot.robots.alohamini import LeKiwiClient, LeKiwiClientConfig, LeKiwiSim
from lerobot.teleoperators.keyboard.teleop_keyboard import KeyboardTeleop, KeyboardTeleopConfig
from lerobot.teleoperators.bi_so100_leader import BiSO100Leader, BiSO100LeaderConfig
from lerobot.utils.robot_utils import busy_wait
from lerobot.utils.visualization_utils import init_rerun, log_rerun_data

# ============ Parameter Section ============ #
parser = argparse.ArgumentParser()
parser.add_argument("--use_dummy", action="store_true", help="Do not connect robot, only print actions")
parser.add_argument("--use_dummy", action="store_true", help="Do not connect robot, use simulation")
parser.add_argument("--fps", type=int, default=30, help="Main loop frequency (frames per second)")
parser.add_argument("--remote_ip", type=str, default="127.0.0.1", help="LeKiwi host IP address")
parser.add_argument("--left_arm_port", type=str, default="/dev/am_arm_leader_left", help="Left leader arm port")
Expand All @@ -23,9 +23,6 @@
FPS = args.fps
# ========================================== #

if USE_DUMMY:
print("🧪 USE_DUMMY mode enabled: robot will not connect, only print actions.")

# Create configs
robot_config = LeKiwiClientConfig(remote_ip=args.remote_ip, id="my_alohamini")
bi_cfg = BiSO100LeaderConfig(
Expand All @@ -36,14 +33,15 @@
leader = BiSO100Leader(bi_cfg)
keyboard_config = KeyboardTeleopConfig(id="my_laptop_keyboard")
keyboard = KeyboardTeleop(keyboard_config)
robot = LeKiwiClient(robot_config)

# Connection logic
if not USE_DUMMY:
robot.connect()
if USE_DUMMY:
print("🧪 USE_DUMMY mode enabled: Using LeKiwiSim.")
robot = LeKiwiSim(robot_config)
else:
print("🧪 robot.connect() skipped, only printing actions.")
robot = LeKiwiClient(robot_config)

# Connection logic
robot.connect() # Sim connect or Client connect
leader.connect()
keyboard.connect()

Expand All @@ -56,21 +54,22 @@
while True:
t0 = time.perf_counter()

observation = robot.get_observation() if not USE_DUMMY else {}
# Get observation (Sim provides real-time state, Client provides remote state)
observation = robot.get_observation()

arm_actions = leader.get_action()
arm_actions = {f"arm_{k}": v for k, v in arm_actions.items()}
keyboard_keys = keyboard.get_action()

# Use robot-specific mapping (works for Client and Sim now)
base_action = robot._from_keyboard_to_base_action(keyboard_keys)
lift_action = robot._from_keyboard_to_lift_action(keyboard_keys)

action = {**arm_actions, **base_action, **lift_action}
log_rerun_data(observation, action)

if USE_DUMMY:
print(f"[USE_DUMMY] action → {action}")
else:
robot.send_action(action)
print(f"Sent action → {action}")
# Send action (Sim updates state, Client sends to host)
robot.send_action(action)
print(f"Sent action → {action}")

busy_wait(max(1.0 / FPS - (time.perf_counter() - t0), 0.0))
Loading
Loading