feat: 增加扫黑除暴假动作

feat: 换用 deepseek 大模型
pref: 修改应急避险2停车条件

app.py

@@ -10,14 +10,14 @@ import os
 import time
 import subprocess
 import signal
-import importlib
+import base64
+import json
 from main_upper import main_func
 server_command = [
     {"path": "/home/evan/Workplace/project_capture/build/", "script": "./capture"},
     {"path": "/home/evan/Workplace/project_infer/lane_server/", "script": "lane_infer_server.py"},
-    # {"path": "/home/evan/Workplace/project_infer/lane_server/", "script": "lane_infer_server1.py"},
     {"path": "/home/evan/Workplace/project_infer/yolo_server/", "script": "yolo_infer_server.py"},
-
+    {"path": "/home/evan/Workplace/project_infer/ocr_server/", "script": "ocr_infer_server.py"},
 ]
 processes = []
 
@@ -52,7 +52,7 @@ logger.add(handler, format="{time:MM-DD HH:mm:ss} {message}", level="DEBUG")
 fileOptions_path = '/home/evan/Workplace/project_main'
 fileOptions_list = ['cfg_args.toml','cfg_main.toml', 'cfg_subtask.toml']
 cfg_args_path = os.path.join(fileOptions_path, 'cfg_args.toml')
-
+cfg_move_area_path = os.path.join(fileOptions_path, 'cfg_move_area.json')
 
 @app.route('/')
 def index():
@@ -65,6 +65,17 @@ def run():
     config_args['lane_mode']['mode_index'] = int(mode_index)
     with open(cfg_args_path, 'w') as config_file:
         toml.dump(config_args, config_file)
+    try:
+        action_base64 = request.args.get('action')
+        decoded_bytes = base64.b64decode(action_base64)
+        decoded_str = decoded_bytes.decode('utf-8')
+        json_data = json.loads(decoded_str)
+        with open(cfg_move_area_path, 'w') as json_file:
+            json.dump(json_data, json_file)
+    except:
+        # 当该字段没有传入参数时 清空配置文件 该任务按照正常流程去做
+        with open(cfg_move_area_path, 'w') as json_file:
+            pass
     return render_template('index2.html')
 
 # @app.route('/csdn')

cfg_args.toml

@@ -10,5 +10,5 @@ UpTower_enable = true
 GetRBall_enable = true
 PutBBall_enable = true
 PutHanoi_enable = true
-MoveArea_enable = false
+MoveArea_enable = true
 KickAss_enable = true

cfg_main.toml

@@ -1,6 +1,6 @@
 [debug]
 logger_filename = "log/file_{time}.log"
-logger_format = "{time} {level} {message}"
+logger_format = "[{level}] {file}:{line} <{time}> {message}"
 
 
 
@@ -13,7 +13,7 @@ GetRBall_counts = 10
 PutBBall_counts = 15
 PutHanoi1_counts = 7
 PutHanoi2_counts = 2
-PutHanoi3_counts = 5
+PutHanoi3_counts = 2
 MoveArea1_counts = 6
-MoveArea2_counts = 1700
+MoveArea2_counts = 10
 KickAss_counts = 10

cfg_subtask.toml

@@ -20,17 +20,17 @@ pid_ki = 0
 pid_kd = 0
 
 [get_rball]
-pid_kp = 1.5
+pid_kp = 1.0
 pid_ki = 0
 pid_kd = 0
 
 [put_bball]
-pid_kp = 2.0
+pid_kp = 1.5
 pid_ki = 0
 pid_kd = 0
 
 [put_hanoi1]
-pid_kp = 0.7
+pid_kp = 0.5
 pid_ki = 0
 pid_kd = 0
 
@@ -42,7 +42,7 @@ pos_gap = 160
 first_target = "mp"
 
 [put_hanoi3]
-pid_kp = 1.7
+pid_kp = 1.5
 pid_ki = 0
 pid_kd = 0
 
@@ -50,7 +50,7 @@ pid_kd = 0
 pid_kp = 1.2
 pid_ki = 0
 pid_kd = 0
-llm_enable = false
+llm_enable = true
 
 [kick_ass]
 pid_kp = 0.8

subtask.py

@@ -2,7 +2,8 @@ from enum import Enum
 from loguru import logger
 from utils import label_filter
 from utils import tlabel
-from utils import LLM
+# from utils import LLM
+from utils import LLM_deepseek
 from utils import CountRecord
 import utils
 import toml
@@ -11,10 +12,14 @@ import time
 import variable as var
 import action as act
 import re
-import threading
+import math
-import ctypes
+import json
+import json5
+# import threading
+# import ctypes
 cfg = None
 cfg_args = None
+cfg_move_area = None
 by_cmd = None
 filter = None
 llm_bot = None
@@ -47,9 +52,15 @@ def import_obj(_by_cmd, skip_queue):
 
     global cfg
     global cfg_args
+    global cfg_move_area
     global global_skip_queue
     cfg = toml.load('/home/evan/Workplace/project_main/cfg_subtask.toml') # 加载任务配置
     cfg_args = toml.load('/home/evan/Workplace/project_main/cfg_args.toml')
+    try:
+        with open('/home/evan/Workplace/project_main/cfg_move_area.json', 'r') as f:
+            cfg_move_area = json.load(f)
+    except:
+        cfg_move_area = None
     by_cmd = _by_cmd
     global_skip_queue = skip_queue
 
@@ -60,14 +71,14 @@ def import_obj(_by_cmd, skip_queue):
     logger.info("subtask yolo client init")
 
     # ocr socket 客户端
-    # context1 = zmq.Context()
+    context1 = zmq.Context()
-    # ocr_socket = context1.socket(zmq.REQ)
+    ocr_socket = context1.socket(zmq.REQ)
-    # ocr_socket.connect("tcp://localhost:6668")
+    ocr_socket.connect("tcp://localhost:6668")
-    # logger.info("subtask ocr client init")
+    logger.info("subtask ocr client init")
 
     filter = label_filter(socket)
     if cfg['move_area']['llm_enable']:
-        llm_bot = LLM()
+        llm_bot = LLM_deepseek()
 def car_stop():
     for _ in range(3):
         by_cmd.send_speed_x(0)
@@ -452,26 +463,26 @@ class get_block1():
         calibrate_new(var.first_block, offset = 16, run = True, run_speed = 5)
         logger.info("抓取块")
 
-        by_cmd.send_position_axis_z(30, 60)
+        by_cmd.send_position_axis_z(30, 80)
         time.sleep(1)
         by_cmd.send_angle_claw(63)
         by_cmd.send_position_axis_x(1, 20)
         time.sleep(1)
         by_cmd.send_angle_claw(25)
         time.sleep(0.5)
-        by_cmd.send_position_axis_z(30, 90)
+        by_cmd.send_position_axis_z(30, 110)
         time.sleep(0.5)
 
         by_cmd.send_angle_claw_arm(175)
         time.sleep(0.1)
         by_cmd.send_position_axis_x(1, 100)
         time.sleep(1)
-        by_cmd.send_position_axis_z(30, 70)
+        by_cmd.send_position_axis_z(30, 100)
         time.sleep(0.5)
         by_cmd.send_angle_claw(63)
         time.sleep(0.5)
 
-        by_cmd.send_position_axis_z(30, 130)
+        by_cmd.send_position_axis_z(30, 150)
         time.sleep(1)
         by_cmd.send_position_axis_x(1, 140)
         by_cmd.send_angle_claw_arm(225)
@@ -565,18 +576,18 @@ class put_block():
         # 放置第二個塊
         by_cmd.send_angle_storage(20)
         by_cmd.send_position_axis_x(1, 110)
-        by_cmd.send_position_axis_z(30, 120)
+        by_cmd.send_position_axis_z(30, 140)
         time.sleep(1.5)
         by_cmd.send_angle_claw_arm(180)
         by_cmd.send_angle_claw(85)
         # by_cmd.send_angle_storage(0)
         time.sleep(1)
-        by_cmd.send_position_axis_z(30,70)
+        by_cmd.send_position_axis_z(30,100)
         time.sleep(1)
         by_cmd.send_angle_claw(25)
         by_cmd.send_distance_x(-10, 110)
         time.sleep(1)
-        by_cmd.send_position_axis_z(30, 110)
+        by_cmd.send_position_axis_z(30, 130)
         time.sleep(1)
         by_cmd.send_angle_claw_arm(225)
         time.sleep(1)
@@ -592,7 +603,7 @@ class put_block():
     def after(self):
         var.pid_turning.set(cfg["put_block"]["pid_kp"], cfg["put_block"]["pid_ki"], cfg["put_block"]["pid_kd"])
         # 下一动作预备位置
-        while by_cmd.send_position_axis_z(30, 130) == -1:
+        while by_cmd.send_position_axis_z(30, 150) == -1:
             pass
         time.sleep(1)
         while by_cmd.send_position_axis_x(1, 0) == -1:
@@ -661,7 +672,7 @@ class get_bball():
             time.sleep(0.5)
             by_cmd.send_angle_claw_arm(45)
             time.sleep(1)
-            by_cmd.send_position_axis_z(30, 135)
+            by_cmd.send_position_axis_z(30, 155)
             # 继续向前走
             # by_cmd.send_speed_x(4)
         pass
@@ -671,10 +682,10 @@ class get_bball():
         var.pid_turning.set(cfg["get_bball"]["pid_kp"], cfg["get_bball"]["pid_ki"], cfg["get_bball"]["pid_kd"])
         # 下一动作预备位置
         by_cmd.send_angle_claw(30)
-        time.sleep(0.5)
+        # time.sleep(0.5)
         while by_cmd.send_position_axis_z(30, 0) == -1:
             pass
-        time.sleep(1)
+        time.sleep(0.5)
         # # 任务检查间隔
         # time.sleep(1)
 
@@ -702,13 +713,13 @@ class up_tower():
         by_cmd.send_distance_x(-10, 120)
         time.sleep(1)
         # 上古參數
-        by_cmd.send_distance_y(-10, 80)
+        by_cmd.send_distance_y(-10, 50) # 80
         # 6_9 模型參數
         # by_cmd.send_distance_y(-10, 40)
         # 7_12_3 模型參數
         # by_cmd.send_distance_y(-10, 50)
-        time.sleep(2)
+        # time.sleep(2)
-        car_stop()
+        # car_stop()
         # FIXME 如果下發 distance 後直接 car_stop，則 distance 執行時間僅由指令間處理延時決定
         # time.sleep(3)
         # by_cmd.send_speed_y(-10)
@@ -758,21 +769,22 @@ class get_rball():
         # 靠近塔
         by_cmd.send_angle_scoop(20)
         # 上古參數
-        by_cmd.send_distance_y(-15, 70) # 50
+        # by_cmd.send_distance_y(-15, 50) # 50 # 70
+        by_cmd.send_distance_y(-15, 40) # 50 # 70
         # 6_9 參數
         # by_cmd.send_distance_y(-15, 35)
         # time.sleep(2)
         # 7_12_3 參數
         # by_cmd.send_distance_y(-15, 45)
-        time.sleep(2)
+        # time.sleep(2)
-        car_stop()
+        # car_stop()
         calibrate_new(tlabel.RBALL,offset = 44, run = True)
         time.sleep(1)
         logger.info("抓红球")
         # by_cmd.send_angle_scoop(12)
         time.sleep(0.5)
-        by_cmd.send_position_axis_z(30, 170)
+        by_cmd.send_position_axis_z(30, 200)
-        time.sleep(2.5)
+        time.sleep(3)
         by_cmd.send_angle_scoop(12)
         time.sleep(0.5)
         by_cmd.send_angle_scoop(7)
@@ -863,7 +875,7 @@ class put_hanoi1():
             by_cmd.send_speed_omega(0)
         time.sleep(0.2)
         
-        by_cmd.send_position_axis_z(30, 130)
+        by_cmd.send_position_axis_z(30, 150)
 
         # 校准牌子
         if utils.direction_right > utils.direction_left:
@@ -1005,7 +1017,7 @@ class put_hanoi2():
         time.sleep(0.5)
         logger.info("抓大平台")
         if utils.direction is tlabel.RMARK:
-            by_cmd.send_position_axis_z(30, 10)
+            by_cmd.send_position_axis_z(30, 30)
             by_cmd.send_position_axis_x(1, 150)
             by_cmd.send_angle_claw(63)
             time.sleep(2)
@@ -1017,7 +1029,7 @@ class put_hanoi2():
             time.sleep(1)
             pass
         else:
-            by_cmd.send_position_axis_z(30, 10)
+            by_cmd.send_position_axis_z(30, 30)
             by_cmd.send_position_axis_x(1, 40)
             by_cmd.send_angle_claw(63)
             time.sleep(2)
@@ -1065,7 +1077,7 @@ class put_hanoi2():
         time.sleep(0.5)
         logger.info("抓中平台")
         if utils.direction is tlabel.RMARK:
-            by_cmd.send_position_axis_z(30, 10)
+            by_cmd.send_position_axis_z(30, 30)
             by_cmd.send_position_axis_x(1, 150)
             by_cmd.send_angle_claw(55)
             time.sleep(2)
@@ -1077,7 +1089,7 @@ class put_hanoi2():
             time.sleep(1)
             pass
         else:
-            by_cmd.send_position_axis_z(30, 10)
+            by_cmd.send_position_axis_z(30, 30)
             by_cmd.send_position_axis_x(1, 40)
             by_cmd.send_angle_claw(55)
             time.sleep(2)
@@ -1096,7 +1108,7 @@ class put_hanoi2():
         time.sleep(0.5)
         logger.info("放中平台")
         if utils.direction is tlabel.RMARK:
-            by_cmd.send_position_axis_z(30, 120)
+            by_cmd.send_position_axis_z(30, 140)
             time.sleep(2)
             by_cmd.send_position_axis_x(1, 150)
             time.sleep(2)
@@ -1108,7 +1120,7 @@ class put_hanoi2():
             time.sleep(1)
             pass
         else:
-            by_cmd.send_position_axis_z(30, 120)
+            by_cmd.send_position_axis_z(30, 140)
             time.sleep(2)
             by_cmd.send_position_axis_x(1, 40)
             time.sleep(2)
@@ -1126,7 +1138,7 @@ class put_hanoi2():
         time.sleep(0.5)
         logger.info("抓小平台")
         if utils.direction is tlabel.RMARK:
-            by_cmd.send_position_axis_z(30, 10)
+            by_cmd.send_position_axis_z(30, 30)
             by_cmd.send_position_axis_x(1, 150)
             by_cmd.send_angle_claw(50)
             time.sleep(2)
@@ -1138,7 +1150,7 @@ class put_hanoi2():
             time.sleep(2)
             pass
         else:
-            by_cmd.send_position_axis_z(30, 10)
+            by_cmd.send_position_axis_z(30, 30)
             by_cmd.send_position_axis_x(1, 40)
             by_cmd.send_angle_claw(50)
             time.sleep(2)
@@ -1157,7 +1169,7 @@ class put_hanoi2():
         time.sleep(0.5)
         logger.info("放小平台")
         if utils.direction is tlabel.RMARK:
-            by_cmd.send_position_axis_z(30, 190) # 170
+            by_cmd.send_position_axis_z(30, 200) # 170 #190(new)
             time.sleep(1.5)
             by_cmd.send_position_axis_x(1, 150)
             time.sleep(2)
@@ -1173,7 +1185,7 @@ class put_hanoi2():
             # car_stop()
             pass
         else:
-            by_cmd.send_position_axis_z(30, 190)
+            by_cmd.send_position_axis_z(30, 200)
             time.sleep(1.5)
             by_cmd.send_position_axis_x(1, 40)
             time.sleep(2)
@@ -1201,7 +1213,7 @@ class put_hanoi2():
 
 class put_hanoi3():
     def init(self):
-        while by_cmd.send_position_axis_z(30, 130) == -1:
+        while by_cmd.send_position_axis_z(30, 150) == -1:
             pass
         time.sleep(3)
         logger.info("完成任务，爪回左侧")
@@ -1219,14 +1231,16 @@ class put_hanoi3():
         time.sleep(1)
         return True
     def exec(self):
-        while by_cmd.send_position_axis_z(30, 100) == -1:
+        while by_cmd.send_position_axis_z(30, 120) == -1:
             pass
         pass
     def nexec(self):
         pass
     def after(self):
+        by_cmd.send_angle_storage(20)
         by_cmd.send_position_axis_x(1, 150)
-        var.pid_turning.set(cfg["put_hanoi3"]["pid_kp"], cfg["put_hanoi3"]["pid_ki"], cfg["put_hanoi3"]["pid_kd"])
+        var.pid_turning.set(cfg["put_hanoi3"]["pid_kp"] - 0.2, cfg["put_hanoi3"]["pid_ki"], cfg["put_hanoi3"]["pid_kd"])
+        # FIXME 此处 -0.2 在 `2e6ce3e1f7d326d6ce8110855e2339ebc03ab2da` 前没有
 
 # 应急避险 第一阶段 找目标牌
 class move_area1():
@@ -1234,6 +1248,7 @@ class move_area1():
         logger.info("应急避险第一阶段初始化")
         while (by_cmd.send_angle_camera(0) == -1):
             by_cmd.send_angle_camera(0)
+        filter.switch_camera(1)
     def find(self):
         ret = filter.find(tlabel.SIGN)
         if ret:
@@ -1245,7 +1260,7 @@ class move_area1():
         car_stop()
         time.sleep(1)
         # calibrate_new(tlabel.SIGN, offset = 8, run = True)
-        calibrate_new(tlabel.SIGN, offset = -30, run = True, run_speed = 5)
+        calibrate_new(tlabel.SIGN, offset = -1, run = True, run_speed = 5)
         time.sleep(1)
 
         by_cmd.send_position_axis_x(1, 50)
@@ -1253,33 +1268,42 @@ class move_area1():
 
         # filter_w = (148, 560)
         # filter_h = (165, 390)
-        
+        if cfg_move_area == None:
-        counts = 0
+            counts = 0
-        while True:
+            while True:
-            ocr_socket.send_string("")
+                ocr_socket.send_string("")
-            resp = ocr_socket.recv_pyobj()
+                resp = ocr_socket.recv_pyobj()
-            var.llm_text = ''
+                var.llm_text = ''
-            counts += 1
+                counts += 1
-            if resp.get('code') == 0:
+                if resp.get('code') == 0:
-                for item in resp.get('content'):
+                    for item in resp.get('content'):
-                    
+                        
-                    if item['probability']['average'] < 0.80:
+                        if item['probability']['average'] < 0.80:
-                        continue
+                            continue
-                    # box = item['location']
+                        # box = item['location']
-                    # center_x = box['left'] + box['width'] / 2
+                        # center_x = box['left'] + box['width'] / 2
-                    # center_y = box['top'] + box['height'] / 2
+                        # center_y = box['top'] + box['height'] / 2
-                    # if center_x < filter_w[0] or center_x > filter_w[1] \
+                        # if center_x < filter_w[0] or center_x > filter_w[1] \
-                    #         or center_y < filter_h[0] or center_y > filter_h[1]:
+                        #         or center_y < filter_h[0] or center_y > filter_h[1]:
-                    #     continue
+                        #     continue
-                    var.llm_text += item['words']
+                        var.llm_text += item['words']
-                break
+                    break
-            if counts >= 2:
+                if counts >= 2:
+                    var.skip_llm_task_flag = True
+                    return
+            logger.error(f"OCR 检出字符：\"{var.llm_text}\"")
+
+            
+
+            if len(var.llm_text) < 5:
                 var.skip_llm_task_flag = True
                 return
-        logger.error(var.llm_text)
+            else:
-        if len(var.llm_text) < 3:
+                # 当有效字符大于 5 个文字时 才请求大模型
-            var.skip_llm_task_flag = True
+                llm_bot.request(var.llm_text)
-            return
+        else:
+            # 不需要文字识别 直接使用传入的参数执行 action
+            pass
 
         var.task_speed = 9 # 12
 
@@ -1296,133 +1320,247 @@ class move_area1():
 
 # 应急避险 第二阶段 找停车区域
 class move_area2():
+    def __init__(self):
+        self.action_dict = {
+            'beep_seconds': self.beep_seconds,
+            'beep_counts': self.beep_counts,
+            'light_seconds': self.light_seconds,
+            'light_counts': self.light_counts,
+            'beep_light_counts': self.beep_light_counts,
+            'beep_light_seconds': self.beep_light_seconds,
+            'go_front': self.go_front,
+            'go_back': self.go_back,
+            'go_left': self.go_left,
+            'go_right': self.go_right,
+            'go_left_rotate': self.go_left_rotate,
+            'go_right_rotate': self.go_right_rotate,
+            'go_sleep': self.go_sleep
+        }
+        self.front_time = 0
+        self.back_time = 0
+        self.left_time = 0
+        self.right_time = 0
+        self.sum_rotate_angle = 0
+        self.abs_x = 0 # 为了和程序指令适配，其中 x y 方向互换
+        self.abs_y = 0
+        self.abs_w = 0
+        pass
     def init(self):
         logger.info("应急避险第二阶段初始化")
-        self.offset = 15
+        self.offset = 60
         self.delta_x = 0
         self.delta_y = 0
         self.delta_omage = 0
     def find(self):
-        # time.sleep(0.001)
+        if var.skip_llm_task_flag and cfg_move_area == None:
-        if var.skip_llm_task_flag:
             return 5000
         ret, box = filter.get(tlabel.SHELTER)
         if ret:
             error = (box[0][2] + box[0][0] - 320) / 2 + self.offset
-            if abs(error) < 20:
+            # 增加了一个宽度过滤
+            if abs(error) < 30 and abs(box[0][2] - box[0][0]) > 180:
                 return 5000
         return False
-    def sub_light(self, delay_time):
+    def add_item(self, item):
-        by_cmd.send_light(1)
+        # FIXME 没有对传入参数的范围进行校验，如果出现单位错误，可能会导致无法回位的问题
-        time.sleep(delay_time)
+        try:
-        by_cmd.send_light(0)
+            return self.action_dict[item.get('action')](item.get('time'))
-    def sub_beep(self,delay_time):
+        except:
+            pass
+        return False
+    def beep_seconds(self, _time):
         by_cmd.send_beep(1)
-        time.sleep(delay_time)
+        time.sleep(_time * 0.7)
         by_cmd.send_beep(0)
-    def sub_move(self, x, y):
+        return True
-        # FIXME 如果同時有 xy，是否會造成 delay 不足
+    def beep_counts(self, _time):
-        self.delta_x += x
+        for _ in range(_time):
-        self.delta_y += y
+            by_cmd.send_beep(1)
-        
+            time.sleep(0.3)
-        if x != 0:
+            by_cmd.send_beep(0)
-            delay_time = int(abs(x) * 500)
+            time.sleep(0.2)
-            if x > 0:
+        return True
-                by_cmd.send_distance_x(15, delay_time)
+    def light_seconds(self, _time):
-            else:
+        by_cmd.send_light(1)
-                by_cmd.send_distance_x(-15, delay_time)
+        time.sleep(_time * 0.7)
-        elif y != 0:
+        by_cmd.send_light(0)
-            delay_time = int(abs(y) * 500)
+        return True
-            if y > 0: # 向左
+    def light_counts(self, _time):
-                by_cmd.send_distance_y(-15, delay_time)
+        for _ in range(_time):
-            else:
+            by_cmd.send_light(1)
-                by_cmd.send_distance_y(15, delay_time)
+            time.sleep(0.3)
-        time.sleep(delay_time / 500)
+            by_cmd.send_light(0)
-        car_stop()
+            time.sleep(0.2)
-        pass
+        return True
-    def sub_turn(self, angle):
+    def beep_light_counts(self, _time):
-        self.delta_omage += angle
+        for _ in range(_time):
-        delay_time = int(abs(angle) * 400 / 90)
+            by_cmd.send_beep(1)
-        if angle < 0:
+            by_cmd.send_light(1)
-            # 左转
+            time.sleep(0.3)
-            by_cmd.send_angle_omega(+55, delay_time)
+            by_cmd.send_beep(0)
+            by_cmd.send_light(0)
+            time.sleep(0.2)
+        return True
+    def beep_light_seconds(self, _time):
+        by_cmd.send_beep(1)
+        by_cmd.send_light(1)
+        time.sleep(_time * 0.3)
+        by_cmd.send_beep(0)
+        by_cmd.send_light(0)
+        return True
+    def go_front(self, _time):
+        self.abs_y -=  math.sin(self.abs_w) * _time
+        self.abs_x +=  math.cos(self.abs_w) * _time
+        logger.info(f"向前移动：[目标位置 ({self.abs_y:.2f}, {self.abs_x:.2f}) - 角度 {math.degrees(self.abs_w)} ]")
+        speed_time = int(abs(_time) * 750)
+        by_cmd.send_distance_x(10, speed_time)
+        time.sleep(speed_time / 100)
+        self.front_time += speed_time
+        return True
+    def go_back(self, _time):
+        self.abs_y +=  math.sin(self.abs_w) * _time
+        self.abs_x -=  math.cos(self.abs_w) * _time
+        logger.info(f"向后移动：[目标位置 ({self.abs_y:.2f}, {self.abs_x:.2f}) - 角度 {math.degrees(self.abs_w)} ]")
+        speed_time = int(abs(_time) * 750)
+        by_cmd.send_distance_x(-10, speed_time)
+        time.sleep(speed_time / 100)
+        self.back_time += speed_time
+        return True
+    def go_left(self, _time):
+        self.abs_y -=  math.cos(self.abs_w) * _time
+        self.abs_x -=  math.sin(self.abs_w) * _time
+        logger.info(f"向左移动：[目标位置 ({self.abs_y:.2f}, {self.abs_x:.2f}) - 角度 {math.degrees(self.abs_w)} ]")
+        speed_time = int(abs(_time) * 750)
+        by_cmd.send_distance_y(-10, speed_time)
+        time.sleep(speed_time / 100)
+        self.left_time += speed_time
+        return True
+    def go_right(self, _time):
+        self.abs_y +=  math.cos(self.abs_w) * _time
+        self.abs_x +=  math.sin(self.abs_w) * _time
+        logger.info(f"向右移动：[目标位置 ({self.abs_y:.2f}, {self.abs_x:.2f}) - 角度 {math.degrees(self.abs_w)} ]")
+        speed_time = int(abs(_time) * 750)
+        by_cmd.send_distance_y(10, speed_time)
+        time.sleep(speed_time / 100)
+        self.right_time += speed_time
+        return True
+    def go_shift(self, _dis_x, _dis_y):
+        direct_x = 1.0 if (_dis_x > 0) else -1.0
+        direct_y = 1.0 if (_dis_y > 0) else -1.0
+        self.abs_y -=  math.sin(self.abs_w) * _dis_x
+        self.abs_x +=  math.cos(self.abs_w) * _dis_x
+        self.abs_y +=  math.cos(self.abs_w) * _dis_y
+        self.abs_x +=  math.sin(self.abs_w) * _dis_y
+        logger.info(f"水平移动：[目标位置 ({self.abs_y:.2f}, {self.abs_x:.2f}) - 角度 {math.degrees(self.abs_w)} ]")
+        speed_time_x = int(abs(_dis_x) * 750)
+        speed_time_y = int(abs(_dis_y) * 750)
+        by_cmd.send_distance_x(10 * direct_x, speed_time_x)
+        by_cmd.send_distance_y(10 * direct_y, speed_time_y)
+        time.sleep(max(speed_time_x, speed_time_y) / 150) #FIXME 除以 100 是否正确
+        return True
+    def go_left_rotate(self, _time):
+        self.abs_w += math.radians(_time) # 弧度制
+        logger.info(f"向左旋转：[目标位置 ({self.abs_y:.2f}, {self.abs_x:.2f}) - 角度 {math.degrees(self.abs_w)} ]")
+        self.sum_rotate_angle -= _time
+        speed_time = int(abs(_time) * 3.8)
+        by_cmd.send_angle_omega(50, speed_time)
+        time.sleep(speed_time / 200 + 0.5)
+        return True
+    def go_right_rotate(self, _time):
+        self.abs_w -= math.radians(_time) # 弧度制
+        logger.info(f"向右旋转：[目标位置 ({self.abs_y:.2f}, {self.abs_x:.2f}) - 角度 {math.degrees(self.abs_w)} ]")
+        self.sum_rotate_angle += _time
+        speed_time = int(abs(_time) * 3.8)
+        by_cmd.send_angle_omega(-50, speed_time)
+        time.sleep(speed_time / 200 + 0.5)
+        return True
+    def go_sleep(self, _time):
+        time.sleep(_time*0.7)
+        return True
+    def reset(self):
+        logger.info(f"开始复位：[当前位置 ({self.abs_y:.2f}, {self.abs_x:.2f}) - 角度 {math.degrees(self.abs_w)}]")
+        # 归一化角度到 0-2pi
+        left_dregee = math.degrees(self.abs_w % (2 * math.pi))
+        # 确定旋转方向 (寻找回正角度最小旋转方向)
+        if math.sin(self.abs_w) < 0:
+            logger.info(f"需要左旋 {360.0 - left_dregee} 回正")
+            self.go_left_rotate(360.0 - left_dregee)
         else:
-            # 右转
+            logger.info(f"需要右旋 {left_dregee} 回正")
-            by_cmd.send_angle_omega(-55, delay_time)
+            self.go_right_rotate(left_dregee)
-        time.sleep(delay_time / 300 * 1.5)
+        time.sleep(0.1)
+
+        # 在框中原点添加向左 0.6m 的偏移值，以便直接回到赛道
+        self.go_shift(self.abs_x * -1.0, self.abs_y * -1.0 - 0.6) 
+
+        logger.info(f"回正后最终位置： ({self.abs_y:.2f}, {self.abs_x:.2f}), 角度： {math.degrees(self.abs_w % (2 * math.pi))}")
     def exec(self):
         var.task_speed = 0
-        if var.skip_llm_task_flag:
+        if var.skip_llm_task_flag and cfg_move_area == None:
-            logger.error("ocr 识别出错 直接跳过改任务")
+            logger.error("ocr 识别出错 直接跳过该任务")
             return
         logger.info("开始寻找停车区域")
         car_stop()
-        calibrate_new(tlabel.SHELTER, offset = 15, run = True)
+        calibrate_new(tlabel.SHELTER, offset = 30, run = True)
         time.sleep(0.5)
-        # 调用大模型 然后执行动作
+        if cfg_move_area == None:
-        try:
+            resp = None
-            resp = llm_bot.get_command_json(var.llm_text)
+            # 调用大模型 然后执行动作
-            logger.info(resp)
+            try:
-        except:
+                logger.info("llm 阻塞等待服务器返回中")
-            logger.error("大模型超时，跳过任务")
+                start_wait_time = time.perf_counter()
-            return
+                while True:
-        
+                    if llm_bot.success_status.isSet():
-        
+                        resp = llm_bot.response.choices[0].message.content
-        try:
+                        logger.info(f"llm 返回原数据 {resp}")
-            resp_commands = eval(re.findall("```json(.*?)```", resp, re.S)[0])
+                        break
-            if len(resp_commands) == 0:
+                    now_time = time.perf_counter()
+                    if llm_bot.error_status.isSet() or (now_time - start_wait_time) > 6.5 :
+                        logger.error("大模型 llm_bot 超时，跳过任务")
+                        return
+                    
+            except:
+                logger.error("大模型 llm_bot 未知错误，跳过任务")
                 return
-            # 进入停车区域
+
-            # by_cmd.send_speed_y(15)
+            try:
-            by_cmd.send_distance_y(25, 180)
+                json_text = re.findall("```json(.*?)```", resp, re.S)
-            time.sleep(1)
+                if len(json_text) == 0:
-            # time.sleep(1.25)
+                    # 返回的内容不带'''json
-            car_stop()
+                    resp_commands = eval(resp)
-            logger.info(resp_commands)
-            for command in resp_commands:
-                logger.info(command)
-                if command['func'] == 'move':
-                    self.sub_move(float(command['x']), float(command['y']))
-                elif command['func'] == 'light':
-                    self.sub_light(int(command['time']))
-                elif command['func'] == 'beep':
-                    self.sub_beep(int(command['time']))
-                elif command['func'] == 'turn':
-                    self.sub_turn(int(command['angle']))
-                    pass
                 else:
-                    continue
+                    resp_commands = json5.loads(json_text[0])
+            
+                    
+                logger.info(f"解析后的动作序列 {resp_commands}")
+                if len(resp_commands) == 0:
+                    return
+                action_list = resp_commands
+                # 进入停车区域
+                by_cmd.send_distance_y(10, 450)
+                time.sleep((450 * 5 / 1000) + 0.5)
+                for action in action_list:
+                    self.add_item(action)
+                    time.sleep(0.1)
                 time.sleep(0.5)
-        except:
+                self.reset()
+            except:
+                logger.warning("任务解析失败并退出，文心一言真是废物 (毋庸置疑)")
+                pass
+            
+        else:
+            # 无需调用大模型 直接开始执行传入的参数
+            try:
+                by_cmd.send_distance_y(10, 450)
+                time.sleep((450 * 5 / 1000) + 0.5)
+                for action in cfg_move_area:
+                    self.add_item(action)
+                    time.sleep(0.1)
+                time.sleep(0.5)
+                self.reset()
+            except:
+                pass
             pass
-        time.sleep(1)
-        # 回到原位
-        
-        delay_time = int(abs(self.delta_omage) * 400 / 90)
-        if int(abs(self.delta_omage)) == 360:
-            delay_time = 0
-        if self.delta_omage < 0:
-            # 左转
-            by_cmd.send_angle_omega(-55, delay_time)
-        else:
-            # 右转
-            by_cmd.send_angle_omega(55, delay_time)
-        time.sleep(delay_time / 300 * 1.5)
-        if self.delta_y > 0:
-            # 向左移动的距离就要比进入的时候少一些 因为 action 已经向左运动了
-            delay_time = 180 - (self.delta_y * 500)
-        else:
-            delay_time = 180 + (abs(self.delta_y) * 500)
-        # 离开停车区域
-        by_cmd.send_distance_y(-25, delay_time)
-
-        time.sleep(delay_time * 5e-3)
-        
-        car_stop()
-
-        # FIXME 移动距离指令下发后未完成，再发送速度指令，将不会清除未完成的速度值
-        # by_cmd.send_distance_y(-15, 300)
-        pass
     def nexec(self):
         logger.warning("正在跳過大模型任務")
         time.sleep(2)
@@ -1460,18 +1598,43 @@ class kick_ass():
         calibrate_new(tlabel.SIGN, offset = 8, run = True)
         by_cmd.send_angle_claw(15)
         time.sleep(0.5)
-        by_cmd.send_position_axis_z(30, 60)
+        by_cmd.send_position_axis_z(30, 80)
         by_cmd.send_position_axis_x(1, 130)
 
+        # 移动到中间
+        by_cmd.send_distance_x(10, 295)
+        time.sleep(4)
         if self.target_person == 1:
-            target_distance = self.pos_gap1
+            by_cmd.send_distance_x(-10, 150)
+            time.sleep(1.5)
+            car_stop()
+        elif self.target_person == 2:
+            by_cmd.send_distance_x(-10, 50)
+            time.sleep(1.5)
+            car_stop()
+        elif self.target_person == 3:
+            by_cmd.send_distance_x(10, 50)
+            time.sleep(1.5)
+            car_stop()
         else:
-            target_distance = self.pos_gap1 + (self.target_person - 1) * self.pos_gap2 + (self.target_person - 1) * 10
+            by_cmd.send_distance_x(10, 150)
-        by_cmd.send_distance_x(10, target_distance)
+            time.sleep(1.5)
-        
+            car_stop()
-        logger.info(f"target distance {target_distance}")
+
-        time.sleep(1.5 + (self.target_person - 1) * 0.7 )
+        # 先移动到第一个人的地方 假动作
-        car_stop()
+        # by_cmd.send_distance_x(10, self.pos_gap1)
+        # time.sleep(1.5)
+
+        # if self.target_person == 1:
+        #     # target_distance = self.pos_gap1
+        #     pass
+        # else:
+        #     # target_distance = self.pos_gap1 + (self.target_person - 1) * self.pos_gap2 + (self.target_person - 1) * 10
+        #     target_distance = (self.target_person - 1) * self.pos_gap2 + (self.target_person - 1) * 10
+        #     by_cmd.send_distance_x(10, target_distance)
+        #     logger.info(f"target distance {target_distance}")
+        #     time.sleep(1.5 + (self.target_person - 1) * 0.7 )
+        # car_stop()
         
         # by_cmd.send_angle_claw_arm(225)
         # time.sleep(0.5)

test/test_action.py

@@ -0,0 +1,241 @@
+import os
+import sys
+import math
+parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(parent_dir)
+from by_cmd_py import by_cmd_py
+import time
+import zmq
+import numpy as np
+context = zmq.Context()
+socket = context.socket(zmq.REQ)
+socket.connect("tcp://localhost:6666")
+
+
+def car_stop():
+    for _ in range(3):
+        cmd_py_obj.send_speed_x(0)
+        time.sleep(0.2)
+        cmd_py_obj.send_speed_y(0)
+        time.sleep(0.2)
+        cmd_py_obj.send_speed_omega(0)
+
+class LLM_Action:
+    def __init__(self,cmd_py_obj):
+        self.by_cmd = cmd_py_obj
+        self.action_dict = {
+            'beep_seconds': self.beep_seconds,
+            'beep_counts': self.beep_counts,
+            'light_seconds': self.light_seconds,
+            'light_counts': self.light_counts,
+            'beep_light_counts': self.beep_light_counts,
+            'beep_light_seconds': self.beep_light_seconds,
+            'go_front': self.go_front,
+            'go_back': self.go_back,
+            'go_left': self.go_left,
+            'go_right': self.go_right,
+            'go_left_rotate': self.go_left_rotate,
+            'go_right_rotate': self.go_right_rotate,
+            'go_sleep': self.go_sleep
+        }
+        self.front_time = 0
+        self.back_time = 0
+        self.left_time = 0
+        self.right_time = 0
+        self.sum_rotate_angle = 0
+        self.abs_x = 0 # 为了和程序指令适配，其中 x y 方向互换
+        self.abs_y = 0
+        self.abs_w = 0
+        pass
+    def __call__(self, item):
+        try:
+            return self.action_dict[item.get('action')](item.get('time'))
+        except:
+            pass
+        return False
+    def beep_seconds(self, _time):
+        self.by_cmd.send_beep(1)
+        time.sleep(_time * 0.7)
+        self.by_cmd.send_beep(0)
+        return True
+    def beep_counts(self, _time):
+        for _ in range(_time):
+            self.by_cmd.send_beep(1)
+            time.sleep(0.3)
+            self.by_cmd.send_beep(0)
+            time.sleep(0.2)
+        return True
+    def light_seconds(self, _time):
+        self.by_cmd.send_light(1)
+        time.sleep(_time * 0.7)
+        self.by_cmd.send_light(0)
+        return True
+    def light_counts(self, _time):
+        for _ in range(_time):
+            self.by_cmd.send_light(1)
+            time.sleep(0.3)
+            self.by_cmd.send_light(0)
+            time.sleep(0.2)
+        return True
+    def beep_light_counts(self, _time):
+        for _ in range(_time):
+            self.by_cmd.send_beep(1)
+            self.by_cmd.send_light(1)
+            time.sleep(0.3)
+            self.by_cmd.send_beep(0)
+            self.by_cmd.send_light(0)
+            time.sleep(0.2)
+        return True
+    def beep_light_seconds(self, _time):
+        self.by_cmd.send_beep(1)
+        self.by_cmd.send_light(1)
+        time.sleep(_time * 0.3)
+        self.by_cmd.send_beep(0)
+        self.by_cmd.send_light(0)
+        return True
+    def go_front(self, _time):
+        self.abs_y -=  math.sin(self.abs_w) * _time
+        self.abs_x +=  math.cos(self.abs_w) * _time
+        print(f"abs postion ({self.abs_y:.2f}, {self.abs_x:.2f}) - angle {math.degrees(self.abs_w)} drgee")
+        speed_time = int(abs(_time) * 750)
+        self.by_cmd.send_distance_x(10, speed_time)
+        time.sleep(speed_time / 100)
+        self.front_time += speed_time
+        return True
+    def go_back(self, _time):
+        self.abs_y +=  math.sin(self.abs_w) * _time
+        self.abs_x -=  math.cos(self.abs_w) * _time
+        print(f"abs postion ({self.abs_y:.2f}, {self.abs_x:.2f}) - angle {math.degrees(self.abs_w)} drgee")
+        speed_time = int(abs(_time) * 750)
+        self.by_cmd.send_distance_x(-10, speed_time)
+        time.sleep(speed_time / 100)
+        self.back_time += speed_time
+        return True
+    def go_left(self, _time):
+        self.abs_y -=  math.cos(self.abs_w) * _time
+        self.abs_x -=  math.sin(self.abs_w) * _time
+        print(f"abs postion ({self.abs_y:.2f}, {self.abs_x:.2f}) - angle {math.degrees(self.abs_w)} drgee")
+        speed_time = int(abs(_time) * 750)
+        self.by_cmd.send_distance_y(-10, speed_time)
+        time.sleep(speed_time / 100)
+        self.left_time += speed_time
+        return True
+    def go_right(self, _time):
+        self.abs_y +=  math.cos(self.abs_w) * _time
+        self.abs_x +=  math.sin(self.abs_w) * _time
+        print(f"abs postion ({self.abs_y:.2f}, {self.abs_x:.2f}) - angle {math.degrees(self.abs_w)} drgee")
+        speed_time = int(abs(_time) * 750)
+        self.by_cmd.send_distance_y(10, speed_time)
+        time.sleep(speed_time / 100)
+        self.right_time += speed_time
+        return True
+    def go_shift(self, _dis_x, _dis_y):
+        direct_x = 1.0 if (_dis_x > 0) else -1.0
+        direct_y = 1.0 if (_dis_y > 0) else -1.0
+        self.abs_y -=  math.sin(self.abs_w) * _dis_x
+        self.abs_x +=  math.cos(self.abs_w) * _dis_x
+        self.abs_y +=  math.cos(self.abs_w) * _dis_y
+        self.abs_x +=  math.sin(self.abs_w) * _dis_y
+        print(f"abs postion ({self.abs_y:.2f}, {self.abs_x:.2f}) - angle {math.degrees(self.abs_w)} drgee")
+        speed_time_x = int(abs(_dis_x) * 750)
+        speed_time_y = int(abs(_dis_y) * 750)
+        self.by_cmd.send_distance_x(10 * direct_x, speed_time_x)
+        self.by_cmd.send_distance_y(10 * direct_y, speed_time_y)
+        time.sleep(max(speed_time_x, speed_time_y) / 100) #FIXME 除以 100 是否正确
+        return True
+    def go_left_rotate(self, _time):
+        self.abs_w += math.radians(_time) # 弧度制
+        print(f"abs postion ({self.abs_y:.2f}, {self.abs_x:.2f}) - angle {math.degrees(self.abs_w)} drgee")
+        self.sum_rotate_angle -= _time
+        speed_time = int(abs(_time) * 4.35)
+        self.by_cmd.send_angle_omega(50, speed_time)
+        time.sleep(speed_time / 200 + 0.5)
+        # time.sleep(speed_time / _time / 2)
+        return True
+    def go_right_rotate(self, _time):
+        self.abs_w -= math.radians(_time) # 弧度制
+        print(f"abs postion ({self.abs_y:.2f}, {self.abs_x:.2f}) - angle {math.degrees(self.abs_w)} drgee")
+        self.sum_rotate_angle += _time
+        speed_time = int(abs(_time) * 4.35)
+        self.by_cmd.send_angle_omega(-50, speed_time)
+        time.sleep(speed_time / 200 + 0.5)
+        # time.sleep(speed_time / _time / 2)
+        return True
+    def go_sleep(self, _time):
+        time.sleep(_time*0.7)
+        return True
+    def reset(self):
+        print(f"开始复位：[当前位置 ({self.abs_y:.2f}, {self.abs_x:.2f}) - 角度 {math.degrees(self.abs_w)}]")
+        # 先复位角度
+        # if self.sum_rotate_angle > 0:
+        #     self.sum_rotate_angle = self.sum_rotate_angle % 360
+        # else:
+        #     self.sum_rotate_angle = -(abs(self.sum_rotate_angle) % 360)
+        # if self.sum_rotate_angle > 0:
+        #     # 采用左转回正
+        #     self.go_left_rotate(self.sum_rotate_angle)
+        #     # speed_time = int(abs(self.sum_rotate_angle) * 7.25)
+        #     # self.by_cmd.send_angle_omega(30, speed_time)
+        #     pass
+        # else:
+        #     # 采用右转回正
+        #     self.go_right_rotate(abs(self.sum_rotate_angle))
+        #     # speed_time = int(abs(self.sum_rotate_angle) * 7.25)
+        #     # self.by_cmd.send_angle_omega(-30, speed_time)
+        left_dregee = math.degrees(self.abs_w % (2 * math.pi)) #归一化角度到 0-2pi
+        if math.sin(self.abs_w) < 0:
+            print(f"需要左旋 {360.0 - left_dregee} 回正")
+            self.go_left_rotate(360.0 - left_dregee)
+        else:
+            print(f"需要右旋 {left_dregee} 回正")
+            self.go_right_rotate(left_dregee)
+        time.sleep(0.1)
+
+        self.go_shift(self.abs_x * -1.0, self.abs_y * -1.0 - 0.6) # 左移 0.6m 回到赛道
+        # # 再回正 x 轴
+        # if self.front_time > self.back_time:
+        #     # 采用后退回正
+        #     speed_time = self.front_time - self.back_time
+        #     self.by_cmd.send_distance_x(-10, speed_time)
+        # else:
+        #     speed_time = self.back_time - self.front_time
+        #     self.by_cmd.send_distance_x(10, speed_time)
+        # time.sleep(speed_time / 100)
+        # time.sleep(0.1)
+          
+        # # 最后回正 y 轴
+        # speed_time = self.left_time - self.right_time
+        # if speed_time < 0:
+        #     speed_time = 4500 + abs(speed_time)
+        # else:
+        #     speed_time = 4500 - speed_time
+        # self.by_cmd.send_distance_y(-10, speed_time / 15 + 100)
+        # print(speed_time * 1e-3 * 0.9)
+        # time.sleep(speed_time * 1e-3 * 0.9)
+
+        print(f"回正后最终位置： ({self.abs_y:.2f}, {self.abs_x:.2f}), 角度： {math.degrees(self.abs_w % (2 * math.pi))}")
+
+
+
+
+
+if __name__ == "__main__":
+
+
+    cmd_py_obj = by_cmd_py()
+    # cmd_py_obj.send_angle_omega(-30, 20 * 7.25)
+    # time.sleep(20 * 7.25 / 20 / 2)
+    llm_act = LLM_Action(cmd_py_obj)
+    action_list = [{"index":0,"action":"go_left_rotate","time":270},{"index":1,"action":"go_back","time":0.2},{"index":2,"action":"go_left","time":0.2},{"index":2,"action":"beep_counts","time":3}]
+    
+    cmd_py_obj.send_distance_y(10, 450)
+    time.sleep((450 * 5 / 1000) + 0.5)
+    # time.sleep(15 * 300 * 1e-3 * 0.7)
+    # car_stop()
+    for action in action_list:
+        llm_act(action)
+        time.sleep(0.1)
+        pass
+    time.sleep(1)
+    llm_act.reset()
+    # car_stop()

test/test_filter.py

@@ -16,7 +16,7 @@ socket.connect("tcp://localhost:6667")
 logger.info("subtask yolo client init")
 
 filter = label_filter(socket)
-filter.switch_camera(2)
+filter.switch_camera(1)
 
 
 # find_counts = 0
@@ -25,10 +25,13 @@ filter.switch_camera(2)
 # label = tlabel.TPLATFORM
 while True:
     time.sleep(0.2)
-    ret, box = filter.get(tlabel.BASE)
+    ret, box = filter.get(tlabel.SIGN)
     if ret:
         error = (box[0][2] + box[0][0] - 320) / 2
         logger.error(error)
+        # if abs(error) < 30 and abs(box[0][2] - box[0][0]) > 180:
+        #     # height = box[0][3] - box[0][1]
+        #     logger.error(111)
 
     # label = tlabel.HOSPITAL
     # ret, box = filter.get(label)

test/test_llm.py

@@ -13,18 +13,48 @@ class LLM:
     def __init__(self):
         
         self.model = 'ernie-3.5'
-        self.prompt = '''你是一个机器人动作规划者，需要把我的话翻译成机器人动作规划并生成对应的 json 结果，机器人工作空间参考右手坐标系。
+        self.prompt = '''
-                    严格按照下面的描述生成给定格式 json，从现在开始你仅仅给我返回 json 数据'''
+                        你是一个机器人动作规划者，需要把我的话翻译成机器人动作规划并生成对应的 JSON 结果。请注意，只能使用以下指定的动作，不能创造新的动作：
-        self.prompt += '''正确的示例如下：
+                        允许的动作及其对应格式如下：
-                        向左移 0.1m, 向左转弯 85 度  [{'func': 'move', 'x': 0, 'y': 0.1},{'func': 'turn','angle': -85}],
+                        - 向左移：{"index":N,"action":"go_left","time":T}
-                        向右移 0.2m, 向前 0.1m [{'func': 'move', 'x': 0, 'y': -0.2},{'func': 'move', 'x': 0.1, 'y': 0}],
+                        - 向右移：{"index":N,"action":"go_right","time":T}
-                        向右转 85 度，向右移 0.1m [{'func': 'turn','angle': 85},{'func': 'move', 'x': 0, 'y': -0.1}],
+                        - 向前移：{"index":N,"action":"go_front","time":T}
-                        原地左转 38 度 [{'func': 'turn','angle': -38}],
+                        - 向后移：{"index":N,"action":"go_back","time":T}
-                        蜂鸣器发声 5 秒 [{'func': 'beep', 'time': 5}]
+                        - 向左转：{"index":N,"action":"go_left_rotate","time":T}
-                        发光或者照亮 5 秒 [{'func': 'light', 'time': 5}]
+                        - 向右转：{"index":N,"action":"go_right_rotate","time":T}
-                        向右走 30cm，照亮 2s [{'func': 'move', 'x': 0, 'y': -0.3}, {'func': 'light', 'time': 2}],
+                        - 蜂鸣器发声：{"index":N,"action":"beep_seconds","time":T}
-                        向左移 0.2m, 向后 0.1m [{'func': 'move', 'x': 0, 'y': 0.2},{'func': 'move', 'x': -0.1, 'y': 0}],
+                        - 蜂鸣器发声次数：{"index":N,"action":"beep_counts","time":T}
+                        - 发光或者照亮：{"index":N,"action":"light_seconds","time":T}
+                        - 发光次数或者闪烁次数：{"index":N,"action":"light_counts","time":T}
+                        - 发光并伴随蜂鸣器：{"index":N,"action":"beep_light_counts","time":T}
+                        - 等待{"index":N,"action":"go_sleep","time":T}
+                        示例输入输出如下：
+                        输入：向左移 0.1m, 向左转弯 85 度
+                        输出：[{"index":0,"action":"go_left","time":0.1},{"index":1,"action":"go_left_rotate","time":85}]
+                        输入：向右移 0.2m, 向前 0.1m
+                        输出：[{"index":0,"action":"go_right","time":0.2},{"index":1,"action":"go_front","time":0.1}]
+                        输入：向右转 90 度，向右移 0.1m
+                        输出：[{"index":0,"action":"go_right_rotate","time":90},{"index":1,"action":"go_right","time":0.1}]
+                        输入：原地左转 38 度
+                        输出：[{"index":0,"action":"go_left_rotate","time":38}]
+                        输入：蜂鸣器发声 5 秒
+                        输出：[{"index":0,"action":"beep_seconds","time":5}]
+                        输入：发光或者照亮 5 秒
+                        输出：[{"index":0,"action":"light_seconds","time":5}]
+                        输入：向右走 30cm, 照亮 2s
+                        输出：[{"index":0,"action":"go_right","time":0.3},{"index":1,"action":"light_seconds","time":2}]
+                        输入：向左移 0.2m, 向后 0.1m
+                        输出：[{"index":0,"action":"go_left","time":0.2},{"index":1,"action":"go_back","time":0.1}]
+                        输入：鸣叫 3 声
+                        输出：[{"index":0,"action":"beep_counts","time":3}]
+                        输入：前行零点五米
+                        输出：[{"index":0,"action":"go_front","time":0.5}]
+                        输入：闪烁灯光 1 次并伴有蜂鸣器
+                        输出：[{"index":0,"action":"beep_light_counts","time": 1}]
+                        输入：灯光闪烁 3 次同时蜂鸣器也叫 3 次
+                        输出：[{"index":0,"action":"beep_light_counts","time": 3}]
                     '''
+        self.prompt += '''请根据上面的示例，解析该任务文本，并返回相应的 JSON 字段。确保 JSON 中包含了键 index action 和 time 以及相应的值'''
         self.messages = []
         self.resp = None
         self.reset()

test/test_ocr.py

@@ -17,13 +17,7 @@ while True:
     if resp.get('code') == 0:
         text = ''
         for item in resp.get('content'):
-            if item['probability']['average'] < 0.90:
+            if item['probability']['average'] < 0.80:
-                continue
-            box = item['location']
-            center_x = box['left'] + box['width'] / 2
-            center_y = box['top'] + box['height'] / 2
-            if center_x < filter_w[0] or center_x > filter_w[1] \
-                    or center_y < filter_h[0] or center_y > filter_h[1]:
                 continue
             text += item['words']
         print(text)

test/test_ocr_camera.py

@@ -0,0 +1,17 @@
+import cv2
+import time
+
+cap = cv2.VideoCapture(20)
+cap.set(cv2.CAP_PROP_AUTOFOCUS, 0)
+cap.set(cv2.CAP_PROP_FOCUS, 280)
+cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640)
+cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480)
+
+
+while True:
+    ret, frame = cap.read()
+    if ret:
+        cv2.imshow('src', frame)
+    time.sleep(0.2)
+    if cv2.waitKey(25) & 0xFF == ord('q'):
+        break

utils.py

@@ -2,6 +2,7 @@
 from enum import Enum
 import numpy as np
 import erniebot
+from openai import OpenAI
 from simple_pid import PID
 from loguru import logger
 import threading
@@ -372,55 +373,234 @@ class label_filter:
                 error = (boxes[center_x_index][4] + boxes[center_x_index][2] - self.img_size[0]) / 2
                 return (True, error)
         return (False, 0)
-
+class LLM_deepseek:
-
+    def __init__(self):
+        self.response = None
+        self.success_status = threading.Event()
+        self.error_status = threading.Event()
+        self.success_status.clear()
+        self.error_status.clear()
+        self.chat = ''
+        self.client = OpenAI(api_key="sk-c2e1073883304143981a9750b97c3518", base_url="https://api.deepseek.com")
+        self.prompt = '''
+            你是一个机器人动作规划者，请把我的话翻译成机器人动作规划并生成对应的 JSON 结果。请注意，只能使用以下指定的动作，不能创造新的动作：
+            允许的动作及其对应格式如下：
+            [{'properties': {'index': {'title': 'Index', 'type': 'integer'}, 'action': {'title': 'Action', 'type': 'string'}, 'time': {'title': 'Time', 'type': 'number'}}, 'required': ['index', 'action', 'time'], 'title': 'Action', 'type': 'object'}]
+            我不允许你自我创造出新的 action,action 字段仅仅包括以下内容：
+            go_right 向右移动
+            go_left 向左移动
+            go_front 向前移动
+            go_back 向后移动
+            go_left_rotate 向左旋转
+            go_right_rotate 向右旋转
+            beep_seconds 蜂鸣器鸣叫的时间
+            beep_counts 蜂鸣器鸣叫的次数
+            light_seconds 灯光发光的时间
+            light_counts 灯光闪烁的次数
+            beep_light_counts 灯光和蜂鸣器一起闪烁的次数
+            go_sleep 什么都不做
+            我的话和你的回复示例如下：
+            我的话：向左移 0.1m, 向左转弯 85 度
+            你的回复：[{"index":0,"action":"go_left","time":0.1},{"index":1,"action":"go_left_rotate","time":85}]
+            我的话：向右移 0.2m, 向前 0.1m
+            你的回复：[{"index":0,"action":"go_right","time":0.2},{"index":1,"action":"go_front","time":0.1}]
+            我的话：向右转 90 度，向右移 0.1m
+            你的回复：[{"index":0,"action":"go_right_rotate","time":90},{"index":1,"action":"go_right","time":0.1}]
+            我的话：原地左转 38 度
+            你的回复：[{"index":0,"action":"go_left_rotate","time":38}]
+            我的话：蜂鸣器发声 5 秒
+            你的回复：[{"index":0,"action":"beep_seconds","time":5}]
+            我的话：发光或者照亮 5 秒
+            你的回复：[{"index":0,"action":"light_seconds","time":5}]
+            我的话：向右走 30cm，照亮 2s
+            你的回复：[{"index":0,"action":"go_right","time":0.3},{"index":1,"action":"light_seconds","time":2}]
+            我的话：向左移 0.2m, 向后 0.1m
+            你的回复：[{"index":0,"action":"go_left","time":0.2},{"index":1,"action":"go_back","time":0.1}]
+            我的话：鸣叫 3 声
+            你的回复：[{"index":0,"action":"beep_counts","time":3}]
+            我的话：前行零点五米
+            你的回复：[{"index":0,"action":"go_front","time":0.5}]
+            我的话：闪烁灯光 1 次并伴有蜂鸣器
+            你的回复：[{"index":0,"action":"beep_light_counts","time": 1}]
+            我的话：灯光闪烁 3 次同时蜂鸣器也叫 3 次
+            你的回复：[{"index":0,"action":"beep_light_counts","time": 3}]
+            
+            强调一下，对于‘离开’这个指令，请忽略，这对我很重要！
+        '''
+    def request_thread(self):
+        logger.info("llm 请求远程服务器中 (request_thread)")
+        try:
+            self.response = self.client.chat.completions.create(
+                model="deepseek-chat",
+                messages=[
+                    {"role": "system", "content": self.prompt},
+                    {"role": "user", "content": '我的话如下：' + self.chat},
+                ],
+                stream=False,
+                temperature=0.7
+            )
+            logger.info("llm 远程服务器正常返回 (request_thread)")
+            self.success_status.set()
+        except:
+            logger.warning("llm 请求失败或返回异常，先检查网络连接 (request_thread)")
+            self.error_status.set()
+    def request(self, _chat):
+        self.chat = _chat
+        thread = threading.Thread(target=self.request_thread, daemon=True)
+        thread.start()
+        logger.info("llm 开启请求线程")
+    def get_command_json(self,chat = ''):
+        # response = self.client.chat.completions.create(
+        #     model="deepseek-chat",
+        #     messages=[
+        #         {"role": "system", "content": self.prompt},
+        #         {"role": "user", "content": '我的话如下：' + chat},
+        #     ],
+        #     stream=False,
+        #     temperature=0.7
+        # )
+        logger.info("llm 阻塞等待服务器返回中")
+        while not self.status: # FIXME 阻塞等待是否合适
+            pass
+        logger.info("llm 收到返回")
+        return self.response.choices[0].message.content
 class LLM:
     def __init__(self):
         self.init_done_flag = False
         erniebot.api_type = "qianfan"
         erniebot.ak = "jReawMtWhPu0wrxN9Rp1MzZX"
         erniebot.sk = "eowS1BqsNgD2i0C9xNnHUVOSNuAzVTh6"
-        self.model = 'ernie-3.5'
+        self.model = 'ernie-lite'
-        self.prompt = '''你是一个机器人动作规划者，需要把我的话翻译成机器人动作规划并生成对应的 json 结果，机器人工作空间参考右手坐标系。
+        # self.prompt = '''你是一个机器人动作规划者，需要把我的话翻译成机器人动作规划并生成对应的 json 结果，机器人工作空间参考右手坐标系。
-                    严格按照下面的描述生成给定格式 json，从现在开始你仅仅给我返回 json 数据！'''
+        #             严格按照下面的描述生成给定格式 json，从现在开始你仅仅给我返回 json 数据！'''
-        self.prompt += '''正确的示例如下：
+        # self.prompt += '''正确的示例如下：
-                        向左移 0.1m, 向左转弯 85 度  [{'func': 'move', 'x': 0, 'y': 0.1},{'func': 'turn','angle': -85}],
+        #                 向左移 0.1m, 向左转弯 85 度  [{'func': 'move', 'x': 0, 'y': 0.1},{'func': 'turn','angle': -85}],
-                        向右移 0.2m, 向前 0.1m [{'func': 'move', 'x': 0, 'y': -0.2},{'func': 'move', 'x': 0.1, 'y': 0}],
+        #                 向右移 0.2m, 向前 0.1m [{'func': 'move', 'x': 0, 'y': -0.2},{'func': 'move', 'x': 0.1, 'y': 0}],
-                        向右转 85 度，向右移 0.1m [{'func': 'turn','angle': 85},{'func': 'move', 'x': 0, 'y': -0.1}],
+        #                 向右转 85 度，向右移 0.1m [{'func': 'turn','angle': 85},{'func': 'move', 'x': 0, 'y': -0.1}],
-                        原地左转 38 度 [{'func': 'turn','angle': -38}],
+        #                 原地左转 38 度 [{'func': 'turn','angle': -38}],
-                        蜂鸣器发声 5 秒 [{'func': 'beep', 'time': 5}]
+        #                 蜂鸣器发声 5 秒 [{'func': 'beep', 'time': 5}]
-                        发光或者照亮 5 秒 [{'func': 'light', 'time': 5}]
+        #                 发光或者照亮 5 秒 [{'func': 'light', 'time': 5}]
-                        向右走 30cm，照亮 2s [{'func': 'move', 'x': 0, 'y': -0.3}, {'func': 'light', 'time': 2}],
+        #                 向右走 30cm，照亮 2s [{'func': 'move', 'x': 0, 'y': -0.3}, {'func': 'light', 'time': 2}],
-                        向左移 0.2m, 向后 0.1m [{'func': 'move', 'x': 0, 'y': 0.2},{'func': 'move', 'x': -0.1, 'y': 0}],
+        #                 向左移 0.2m, 向后 0.1m [{'func': 'move', 'x': 0, 'y': 0.2},{'func': 'move', 'x': -0.1, 'y': 0}],
-                        鸣叫 3 声 [{'func': 'beep', 'time': 3}]
+        #                 鸣叫 3 声 [{'func': 'beep', 'time': 3}]
-                        前行零点五米 [{'func': 'move', 'x': 0.5, 'y': 0}]
+        #                 前行零点五米 [{'func': 'move', 'x': 0.5, 'y': 0}]
+        #             '''
+        # self.prompt = '''
+        #                 你是一个机器人动作规划者，需要把我的话翻译成机器人动作规划并生成对应的 JSON 结果。请注意，只能使用以下指定的动作，不能创造新的动作：
+        #                 允许的动作及其对应格式如下：
+        #                 - 向左移：{"index":N,"action":"go_left","time":T}
+        #                 - 向右移：{"index":N,"action":"go_right","time":T}
+        #                 - 向前移：{"index":N,"action":"go_front","time":T}
+        #                 - 向后移：{"index":N,"action":"go_back","time":T}
+        #                 - 向左转：{"index":N,"action":"go_left_rotate","time":T}
+        #                 - 向右转：{"index":N,"action":"go_right_rotate","time":T}
+        #                 - 蜂鸣器发声：{"index":N,"action":"beep_seconds","time":T}
+        #                 - 蜂鸣器发声次数：{"index":N,"action":"beep_counts","time":T}
+        #                 - 发光或者照亮：{"index":N,"action":"light_seconds","time":T}
+        #                 - 发光次数或者闪烁次数：{"index":N,"action":"light_counts","time":T}
+        #                 - 发光并伴随蜂鸣器：{"index":N,"action":"beep_light_counts","time":T}
+        #                 - 等待{"index":N,"action":"go_sleep","time":T}
+        #                 示例输入输出如下：
+        #                 输入：向左移 0.1m, 向左转弯 85 度
+        #                 输出：[{"index":0,"action":"go_left","time":0.1},{"index":1,"action":"go_left_rotate","time":85}]
+        #                 输入：向右移 0.2m, 向前 0.1m
+        #                 输出：[{"index":0,"action":"go_right","time":0.2},{"index":1,"action":"go_front","time":0.1}]
+        #                 输入：向右转 90 度，向右移 0.1m
+        #                 输出：[{"index":0,"action":"go_right_rotate","time":90},{"index":1,"action":"go_right","time":0.1}]
+        #                 输入：原地左转 38 度
+        #                 输出：[{"index":0,"action":"go_left_rotate","time":38}]
+        #                 输入：蜂鸣器发声 5 秒
+        #                 输出：[{"index":0,"action":"beep_seconds","time":5}]
+        #                 输入：发光或者照亮 5 秒
+        #                 输出：[{"index":0,"action":"light_seconds","time":5}]
+        #                 输入：向右走 30cm, 照亮 2s
+        #                 输出：[{"index":0,"action":"go_right","time":0.3},{"index":1,"action":"light_seconds","time":2}]
+        #                 输入：向左移 0.2m, 向后 0.1m
+        #                 输出：[{"index":0,"action":"go_left","time":0.2},{"index":1,"action":"go_back","time":0.1}]
+        #                 输入：鸣叫 3 声
+        #                 输出：[{"index":0,"action":"beep_counts","time":3}]
+        #                 输入：前行零点五米
+        #                 输出：[{"index":0,"action":"go_front","time":0.5}]
+        #                 输入：闪烁灯光 1 次并伴有蜂鸣器
+        #                 输出：[{"index":0,"action":"beep_light_counts","time": 1}]
+        #                 输入：灯光闪烁 3 次同时蜂鸣器也叫 3 次
+        #                 输出：[{"index":0,"action":"beep_light_counts","time": 3}]
+        #             '''
+        # self.prompt += '''请根据上面的示例，解析该任务文本，并返回相应的 JSON 字段。确保 JSON 中包含了键 index action 和 time 以及相应的值。不要附带其他的解释和注释，只需要 JSON 字段。'''
+        self.prompt = '''
+                        你是一个机器人动作规划者，需要把我的话翻译成机器人动作规划并生成对应的 JSON 结果。请注意，只能使用以下指定的动作，不能创造新的动作：
+                        允许的动作及其对应格式如下：
+                        [{'properties': {'index': {'title': 'Index', 'type': 'integer'}, 'action': {'title': 'Action', 'type': 'string'}, 'time': {'title': 'Time', 'type': 'number'}}, 'required': ['index', 'action', 'time'], 'title': 'Action', 'type': 'object'}]
+                        我不允许你自我创造出新的 action,action 字段仅仅包括以下内容：
+                        go_right 向右移动
+                        go_left 向左移动
+                        go_front 向前移动
+                        go_back 向后移动
+                        go_left_rotate 向左旋转
+                        go_right_rotate 向右旋转
+                        beep_seconds 蜂鸣器鸣叫的时间
+                        beep_counts 蜂鸣器鸣叫的次数
+                        light_seconds 灯光发光的时间
+                        light_counts 灯光闪烁的次数
+                        beep_light_counts 灯光和蜂鸣器一起闪烁的次数
+                        go_sleep 什么都不做
+                        我的话和你的回复示例如下：
+                        我的话：向左移 0.1m, 向左转弯 85 度
+                        你的回复：[{"index":0,"action":"go_left","time":0.1},{"index":1,"action":"go_left_rotate","time":85}]
+                        我的话：向右移 0.2m, 向前 0.1m
+                        你的回复：[{"index":0,"action":"go_right","time":0.2},{"index":1,"action":"go_front","time":0.1}]
+                        我的话：向右转 90 度，向右移 0.1m
+                        你的回复：[{"index":0,"action":"go_right_rotate","time":90},{"index":1,"action":"go_right","time":0.1}]
+                        我的话：原地左转 38 度
+                        你的回复：[{"index":0,"action":"go_left_rotate","time":38}]
+                        我的话：蜂鸣器发声 5 秒
+                        你的回复：[{"index":0,"action":"beep_seconds","time":5}]
+                        我的话：发光或者照亮 5 秒
+                        你的回复：[{"index":0,"action":"light_seconds","time":5}]
+                        我的话：向右走 30cm，照亮 2s
+                        你的回复：[{"index":0,"action":"go_right","time":0.3},{"index":1,"action":"light_seconds","time":2}]
+                        我的话：向左移 0.2m, 向后 0.1m
+                        你的回复：[{"index":0,"action":"go_left","time":0.2},{"index":1,"action":"go_back","time":0.1}]
+                        我的话：鸣叫 3 声
+                        你的回复：[{"index":0,"action":"beep_counts","time":3}]
+                        我的话：前行零点五米
+                        你的回复：[{"index":0,"action":"go_front","time":0.5}]
+                        我的话：闪烁灯光 1 次并伴有蜂鸣器
+                        你的回复：[{"index":0,"action":"beep_light_counts","time": 1}]
+                        我的话：灯光闪烁 3 次同时蜂鸣器也叫 3 次
+                        你的回复：[{"index":0,"action":"beep_light_counts","time": 3}]
+                        
+                        我的话如下：
                     '''
-        self.prompt += '''你只需要根据我的示例解析出指令即可，不要给我其他多余的回复;再次强调 你无需给我其他多余的回复 这对我很重要'''
         self.messages = []
         self.resp = None
         worker = threading.Thread(target=self.reset, daemon=True)  
         worker.start()
     def reset(self):
-        self.messages = [self.make_message(self.prompt)]
+        try:
-        self.resp = erniebot.ChatCompletion.create(
+            self.messages = [self.make_message(self.prompt)]
-                        model=self.model,
+            self.resp = erniebot.ChatCompletion.create(
-                        messages=self.messages,
+                            model=self.model,
-                    )
+                            messages=self.messages,
-        self.messages.append(self.resp.to_message())
+                        )
-        self.init_done_flag = True
+            self.messages.append(self.resp.to_message())
-        logger.info("LLM init done")
+            self.init_done_flag = True
+            logger.info("LLM init done")
+        except:
+            logger.error("LLM init error")
     def make_message(self,content):
         return {'role': 'user', 'content': content}
     def get_command_json(self,chat):
         while self.init_done_flag == False: # 等待初始化 (要是等到调用还没初始化，那就是真寄了)
             pass
+        chat = '我的话如下：' + chat
         self.messages.append(self.make_message(chat))
         self.resp = erniebot.ChatCompletion.create(
                         model=self.model,
                         messages=self.messages,
         )
         self.messages.append(self.resp.to_message())
-        resp = self.resp.get_result().replace(' ', '').replace('\n', '').replace('\t', '')
+        resp = self.resp.get_result().replace(' ', '')
         return resp
 
 class CountRecord: