384 lines
21 KiB
Python
384 lines
21 KiB
Python
"""Exercise the actual Flask tool loop with synthetic provider messages and data."""
|
|
|
|
from copy import deepcopy
|
|
import json
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
import test_model_chat as fixtures
|
|
from test_quote_timing import timing_snapshot
|
|
from test_quotes import frames
|
|
from logistics_chat import db
|
|
from logistics_chat.chat_tools import execute_tool as actual_execute_tool
|
|
from logistics_chat.order_listing import read_orders as actual_read_orders
|
|
from logistics_chat.quote_data import import_snapshot, load_book
|
|
|
|
|
|
def tool_plan(name, arguments, call_id):
|
|
return fixtures.Completion({'role': 'assistant', 'content': '', 'tool_calls': [
|
|
{'id': call_id, 'type': 'function', 'function': {
|
|
'name': name, 'arguments': json.dumps(arguments, ensure_ascii=False)}}]}, 'tool_calls')
|
|
|
|
|
|
def search_arguments(question='美国海派外箱标签有什么要求'):
|
|
return {'question': question, 'country': 'US', 'services': ['普船海派'], 'topic': ''}
|
|
|
|
|
|
def quote_arguments(**updates):
|
|
result = {'question': '请核对当前货物报价', 'request': 'quote', 'selection': 'current',
|
|
'services': [], 'policy_topic': 'auto'}
|
|
result.update(updates)
|
|
return result
|
|
|
|
|
|
def final_facts(*ids):
|
|
return fixtures.reply(json.dumps({'facts': list(ids)}, ensure_ascii=False))
|
|
|
|
|
|
def evidence(body):
|
|
return [json.loads(item['content']) for item in body['messages'] if item['role'] == 'tool']
|
|
|
|
|
|
class ModelToolLoopTests(unittest.TestCase):
|
|
def setUp(self):
|
|
fixtures.ModelChatTests.setUp(self)
|
|
self.snapshot = timing_snapshot()
|
|
self.snapshot['source_sha256'] = '2' * 64
|
|
self.snapshot['documents'] = [{'sheet': '海派', 'cell': 'A30',
|
|
'text': '外箱必须贴入仓条码和唛头；旧标须移除，但产品原产地标识不得遮挡。'}]
|
|
with self.app.app_context():
|
|
connection = db.connect()
|
|
self.book_id = import_snapshot(connection, self.snapshot, activate=True)['book_id']
|
|
connection.commit()
|
|
connection.close()
|
|
self.initial_rows = self.saved()[0]
|
|
|
|
def login(self):
|
|
fixtures.ModelChatTests.login(self)
|
|
|
|
def ask(self, question):
|
|
events = frames(self.client.post('/stream_chat', json={'msg': question, 'conv_id': self.cid}))
|
|
self.assertTrue(events and events[-1].get('d'), events)
|
|
self.assertNotIn('SECRET', str(events))
|
|
return events
|
|
|
|
def saved(self):
|
|
with self.app.app_context():
|
|
connection = db.connect()
|
|
rows = connection.execute('SELECT role,content FROM messages WHERE conv_id=? ORDER BY id',
|
|
(self.cid,)).fetchall()
|
|
state = connection.execute('SELECT conversation_state FROM chat_history WHERE id=?',
|
|
(self.cid,)).fetchone()
|
|
connection.close()
|
|
return rows, json.loads(state[0]) if state and state[0] else None
|
|
|
|
def test_search_read_final_preserves_tool_ids_and_only_renders_selected_facts(self):
|
|
bodies = []
|
|
def provider(body, **kwargs):
|
|
bodies.append(deepcopy(body))
|
|
if len(bodies) == 1:
|
|
return tool_plan('search_knowledge', search_arguments(), 'search_a')
|
|
if len(bodies) == 2:
|
|
facts = evidence(body)
|
|
self.assertEqual(facts[0]['status'], 'ok')
|
|
ref = facts[0]['data']['matches'][0]['reference']
|
|
return tool_plan('read_knowledge', {'question': '查看完整贴标条款', 'reference': ref}, 'read_b')
|
|
self.assertEqual(len(bodies), 3)
|
|
facts = evidence(body)
|
|
self.assertEqual([item['fact_id'] for item in facts], ['fact_0', 'fact_1'])
|
|
self.assertEqual(facts[1]['data_version'], self.book_id)
|
|
return final_facts('fact_1')
|
|
with patch('logistics_chat.ai_client.post_completion', side_effect=provider):
|
|
events = self.ask('美国海派标签怎么贴，需要看完整要求')
|
|
text = events[0]['t']
|
|
self.assertIn('不得遮挡', text)
|
|
self.assertEqual(text.count('不得遮挡'), 1)
|
|
self.assertNotIn('fact_1', text)
|
|
tool_messages = [item for item in bodies[2]['messages'] if item['role'] == 'tool']
|
|
self.assertEqual([item['tool_call_id'] for item in tool_messages], ['search_a', 'read_b'])
|
|
self.assertEqual([role for role, _ in self.saved()[0][len(self.initial_rows):]], ['user', 'assistant'])
|
|
|
|
def test_activating_new_book_between_search_and_read_keeps_the_turn_snapshot(self):
|
|
bodies = []
|
|
def provider(body, **kwargs):
|
|
bodies.append(deepcopy(body))
|
|
if len(bodies) == 1:
|
|
return tool_plan('search_knowledge', search_arguments(), 'search_old')
|
|
if len(bodies) == 2:
|
|
ref = evidence(body)[0]['data']['matches'][0]['reference']
|
|
changed = deepcopy(self.snapshot)
|
|
changed['source_sha256'] = '3' * 64
|
|
changed['documents'][0]['text'] = '新版外箱贴标必须加贴红色标识。'
|
|
connection = db.connect()
|
|
self.new_book_id = import_snapshot(connection, changed, activate=True)['book_id']
|
|
connection.commit()
|
|
connection.close()
|
|
return tool_plan('read_knowledge', {'question': '继续读取刚才的原文', 'reference': ref}, 'read_old')
|
|
facts = evidence(body)
|
|
self.assertEqual([item['data_version'] for item in facts], [self.book_id, self.book_id])
|
|
return final_facts('fact_1')
|
|
with patch('logistics_chat.ai_client.post_completion', side_effect=provider):
|
|
text = self.ask('美国海派外箱标签怎么贴')[0]['t']
|
|
self.assertIn('不得遮挡', text)
|
|
self.assertNotIn('红色', text)
|
|
with self.app.app_context():
|
|
self.assertEqual(load_book()['id'], self.new_book_id)
|
|
|
|
def test_forged_reference_cannot_read_a_document_without_this_turn_search(self):
|
|
responses = [tool_plan('read_knowledge', {'question': '读取资料',
|
|
'reference': f'kb1:b:{self.book_id}:' + 'a' * 24}, 'forged'), final_facts('fact_0')]
|
|
with patch('logistics_chat.ai_client.post_completion', side_effect=responses) as model:
|
|
events = self.ask('给我这份资料的原文')
|
|
fact = evidence(model.call_args_list[1].args[0])[0]
|
|
self.assertEqual(fact['status'], 'not_found')
|
|
self.assertEqual(fact['sources'], [])
|
|
self.assertNotIn('不得遮挡', events[0]['t'])
|
|
|
|
def test_multi_region_faq_read_keeps_the_requested_country_scope(self):
|
|
calls = 0
|
|
def provider(body, **kwargs):
|
|
nonlocal calls
|
|
calls += 1
|
|
if calls == 1:
|
|
return tool_plan('search_knowledge', {'question': '英国各渠道时效',
|
|
'country': 'GB', 'services': [], 'topic': ''}, 'regional_search')
|
|
if calls == 2:
|
|
matches = evidence(body)[0]['data']['matches']
|
|
shared = next(match for match in matches if match['country'] == 'EU,GB')
|
|
return tool_plan('read_knowledge', {'question': '读取这份共同条款',
|
|
'reference': shared['reference']}, 'regional_read')
|
|
self.assertEqual(evidence(body)[-1]['status'], 'ok')
|
|
return final_facts('fact_1')
|
|
with patch('logistics_chat.ai_client.post_completion', side_effect=provider):
|
|
text = self.ask('英国各渠道时效多少')[0]['t']
|
|
self.assertIn('英国', text)
|
|
self.assertNotIn('范围已变化', text)
|
|
|
|
def test_query_inferred_country_survives_search_reference_registration(self):
|
|
calls = 0
|
|
def provider(body, **kwargs):
|
|
nonlocal calls
|
|
calls += 1
|
|
if calls == 1:
|
|
return tool_plan('search_knowledge', {'question': '美国海派外箱标签要求',
|
|
'country': '', 'services': ['普船海派'], 'topic': ''}, 'inferred_search')
|
|
if calls == 2:
|
|
result = evidence(body)[0]
|
|
self.assertEqual(result['data']['resolved_country'], 'US')
|
|
ref = result['data']['matches'][0]['reference']
|
|
return tool_plan('read_knowledge', {'question': '读取完整条款',
|
|
'reference': ref}, 'inferred_read')
|
|
self.assertEqual(evidence(body)[-1]['status'], 'ok')
|
|
return final_facts('fact_1')
|
|
with patch('logistics_chat.ai_client.post_completion', side_effect=provider):
|
|
text = self.ask('美国海派外箱标签有什么要求')[0]['t']
|
|
self.assertIn('不得遮挡', text)
|
|
|
|
def test_repeated_arguments_reuse_evidence_but_return_each_distinct_tool_id(self):
|
|
arguments = search_arguments()
|
|
responses = [tool_plan('search_knowledge', arguments, 'first'),
|
|
tool_plan('search_knowledge', arguments, 'repeat'), final_facts('fact_0')]
|
|
with patch('logistics_chat.ai_client.post_completion', side_effect=responses) as model, \
|
|
patch('logistics_chat.model_chat_service.execute_tool', wraps=actual_execute_tool) as execute:
|
|
events = self.ask('再核对美国海派贴标要求')
|
|
self.assertEqual(execute.call_count, 1)
|
|
facts = evidence(model.call_args_list[2].args[0])
|
|
self.assertEqual([fact['fact_id'] for fact in facts], ['fact_0', 'fact_0'])
|
|
self.assertIn('不得遮挡', events[0]['t'])
|
|
|
|
def test_duplicate_call_id_and_tool_budget_stop_further_execution(self):
|
|
for duplicate in (True, False):
|
|
with self.subTest(duplicate=duplicate):
|
|
self.cid = self.client.post('/api/new_chat').json['id']
|
|
self.app.config['MODEL_MAX_TOOL_CALLS'] = 6 if duplicate else 1
|
|
second = 'first' if duplicate else 'second'
|
|
responses = [tool_plan('search_knowledge', search_arguments(), 'first'),
|
|
tool_plan('search_knowledge', search_arguments('美国海派标签原文'), second)]
|
|
with patch('logistics_chat.ai_client.post_completion', side_effect=responses) as model, \
|
|
patch('logistics_chat.model_chat_service.execute_tool', wraps=actual_execute_tool) as execute:
|
|
events = self.ask('美国海派贴标要求')
|
|
self.assertEqual(model.call_count, 2)
|
|
self.assertEqual(execute.call_count, 1)
|
|
self.assertIn('不得遮挡', events[0]['t'])
|
|
|
|
def test_nested_invalid_parameters_reject_the_entire_batch_before_tools_run(self):
|
|
valid = tool_plan('search_knowledge', search_arguments(), 'good').message['tool_calls'][0]
|
|
malformed = quote_arguments(shipment={'packages': [
|
|
{'count': 1, 'weight_kg': '20', 'tenant_id': 'b'}]})
|
|
invalid = tool_plan('quote_shipping', malformed, 'bad').message['tool_calls'][0]
|
|
response = fixtures.Completion({'role': 'assistant', 'content': '',
|
|
'tool_calls': [valid, invalid]}, 'tool_calls')
|
|
with patch('logistics_chat.ai_client.post_completion', return_value=response) as model, \
|
|
patch('logistics_chat.model_chat_service.execute_tool', wraps=actual_execute_tool) as execute:
|
|
events = self.ask('请协助了解一下')
|
|
self.assertEqual(model.call_count, 1)
|
|
execute.assert_not_called()
|
|
self.assertIn('e', events[0])
|
|
|
|
def test_revoked_login_stops_before_private_tool_facts_reach_the_model(self):
|
|
self.login()
|
|
order_plan = fixtures.plan(('query_orders', '我的异常订单有多少个'))
|
|
def execute(name, arguments, conv_id, **kwargs):
|
|
result = actual_execute_tool(name, arguments, conv_id, **kwargs)
|
|
self.assertTrue(result.get('_customer_data'))
|
|
connection = db.connect()
|
|
connection.execute('DELETE FROM customer_sessions')
|
|
connection.commit()
|
|
connection.close()
|
|
return result
|
|
with patch('logistics_chat.ai_client.post_completion', return_value=order_plan) as model, \
|
|
patch('logistics_chat.model_chat_service.execute_tool', side_effect=execute):
|
|
events = self.ask('盘一下异常订单')
|
|
self.assertEqual(model.call_count, 1)
|
|
self.assertNotIn('等待资料', json.dumps(model.call_args.args[0], ensure_ascii=False))
|
|
self.assertIn('e', events[0])
|
|
self.assertNotIn('等待资料', str(events))
|
|
self.assertEqual(self.saved()[0], self.initial_rows)
|
|
|
|
def test_planner_timeout_after_session_revocation_cannot_fallback_to_private_orders(self):
|
|
self.login()
|
|
def provider(body, **kwargs):
|
|
connection = db.connect()
|
|
connection.execute('DELETE FROM customer_sessions')
|
|
connection.commit()
|
|
connection.close()
|
|
raise fixtures.requests.exceptions.Timeout('synthetic planner timeout')
|
|
with patch('logistics_chat.ai_client.post_completion', side_effect=provider) as model, \
|
|
patch('logistics_chat.order_listing.read_orders', wraps=actual_read_orders) as read:
|
|
events = self.ask('我的异常订单有多少个')
|
|
self.assertEqual(model.call_count, 1)
|
|
read.assert_not_called()
|
|
self.assertTrue(events[0].get('e') or events[0].get('login_required'), events)
|
|
serialized = json.dumps(events, ensure_ascii=False)
|
|
self.assertNotIn('VISIBLE20260001', serialized)
|
|
self.assertNotIn('等待资料', serialized)
|
|
|
|
def test_order_card_does_not_end_a_turn_before_the_policy_task(self):
|
|
self.login()
|
|
responses = [fixtures.plan(('query_orders', '查入仓码INBOUND20260001')),
|
|
tool_plan('search_knowledge', search_arguments(), 'policy_after_order'),
|
|
final_facts('fact_0', 'fact_1')]
|
|
with patch('logistics_chat.ai_client.post_completion', side_effect=responses) as model:
|
|
events = self.ask('查INBOUND20260001这票，另外美国海派的外箱标签有什么要求')
|
|
self.assertEqual(model.call_count, 3)
|
|
second_request_facts = evidence(model.call_args_list[1].args[0])
|
|
self.assertIn('VISIBLE20260001', second_request_facts[0]['result'])
|
|
text = json.dumps(events, ensure_ascii=False)
|
|
self.assertIn('VISIBLE20260001', text)
|
|
self.assertIn('不得遮挡', text)
|
|
|
|
def test_two_structured_quote_calls_apply_addition_once_and_commit_one_state(self):
|
|
start = quote_arguments(shipment={'country': 'US', 'total_weight_kg': '1000',
|
|
'zone': '美东', 'origin': '义乌', 'kinds': ['parcel'], 'tax': 'included'}, new_shipment=True)
|
|
with patch('logistics_chat.ai_client.post_completion', side_effect=[
|
|
tool_plan('quote_shipping', start, 'start'), final_facts('fact_0')]):
|
|
self.ask('这批一吨货去美国美东，从义乌发')
|
|
self.assertEqual(self.saved()[1]['intent']['weight'], '1000')
|
|
changes = [{'field': 'total_weight_kg', 'operation': 'add', 'value': '100'}]
|
|
responses = [tool_plan('quote_shipping', quote_arguments(changes=changes, selection='fastest'), 'fast'),
|
|
tool_plan('quote_shipping', quote_arguments(changes=changes, selection='cheapest'), 'cheap'),
|
|
final_facts('fact_0', 'fact_1')]
|
|
with patch('logistics_chat.ai_client.post_completion', side_effect=responses) as model:
|
|
text = self.ask('再添一百公斤，分别看最快和最便宜')[0]['t']
|
|
facts = evidence(model.call_args_list[2].args[0])
|
|
self.assertEqual([item['confirmed_inputs']['total_weight_kg'] for item in facts], ['1100', '1100'])
|
|
self.assertIn('1100', text)
|
|
self.assertNotIn('1200KG', text)
|
|
rows, state = self.saved()
|
|
self.assertEqual(state['intent']['weight'], '1100')
|
|
self.assertEqual(state['version'], 2)
|
|
self.assertEqual(len(rows) - len(self.initial_rows), 4)
|
|
|
|
def test_later_dimensions_supersede_old_quote_facts_and_do_not_revive_cached_prices(self):
|
|
initial = quote_arguments(services=['普船海派'], shipment={
|
|
'country': 'US', 'total_weight_kg': '10', 'boxes': 1, 'zone': '美西',
|
|
'origin': '义乌', 'kinds': ['parcel']})
|
|
dimensions = quote_arguments(services=['普船海派'], shipment={'dimensions_cm': ['100', '100', '100']})
|
|
bodies = []
|
|
def provider(body, **kwargs):
|
|
bodies.append(deepcopy(body))
|
|
if len(bodies) == 1:
|
|
return tool_plan('quote_shipping', initial, 'without_dimensions')
|
|
if len(bodies) == 2:
|
|
self.assertIn('48元', evidence(body)[0]['result'])
|
|
return tool_plan('quote_shipping', dimensions, 'with_dimensions')
|
|
if len(bodies) == 3:
|
|
first, second = evidence(body)
|
|
self.assertEqual(first['status'], 'superseded')
|
|
self.assertFalse(first['selectable'])
|
|
self.assertNotIn('48元', first['result'])
|
|
self.assertIn('666.67元', second['result'])
|
|
return tool_plan('quote_shipping', initial, 'repeat_original_arguments')
|
|
self.assertEqual(len(bodies), 4)
|
|
facts = evidence(body)
|
|
self.assertEqual([item['fact_id'] for item in facts], ['fact_0', 'fact_1', 'fact_2'])
|
|
self.assertIn('666.67元', facts[2]['result'])
|
|
self.assertNotIn('48元', facts[2]['result'])
|
|
# Even a model that selects a superseded ID cannot render its price.
|
|
return final_facts('fact_0', 'fact_1', 'fact_2')
|
|
with patch('logistics_chat.ai_client.post_completion', side_effect=provider):
|
|
text = self.ask('这票10KG一箱，箱规100乘100乘100厘米，核算普船美西价')[0]['t']
|
|
self.assertNotIn('48元', text)
|
|
self.assertIn('666.67元', text)
|
|
self.assertEqual(len(self.saved()[1]['options']), 1)
|
|
|
|
def test_conflicting_quote_amendment_removes_prior_prices_and_candidate(self):
|
|
initial = quote_arguments(services=['普船海派'], shipment={
|
|
'country': 'US', 'total_weight_kg': '10', 'boxes': 1, 'zone': '美西',
|
|
'origin': '义乌', 'kinds': ['parcel']})
|
|
conflicting = quote_arguments(changes=[{'field': 'total_weight_kg', 'operation': 'add', 'value': '100'}])
|
|
responses = [tool_plan('quote_shipping', initial, 'quote_first'),
|
|
tool_plan('quote_shipping', conflicting, 'conflict_second'),
|
|
final_facts('fact_0', 'fact_1')]
|
|
with patch('logistics_chat.ai_client.post_completion', side_effect=responses) as model:
|
|
text = self.ask('核对这票重量和报价')[0]['t']
|
|
facts = evidence(model.call_args_list[2].args[0])
|
|
self.assertEqual(facts[0]['status'], 'superseded')
|
|
self.assertEqual(facts[1]['status'], 'needs_clarification')
|
|
self.assertIn('冲突', text)
|
|
self.assertNotIn('48元', text)
|
|
self.assertIsNone(self.saved()[1])
|
|
|
|
def test_deleted_conversation_does_not_save_quote_candidate_or_reply(self):
|
|
count = 0
|
|
def provider(body, **kwargs):
|
|
nonlocal count
|
|
count += 1
|
|
if count == 1:
|
|
return tool_plan('quote_shipping', quote_arguments(shipment={
|
|
'country': 'US', 'total_weight_kg': '1000', 'zone': '美东',
|
|
'origin': '义乌', 'kinds': ['parcel']}), 'quote_before_delete')
|
|
connection = db.connect()
|
|
connection.execute('DELETE FROM messages WHERE conv_id=?', (self.cid,))
|
|
connection.execute('DELETE FROM chat_history WHERE id=?', (self.cid,))
|
|
connection.commit()
|
|
connection.close()
|
|
return final_facts('fact_0')
|
|
with patch('logistics_chat.ai_client.post_completion', side_effect=provider):
|
|
events = self.ask('请核算一吨货到美国美东')
|
|
self.assertIn('e', events[0])
|
|
self.assertEqual(self.saved(), ([], None))
|
|
|
|
def test_late_model_reply_rolls_back_its_messages_and_candidate_state(self):
|
|
count = 0
|
|
def provider(body, **kwargs):
|
|
nonlocal count
|
|
count += 1
|
|
if count == 1:
|
|
return tool_plan('quote_shipping', quote_arguments(shipment={
|
|
'country': 'US', 'total_weight_kg': '1000', 'zone': '美东',
|
|
'origin': '义乌', 'kinds': ['parcel']}), 'slow_quote')
|
|
db.save_message(self.cid, 'user', '较新的问题')
|
|
db.save_message(self.cid, 'assistant', '较新的回答')
|
|
return final_facts('fact_0')
|
|
with patch('logistics_chat.ai_client.post_completion', side_effect=provider):
|
|
events = self.ask('这是一条迟到的报价请求')
|
|
self.assertIn('e', events[0])
|
|
rows, state = self.saved()
|
|
self.assertEqual(rows, self.initial_rows + [('user', '较新的问题'), ('assistant', '较新的回答')])
|
|
self.assertIsNone(state)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|