"""Exercise the actual Flask tool loop with synthetic provider messages and data.""" from copy import deepcopy import json import unittest from unittest.mock import patch import test_model_chat as fixtures from test_quote_timing import timing_snapshot from test_quotes import frames from logistics_chat import db from logistics_chat.chat_tools import execute_tool as actual_execute_tool from logistics_chat.order_listing import read_orders as actual_read_orders from logistics_chat.quote_data import import_snapshot, load_book def tool_plan(name, arguments, call_id): return fixtures.Completion({'role': 'assistant', 'content': '', 'tool_calls': [ {'id': call_id, 'type': 'function', 'function': { 'name': name, 'arguments': json.dumps(arguments, ensure_ascii=False)}}]}, 'tool_calls') def search_arguments(question='美国海派外箱标签有什么要求'): return {'question': question, 'country': 'US', 'services': ['普船海派'], 'topic': ''} def quote_arguments(**updates): result = {'question': '请核对当前货物报价', 'request': 'quote', 'selection': 'current', 'services': [], 'policy_topic': 'auto'} result.update(updates) return result def final_facts(*ids): return fixtures.reply(json.dumps({'facts': list(ids)}, ensure_ascii=False)) def evidence(body): return [json.loads(item['content']) for item in body['messages'] if item['role'] == 'tool'] class ModelToolLoopTests(unittest.TestCase): def setUp(self): fixtures.ModelChatTests.setUp(self) self.snapshot = timing_snapshot() self.snapshot['source_sha256'] = '2' * 64 self.snapshot['documents'] = [{'sheet': '海派', 'cell': 'A30', 'text': '外箱必须贴入仓条码和唛头；旧标须移除，但产品原产地标识不得遮挡。'}] with self.app.app_context(): connection = db.connect() self.book_id = import_snapshot(connection, self.snapshot, activate=True)['book_id'] connection.commit() connection.close() self.initial_rows = self.saved()[0] def login(self): fixtures.ModelChatTests.login(self) def ask(self, question): events = frames(self.client.post('/stream_chat', json={'msg': question, 'conv_id': self.cid})) self.assertTrue(events and events[-1].get('d'), events) self.assertNotIn('SECRET', str(events)) return events def saved(self): with self.app.app_context(): connection = db.connect() rows = connection.execute('SELECT role,content FROM messages WHERE conv_id=? ORDER BY id', (self.cid,)).fetchall() state = connection.execute('SELECT conversation_state FROM chat_history WHERE id=?', (self.cid,)).fetchone() connection.close() return rows, json.loads(state[0]) if state and state[0] else None def test_search_read_final_preserves_tool_ids_and_only_renders_selected_facts(self): bodies = [] def provider(body, **kwargs): bodies.append(deepcopy(body)) if len(bodies) == 1: return tool_plan('search_knowledge', search_arguments(), 'search_a') if len(bodies) == 2: facts = evidence(body) self.assertEqual(facts[0]['status'], 'ok') ref = facts[0]['data']['matches'][0]['reference'] return tool_plan('read_knowledge', {'question': '查看完整贴标条款', 'reference': ref}, 'read_b') self.assertEqual(len(bodies), 3) facts = evidence(body) self.assertEqual([item['fact_id'] for item in facts], ['fact_0', 'fact_1']) self.assertEqual(facts[1]['data_version'], self.book_id) return final_facts('fact_1') with patch('logistics_chat.ai_client.post_completion', side_effect=provider): events = self.ask('美国海派标签怎么贴，需要看完整要求') text = events[0]['t'] self.assertIn('不得遮挡', text) self.assertEqual(text.count('不得遮挡'), 1) self.assertNotIn('fact_1', text) tool_messages = [item for item in bodies[2]['messages'] if item['role'] == 'tool'] self.assertEqual([item['tool_call_id'] for item in tool_messages], ['search_a', 'read_b']) self.assertEqual([role for role, _ in self.saved()[0][len(self.initial_rows):]], ['user', 'assistant']) def test_activating_new_book_between_search_and_read_keeps_the_turn_snapshot(self): bodies = [] def provider(body, **kwargs): bodies.append(deepcopy(body)) if len(bodies) == 1: return tool_plan('search_knowledge', search_arguments(), 'search_old') if len(bodies) == 2: ref = evidence(body)[0]['data']['matches'][0]['reference'] changed = deepcopy(self.snapshot) changed['source_sha256'] = '3' * 64 changed['documents'][0]['text'] = '新版外箱贴标必须加贴红色标识。' connection = db.connect() self.new_book_id = import_snapshot(connection, changed, activate=True)['book_id'] connection.commit() connection.close() return tool_plan('read_knowledge', {'question': '继续读取刚才的原文', 'reference': ref}, 'read_old') facts = evidence(body) self.assertEqual([item['data_version'] for item in facts], [self.book_id, self.book_id]) return final_facts('fact_1') with patch('logistics_chat.ai_client.post_completion', side_effect=provider): text = self.ask('美国海派外箱标签怎么贴')[0]['t'] self.assertIn('不得遮挡', text) self.assertNotIn('红色', text) with self.app.app_context(): self.assertEqual(load_book()['id'], self.new_book_id) def test_forged_reference_cannot_read_a_document_without_this_turn_search(self): responses = [tool_plan('read_knowledge', {'question': '读取资料', 'reference': f'kb1:b:{self.book_id}:' + 'a' * 24}, 'forged'), final_facts('fact_0')] with patch('logistics_chat.ai_client.post_completion', side_effect=responses) as model: events = self.ask('给我这份资料的原文') fact = evidence(model.call_args_list[1].args[0])[0] self.assertEqual(fact['status'], 'not_found') self.assertEqual(fact['sources'], []) self.assertNotIn('不得遮挡', events[0]['t']) def test_multi_region_faq_read_keeps_the_requested_country_scope(self): calls = 0 def provider(body, **kwargs): nonlocal calls calls += 1 if calls == 1: return tool_plan('search_knowledge', {'question': '英国各渠道时效', 'country': 'GB', 'services': [], 'topic': ''}, 'regional_search') if calls == 2: matches = evidence(body)[0]['data']['matches'] shared = next(match for match in matches if match['country'] == 'EU,GB') return tool_plan('read_knowledge', {'question': '读取这份共同条款', 'reference': shared['reference']}, 'regional_read') self.assertEqual(evidence(body)[-1]['status'], 'ok') return final_facts('fact_1') with patch('logistics_chat.ai_client.post_completion', side_effect=provider): text = self.ask('英国各渠道时效多少')[0]['t'] self.assertIn('英国', text) self.assertNotIn('范围已变化', text) def test_query_inferred_country_survives_search_reference_registration(self): calls = 0 def provider(body, **kwargs): nonlocal calls calls += 1 if calls == 1: return tool_plan('search_knowledge', {'question': '美国海派外箱标签要求', 'country': '', 'services': ['普船海派'], 'topic': ''}, 'inferred_search') if calls == 2: result = evidence(body)[0] self.assertEqual(result['data']['resolved_country'], 'US') ref = result['data']['matches'][0]['reference'] return tool_plan('read_knowledge', {'question': '读取完整条款', 'reference': ref}, 'inferred_read') self.assertEqual(evidence(body)[-1]['status'], 'ok') return final_facts('fact_1') with patch('logistics_chat.ai_client.post_completion', side_effect=provider): text = self.ask('美国海派外箱标签有什么要求')[0]['t'] self.assertIn('不得遮挡', text) def test_repeated_arguments_reuse_evidence_but_return_each_distinct_tool_id(self): arguments = search_arguments() responses = [tool_plan('search_knowledge', arguments, 'first'), tool_plan('search_knowledge', arguments, 'repeat'), final_facts('fact_0')] with patch('logistics_chat.ai_client.post_completion', side_effect=responses) as model, \ patch('logistics_chat.model_chat_service.execute_tool', wraps=actual_execute_tool) as execute: events = self.ask('再核对美国海派贴标要求') self.assertEqual(execute.call_count, 1) facts = evidence(model.call_args_list[2].args[0]) self.assertEqual([fact['fact_id'] for fact in facts], ['fact_0', 'fact_0']) self.assertIn('不得遮挡', events[0]['t']) def test_duplicate_call_id_and_tool_budget_stop_further_execution(self): for duplicate in (True, False): with self.subTest(duplicate=duplicate): self.cid = self.client.post('/api/new_chat').json['id'] self.app.config['MODEL_MAX_TOOL_CALLS'] = 6 if duplicate else 1 second = 'first' if duplicate else 'second' responses = [tool_plan('search_knowledge', search_arguments(), 'first'), tool_plan('search_knowledge', search_arguments('美国海派标签原文'), second)] with patch('logistics_chat.ai_client.post_completion', side_effect=responses) as model, \ patch('logistics_chat.model_chat_service.execute_tool', wraps=actual_execute_tool) as execute: events = self.ask('美国海派贴标要求') self.assertEqual(model.call_count, 2) self.assertEqual(execute.call_count, 1) self.assertIn('不得遮挡', events[0]['t']) def test_nested_invalid_parameters_reject_the_entire_batch_before_tools_run(self): valid = tool_plan('search_knowledge', search_arguments(), 'good').message['tool_calls'][0] malformed = quote_arguments(shipment={'packages': [ {'count': 1, 'weight_kg': '20', 'tenant_id': 'b'}]}) invalid = tool_plan('quote_shipping', malformed, 'bad').message['tool_calls'][0] response = fixtures.Completion({'role': 'assistant', 'content': '', 'tool_calls': [valid, invalid]}, 'tool_calls') with patch('logistics_chat.ai_client.post_completion', return_value=response) as model, \ patch('logistics_chat.model_chat_service.execute_tool', wraps=actual_execute_tool) as execute: events = self.ask('请协助了解一下') self.assertEqual(model.call_count, 1) execute.assert_not_called() self.assertIn('e', events[0]) def test_revoked_login_stops_before_private_tool_facts_reach_the_model(self): self.login() order_plan = fixtures.plan(('query_orders', '我的异常订单有多少个')) def execute(name, arguments, conv_id, **kwargs): result = actual_execute_tool(name, arguments, conv_id, **kwargs) self.assertTrue(result.get('_customer_data')) connection = db.connect() connection.execute('DELETE FROM customer_sessions') connection.commit() connection.close() return result with patch('logistics_chat.ai_client.post_completion', return_value=order_plan) as model, \ patch('logistics_chat.model_chat_service.execute_tool', side_effect=execute): events = self.ask('盘一下异常订单') self.assertEqual(model.call_count, 1) self.assertNotIn('等待资料', json.dumps(model.call_args.args[0], ensure_ascii=False)) self.assertIn('e', events[0]) self.assertNotIn('等待资料', str(events)) self.assertEqual(self.saved()[0], self.initial_rows) def test_planner_timeout_after_session_revocation_cannot_fallback_to_private_orders(self): self.login() def provider(body, **kwargs): connection = db.connect() connection.execute('DELETE FROM customer_sessions') connection.commit() connection.close() raise fixtures.requests.exceptions.Timeout('synthetic planner timeout') with patch('logistics_chat.ai_client.post_completion', side_effect=provider) as model, \ patch('logistics_chat.order_listing.read_orders', wraps=actual_read_orders) as read: events = self.ask('我的异常订单有多少个') self.assertEqual(model.call_count, 1) read.assert_not_called() self.assertTrue(events[0].get('e') or events[0].get('login_required'), events) serialized = json.dumps(events, ensure_ascii=False) self.assertNotIn('VISIBLE20260001', serialized) self.assertNotIn('等待资料', serialized) def test_order_card_does_not_end_a_turn_before_the_policy_task(self): self.login() responses = [fixtures.plan(('query_orders', '查入仓码INBOUND20260001')), tool_plan('search_knowledge', search_arguments(), 'policy_after_order'), final_facts('fact_0', 'fact_1')] with patch('logistics_chat.ai_client.post_completion', side_effect=responses) as model: events = self.ask('查INBOUND20260001这票，另外美国海派的外箱标签有什么要求') self.assertEqual(model.call_count, 3) second_request_facts = evidence(model.call_args_list[1].args[0]) self.assertIn('VISIBLE20260001', second_request_facts[0]['result']) text = json.dumps(events, ensure_ascii=False) self.assertIn('VISIBLE20260001', text) self.assertIn('不得遮挡', text) def test_two_structured_quote_calls_apply_addition_once_and_commit_one_state(self): start = quote_arguments(shipment={'country': 'US', 'total_weight_kg': '1000', 'zone': '美东', 'origin': '义乌', 'kinds': ['parcel'], 'tax': 'included'}, new_shipment=True) with patch('logistics_chat.ai_client.post_completion', side_effect=[ tool_plan('quote_shipping', start, 'start'), final_facts('fact_0')]): self.ask('这批一吨货去美国美东，从义乌发') self.assertEqual(self.saved()[1]['intent']['weight'], '1000') changes = [{'field': 'total_weight_kg', 'operation': 'add', 'value': '100'}] responses = [tool_plan('quote_shipping', quote_arguments(changes=changes, selection='fastest'), 'fast'), tool_plan('quote_shipping', quote_arguments(changes=changes, selection='cheapest'), 'cheap'), final_facts('fact_0', 'fact_1')] with patch('logistics_chat.ai_client.post_completion', side_effect=responses) as model: text = self.ask('再添一百公斤，分别看最快和最便宜')[0]['t'] facts = evidence(model.call_args_list[2].args[0]) self.assertEqual([item['confirmed_inputs']['total_weight_kg'] for item in facts], ['1100', '1100']) self.assertIn('1100', text) self.assertNotIn('1200KG', text) rows, state = self.saved() self.assertEqual(state['intent']['weight'], '1100') self.assertEqual(state['version'], 2) self.assertEqual(len(rows) - len(self.initial_rows), 4) def test_later_dimensions_supersede_old_quote_facts_and_do_not_revive_cached_prices(self): initial = quote_arguments(services=['普船海派'], shipment={ 'country': 'US', 'total_weight_kg': '10', 'boxes': 1, 'zone': '美西', 'origin': '义乌', 'kinds': ['parcel']}) dimensions = quote_arguments(services=['普船海派'], shipment={'dimensions_cm': ['100', '100', '100']}) bodies = [] def provider(body, **kwargs): bodies.append(deepcopy(body)) if len(bodies) == 1: return tool_plan('quote_shipping', initial, 'without_dimensions') if len(bodies) == 2: self.assertIn('48元', evidence(body)[0]['result']) return tool_plan('quote_shipping', dimensions, 'with_dimensions') if len(bodies) == 3: first, second = evidence(body) self.assertEqual(first['status'], 'superseded') self.assertFalse(first['selectable']) self.assertNotIn('48元', first['result']) self.assertIn('666.67元', second['result']) return tool_plan('quote_shipping', initial, 'repeat_original_arguments') self.assertEqual(len(bodies), 4) facts = evidence(body) self.assertEqual([item['fact_id'] for item in facts], ['fact_0', 'fact_1', 'fact_2']) self.assertIn('666.67元', facts[2]['result']) self.assertNotIn('48元', facts[2]['result']) # Even a model that selects a superseded ID cannot render its price. return final_facts('fact_0', 'fact_1', 'fact_2') with patch('logistics_chat.ai_client.post_completion', side_effect=provider): text = self.ask('这票10KG一箱，箱规100乘100乘100厘米，核算普船美西价')[0]['t'] self.assertNotIn('48元', text) self.assertIn('666.67元', text) self.assertEqual(len(self.saved()[1]['options']), 1) def test_conflicting_quote_amendment_removes_prior_prices_and_candidate(self): initial = quote_arguments(services=['普船海派'], shipment={ 'country': 'US', 'total_weight_kg': '10', 'boxes': 1, 'zone': '美西', 'origin': '义乌', 'kinds': ['parcel']}) conflicting = quote_arguments(changes=[{'field': 'total_weight_kg', 'operation': 'add', 'value': '100'}]) responses = [tool_plan('quote_shipping', initial, 'quote_first'), tool_plan('quote_shipping', conflicting, 'conflict_second'), final_facts('fact_0', 'fact_1')] with patch('logistics_chat.ai_client.post_completion', side_effect=responses) as model: text = self.ask('核对这票重量和报价')[0]['t'] facts = evidence(model.call_args_list[2].args[0]) self.assertEqual(facts[0]['status'], 'superseded') self.assertEqual(facts[1]['status'], 'needs_clarification') self.assertIn('冲突', text) self.assertNotIn('48元', text) self.assertIsNone(self.saved()[1]) def test_deleted_conversation_does_not_save_quote_candidate_or_reply(self): count = 0 def provider(body, **kwargs): nonlocal count count += 1 if count == 1: return tool_plan('quote_shipping', quote_arguments(shipment={ 'country': 'US', 'total_weight_kg': '1000', 'zone': '美东', 'origin': '义乌', 'kinds': ['parcel']}), 'quote_before_delete') connection = db.connect() connection.execute('DELETE FROM messages WHERE conv_id=?', (self.cid,)) connection.execute('DELETE FROM chat_history WHERE id=?', (self.cid,)) connection.commit() connection.close() return final_facts('fact_0') with patch('logistics_chat.ai_client.post_completion', side_effect=provider): events = self.ask('请核算一吨货到美国美东') self.assertIn('e', events[0]) self.assertEqual(self.saved(), ([], None)) def test_late_model_reply_rolls_back_its_messages_and_candidate_state(self): count = 0 def provider(body, **kwargs): nonlocal count count += 1 if count == 1: return tool_plan('quote_shipping', quote_arguments(shipment={ 'country': 'US', 'total_weight_kg': '1000', 'zone': '美东', 'origin': '义乌', 'kinds': ['parcel']}), 'slow_quote') db.save_message(self.cid, 'user', '较新的问题') db.save_message(self.cid, 'assistant', '较新的回答') return final_facts('fact_0') with patch('logistics_chat.ai_client.post_completion', side_effect=provider): events = self.ask('这是一条迟到的报价请求') self.assertIn('e', events[0]) rows, state = self.saved() self.assertEqual(rows, self.initial_rows + [('user', '较新的问题'), ('assistant', '较新的回答')]) self.assertIsNone(state) if __name__ == '__main__': unittest.main()