### 马尔科夫决策过程应用代码实现 #### 使用Python实现简单的马尔科夫决策过程求解器 下面是一个基于价值迭代方法解决简单环境下的马尔可夫决策过程(MDP)的Python示例： ```python import numpy as np class MDP: def __init__(self, transition_matrix, reward_function, discount_factor=0.9): """ 初始化MDP实例 参数: transition_matrix (dict): 转移概率矩阵 {state: {action: {next_state: probability}}} reward_function (dict): 奖励函数 {state: {action: reward}} discount_factor (float): 折扣因子 γ ∈ [0, 1) """ states = set(transition_matrix.keys()) actions = next(iter(transition_matrix.values())).keys() # 检查状态空间和动作空间的一致性 assert all(set(actions) == set(ta.keys()) for ta in transition_matrix.values()), "Actions must be consistent across states" self.states = list(states) self.actions = list(actions) self.transition_matrix = transition_matrix self.reward_function = reward_function self.discount_factor = discount_factor def value_iteration(self, epsilon=0.001): """执行价值迭代算法""" num_states = len(self.states) values = np.zeros(num_states) while True: delta = 0 for s_idx, state in enumerate(self.states): v = values[s_idx] max_value = float('-inf') for action in self.actions: total_reward = sum( prob * (self.reward_function[state][action] + self.discount_factor * values[self.states.index(next_s)]) for next_s, prob in self.transition_matrix[state][action].items() ) if total_reward > max_value: max_value = total_reward values[s_idx] = max_value delta = max(delta, abs(v - values[s_idx])) if delta < epsilon*(1-self.discount_factor)/self.discount_factor: break policy = {} for s_idx, state in enumerate(self.states): best_action = None highest_value = float('-inf') for action in self.actions: expected_return = sum( prob * (self.reward_function[state][action] + self.discount_factor * values[self.states.index(next_s)]) for next_s, prob in self.transition_matrix[state][action].items() ) if expected_return > highest_value: highest_value = expected_return best_action = action policy[state] = best_action return policy, values if __name__ == "__main__": # 定义转移概率矩阵 P(s'|s,a) transitions = { 'A': {'left': {'B': 1}, 'right': {'C': 1}}, 'B': {'up': {'D': 1}, 'down': {'E': 1}}, 'C': {'up': {'F': 1}, 'down': {'G': 1}}, 'D': {}, 'E': {}, 'F': {}, 'G': {} } # 定义奖励函数 R(s,a) rewards = { 'A': {'left': -1, 'right': 5}, 'B': {'up': 10, 'down': -1}, 'C': {'up': 20, 'down': -1}, 'D': {}, 'E': {}, 'F': {}, 'G': {} } mdp_problem = MDP(transitions, rewards) optimal_policy, _ = mdp_problem.value_iteration() print("Optimal Policy:") for k, v in optimal_policy.items(): print(f"From State '{k}' take Action -> '{v}'")[^1] ``` 此段代码展示了如何构建一个基本的价值迭代求解器来找到给定环境中最优策略的方法。这里定义了一个小型网格世界作为例子，其中每个位置代表不同的状态，并且有两个可能的动作（向左或向右）。对于更复杂的应用场景，则需要根据实际情况调整`transitions`和`rewards`参数。