Dueling DDQN และ PER

import torch, torch.nn as nn

class DuelingQNet(nn.Module):
    """Q(s,a) = V(s) + A(s,a) - mean_a A(s,a)"""
    def __init__(self, n_obs, n_act, hidden=128):
        super().__init__()
        self.body = nn.Sequential(nn.Linear(n_obs, hidden), nn.ReLU())
        self.value = nn.Linear(hidden, 1)          # ຫົວ V(s): ໃຊ້ຮ່ວມທຸກ action
        self.advantage = nn.Linear(hidden, n_act)  # ຫົວ A(s,a)

    def forward(self, s):
        h = self.body(s)
        v, adv = self.value(h), self.advantage(h)
        return v + adv - adv.mean(dim=-1, keepdim=True)

net = DuelingQNet(n_obs=4, n_act=2)
print(net(torch.zeros(3, 4)).shape)   # torch.Size([3, 2])

# PER: ຄວາມສຳຄັນ ∝ |TD error|^α, ແກ້ bias ດ້ວຍ importance-sampling weight
td_errors = torch.tensor([0.1, 2.0, 0.5, 0.05])
alpha, beta = 0.6, 0.4
p = (td_errors.abs() + 1e-6) ** alpha
probs = p / p.sum()
w = (len(probs) * probs) ** (-beta)
w = w / w.max()
print("ໂອກາດຖືກສຸ່ມ:", probs.numpy().round(3))
print("IS weight   :", w.numpy().round(3))