DQN และ Double DQN

import random, collections
import numpy as np, torch, torch.nn as nn
import gymnasium as gym

env = gym.make("CartPole-v1")
n_obs, n_act = env.observation_space.shape[0], env.action_space.n
torch.manual_seed(0); random.seed(0)

def make_net():
    return nn.Sequential(nn.Linear(n_obs, 128), nn.ReLU(),
                         nn.Linear(128, 128), nn.ReLU(),
                         nn.Linear(128, n_act))       # state in -> Q ທຸກ action out

online, target = make_net(), make_net()
target.load_state_dict(online.state_dict())           # target network
opt = torch.optim.Adam(online.parameters(), lr=1e-3)
buffer = collections.deque(maxlen=50_000)             # replay buffer
gamma, batch_size, step = 0.99, 64, 0

for ep in range(400):
    s, _ = env.reset(seed=ep)
    done, ep_return = False, 0.0
    while not done:
        eps = max(0.05, 1.0 - step / 10_000)          # epsilon ຫຼຸດແບບເສັ້ນຊື່
        if random.random() < eps:
            a = env.action_space.sample()
        else:
            with torch.no_grad():
                a = int(online(torch.tensor(s)).argmax())
        s2, r, term, trunc, _ = env.step(a)
        buffer.append((s, a, r, s2, term))
        s, done, ep_return, step = s2, term or trunc, ep_return + r, step + 1

        if len(buffer) >= 1000:
            batch = random.sample(buffer, batch_size)   # ສຸ່ມ → ຂໍ້ມູນໃກ້ IID
            S, A, R, S2, T = (torch.tensor(np.array(x)) for x in zip(*batch))
            with torch.no_grad():
                a2 = online(S2).argmax(1, keepdim=True)          # Double DQN:
                q2 = target(S2).gather(1, a2).squeeze(1)         # online ເລືອກ, target ປະເມີນ
                y = R.float() + gamma * q2 * (1 - T.float())
            q = online(S).gather(1, A.long().unsqueeze(1)).squeeze(1)
            loss = nn.functional.huber_loss(q, y)                # Huber loss
            opt.zero_grad(); loss.backward(); opt.step()
        if step % 200 == 0:
            target.load_state_dict(online.state_dict())          # ອັບເດດ target ເປັນໄລຍະ
    if ep % 25 == 0:
        print(f"ເກມທີ {ep:3d}  ຄະແນນ {ep_return:5.0f}")