From 2024c1fb85f2555c2d956ec2ef9e4cccba229482 Mon Sep 17 00:00:00 2001 From: Jonas Eschmann Date: Mon, 18 Nov 2024 21:04:22 -0500 Subject: [PATCH 01/13] good --- examples/pendulum/ppo/train.py | 18 +++++++++--------- tests/test_pytorch_import.py | 11 +++++------ 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/examples/pendulum/ppo/train.py b/examples/pendulum/ppo/train.py index 3e0d980..21e5a17 100644 --- a/examples/pendulum/ppo/train.py +++ b/examples/pendulum/ppo/train.py @@ -25,19 +25,19 @@ def env_factory(**kwargs): selected_env, interface_name=env_name, enable_evaluation=True, - evaluation_interval=10, + evaluation_interval=100, # force_blas=True, num_evaluation_episodes=10, - OPTIMIZER_ALPHA=1e-3, + OPTIMIZER_ALPHA=1e-4, ACTION_ENTROPY_COEFFICIENT = 0.0, - N_EPOCHS = 2, + N_EPOCHS = 1, GAMMA = 0.9, - BATCH_SIZE = 256, - ACTOR_HIDDEN_DIM = 64, - CRITIC_HIDDEN_DIM = 64, - ON_POLICY_RUNNER_STEPS_PER_ENV = 1024, - N_ENVIRONMENTS = 4, - STEP_LIMIT = 74, # 1024 * 4 * 74 ~ 300k steps + BATCH_SIZE = 128, + ACTOR_HIDDEN_DIM = 32, + CRITIC_HIDDEN_DIM = 32, + ON_POLICY_RUNNER_STEPS_PER_ENV = 128, + N_ENVIRONMENTS = 8, + STEP_LIMIT = 2000, # 1024 * 4 * 74 ~ 300k steps ) state = ppo.State(seed) diff --git a/tests/test_pytorch_import.py b/tests/test_pytorch_import.py index f848656..b8ac6b3 100644 --- a/tests/test_pytorch_import.py +++ b/tests/test_pytorch_import.py @@ -1,18 +1,17 @@ import os +import pytest os.environ["RL_TOOLS_IGNORE_TORCH_WARNING"] = "1" -if not "RL_TOOLS_DISABLE_PYTORCH_IMPORT_TEST" in os.environ: +@pytest.mark.skip +def test_pytorch_import(): + if "RL_TOOLS_DISABLE_PYTORCH_IMPORT_TEST" in os.environ: + return import torch import torch.nn as nn from rltools import CACHE_PATH from rltools.onnx import load_mlp, evaluate from rltools.onnx import render from rltools import load_checkpoint_from_path - -@pytest.mark.skip -def test_pytorch_import(): - if "RL_TOOLS_DISABLE_PYTORCH_IMPORT_TEST" in os.environ: - return torch.manual_seed(0) n_input_features = 4 From a9392c5cfdf06b759dab53ec3c4ed5b879afeecf Mon Sep 17 00:00:00 2001 From: Jonas Eschmann Date: Mon, 18 Nov 2024 21:10:48 -0500 Subject: [PATCH 02/13] bump --- examples/custom_environment/my_pendulum/operations_generic.h | 2 +- rltools/external/rl_tools | 2 +- rltools/interface/algorithms/sac/template.h | 2 ++ rltools/interface/algorithms/td3/template.h | 1 + rltools/interface/python_environment/operations_cpu.h | 2 +- 5 files changed, 6 insertions(+), 3 deletions(-) diff --git a/examples/custom_environment/my_pendulum/operations_generic.h b/examples/custom_environment/my_pendulum/operations_generic.h index 273e6e8..1b446de 100644 --- a/examples/custom_environment/my_pendulum/operations_generic.h +++ b/examples/custom_environment/my_pendulum/operations_generic.h @@ -22,7 +22,7 @@ namespace rl_tools{ template static void free(DEVICE& device, const MyPendulum& env){ } template - static void init(DEVICE& device, const MyPendulum& env, typename MyPendulum::Parameters& parameters){ } + static void init(DEVICE& device, const MyPendulum& env){ } template static void initial_parameters(DEVICE& device, const MyPendulum& env, typename MyPendulum::Parameters& parameters){ } template diff --git a/rltools/external/rl_tools b/rltools/external/rl_tools index b1213d8..226bfbc 160000 --- a/rltools/external/rl_tools +++ b/rltools/external/rl_tools @@ -1 +1 @@ -Subproject commit b1213d8fd5af287cc0a4f2a7ff295247ef1b4eac +Subproject commit 226bfbc49b7d571f85271c85d35eb28dcc9327a7 diff --git a/rltools/interface/algorithms/sac/template.h b/rltools/interface/algorithms/sac/template.h index 16a9f1e..8e49b87 100644 --- a/rltools/interface/algorithms/sac/template.h +++ b/rltools/interface/algorithms/sac/template.h @@ -52,6 +52,8 @@ namespace RL_TOOLS_MODULE_NAME{ static constexpr bool COLLECT_EPISODE_STATS = $COLLECT_EPISODE_STATS; static constexpr TI EPISODE_STATS_BUFFER_SIZE = $EPISODE_STATS_BUFFER_SIZE; static constexpr bool SHARED_BATCH = $SHARED_BATCH; + static constexpr bool SAMPLE_ENVIRONMENT_PARAMETERS = true; + using INITIALIZER = rlt::nn::layers::dense::DefaultInitializer; diff --git a/rltools/interface/algorithms/td3/template.h b/rltools/interface/algorithms/td3/template.h index 717b682..5628c73 100644 --- a/rltools/interface/algorithms/td3/template.h +++ b/rltools/interface/algorithms/td3/template.h @@ -45,6 +45,7 @@ namespace RL_TOOLS_MODULE_NAME{ static constexpr TI EPISODE_STATS_BUFFER_SIZE = $EPISODE_STATS_BUFFER_SIZE; static constexpr T EXPLORATION_NOISE = $EXPLORATION_NOISE; static constexpr bool SHARED_BATCH = $SHARED_BATCH; + static constexpr bool SAMPLE_ENVIRONMENT_PARAMETERS = true; struct OPTIMIZER_PARAMETERS: rlt::nn::optimizers::adam::DEFAULT_PARAMETERS_TENSORFLOW{ static constexpr T ALPHA = $OPTIMIZER_ALPHA; diff --git a/rltools/interface/python_environment/operations_cpu.h b/rltools/interface/python_environment/operations_cpu.h index 0824aef..aab6215 100644 --- a/rltools/interface/python_environment/operations_cpu.h +++ b/rltools/interface/python_environment/operations_cpu.h @@ -67,7 +67,7 @@ namespace rl_tools{ // delete env.environment; } template - static void init(DEVICE& device, PythonEnvironment& env, const typename PythonEnvironment::Parameters& parameters){} + static void init(DEVICE& device, PythonEnvironment& env){} template static void initial_parameters(DEVICE& device, const PythonEnvironment& env, typename PythonEnvironment::Parameters& parameters){} template From 56b1a4f7a8a455c1e3873aea112497998dd373ee Mon Sep 17 00:00:00 2001 From: Jonas Eschmann Date: Mon, 18 Nov 2024 21:30:14 -0500 Subject: [PATCH 03/13] bump --- build.sh | 3 ++- pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/build.sh b/build.sh index 454fda2..df67e56 100755 --- a/build.sh +++ b/build.sh @@ -1,4 +1,5 @@ +# auth in ~/.pypirc rm -rf ../python-interface/dist pip install --upgrade build twine python3 -m build -python3 -m twine upload dist/* --password $PYPI_TOKEN \ No newline at end of file +python3 -m twine upload dist/* diff --git a/pyproject.toml b/pyproject.toml index baa2def..f3978fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "rltools" -version = "1.2.1" +version = "1.2.2" authors = [ { name="Jonas Eschmann", email="jonas.eschmann@gmail.com" }, ] From 4ad28d700232a11f592a287a32f3ad1686d7a4f6 Mon Sep 17 00:00:00 2001 From: Jonas Eschmann Date: Mon, 18 Nov 2024 22:07:48 -0500 Subject: [PATCH 04/13] mkl num threads 1 by default --- rltools/src/link_mkl.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rltools/src/link_mkl.py b/rltools/src/link_mkl.py index 6eb16ca..8230dbb 100644 --- a/rltools/src/link_mkl.py +++ b/rltools/src/link_mkl.py @@ -35,6 +35,8 @@ def link_mkl(): if not found: print(f"MKL library {lib} not found.") required_libraries_paths = None + if "MKL_NUM_THREADS" not in os.environ: + os.environ["MKL_NUM_THREADS"] = "1" required_libraries_paths_absolute = [str(p.locate()) for p in required_libraries_paths] required_libraries_search_paths = [str(os.path.dirname(p)) for p in required_libraries_paths_absolute] flags += [ From 5727367a2992c6b983beb213c66e9522c1333a64 Mon Sep 17 00:00:00 2001 From: Jonas Eschmann Date: Mon, 18 Nov 2024 22:08:23 -0500 Subject: [PATCH 05/13] bump --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f3978fa..979b5ed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "rltools" -version = "1.2.2" +version = "1.2.3" authors = [ { name="Jonas Eschmann", email="jonas.eschmann@gmail.com" }, ] From fff03dd3c67d7a8ce7dfc1726f1d501c83334c44 Mon Sep 17 00:00:00 2001 From: Jonas Eschmann Date: Tue, 19 Nov 2024 11:30:25 -0500 Subject: [PATCH 06/13] bump 2.0.0 --- pyproject.toml | 2 +- rltools/external/rl_tools | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 979b5ed..bd3feba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "rltools" -version = "1.2.3" +version = "2.0.0" authors = [ { name="Jonas Eschmann", email="jonas.eschmann@gmail.com" }, ] diff --git a/rltools/external/rl_tools b/rltools/external/rl_tools index 226bfbc..f40f052 160000 --- a/rltools/external/rl_tools +++ b/rltools/external/rl_tools @@ -1 +1 @@ -Subproject commit 226bfbc49b7d571f85271c85d35eb28dcc9327a7 +Subproject commit f40f05276e22819114d0c392cf030729784d836c From ad7569a8a36ec741c48d36a48dee3254f1ac65b4 Mon Sep 17 00:00:00 2001 From: Jonas Eschmann Date: Thu, 11 Sep 2025 19:42:03 -0700 Subject: [PATCH 07/13] bump pre 2.1 --- rltools/external/rl_tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rltools/external/rl_tools b/rltools/external/rl_tools index f40f052..f49774c 160000 --- a/rltools/external/rl_tools +++ b/rltools/external/rl_tools @@ -1 +1 @@ -Subproject commit f40f05276e22819114d0c392cf030729784d836c +Subproject commit f49774c32af7b6687bc9e441cddc373be8234ad9 From 4791c7dda18d131b3b0226ca6a77cce5f6cacf1e Mon Sep 17 00:00:00 2001 From: Jonas Eschmann Date: Thu, 11 Sep 2025 20:06:12 -0700 Subject: [PATCH 08/13] bumpt rlt to 2.1, seems good --- rltools/external/rl_tools | 2 +- rltools/interface/algorithms/sac/template.h | 1 + rltools/interface/algorithms/td3/template.h | 4 ++++ rltools/interface/inference/inference.cpp | 5 ++++- rltools/interface/training/training.cpp | 4 +++- 5 files changed, 13 insertions(+), 3 deletions(-) diff --git a/rltools/external/rl_tools b/rltools/external/rl_tools index f49774c..d025a0d 160000 --- a/rltools/external/rl_tools +++ b/rltools/external/rl_tools @@ -1 +1 @@ -Subproject commit f49774c32af7b6687bc9e441cddc373be8234ad9 +Subproject commit d025a0df3a4b144a40bad57890877637e50d79f2 diff --git a/rltools/interface/algorithms/sac/template.h b/rltools/interface/algorithms/sac/template.h index 8e49b87..a9e20f2 100644 --- a/rltools/interface/algorithms/sac/template.h +++ b/rltools/interface/algorithms/sac/template.h @@ -57,6 +57,7 @@ namespace RL_TOOLS_MODULE_NAME{ using INITIALIZER = rlt::nn::layers::dense::DefaultInitializer; + using BATCH_SAMPLING_PARAMETERS = rlt::rl::components::off_policy_runner::SequentialBatchParameters; struct OPTIMIZER_PARAMETERS: rlt::nn::optimizers::adam::DEFAULT_PARAMETERS_TENSORFLOW{ static constexpr T ALPHA = $OPTIMIZER_ALPHA; diff --git a/rltools/interface/algorithms/td3/template.h b/rltools/interface/algorithms/td3/template.h index 5628c73..a3e39ce 100644 --- a/rltools/interface/algorithms/td3/template.h +++ b/rltools/interface/algorithms/td3/template.h @@ -47,6 +47,10 @@ namespace RL_TOOLS_MODULE_NAME{ static constexpr bool SHARED_BATCH = $SHARED_BATCH; static constexpr bool SAMPLE_ENVIRONMENT_PARAMETERS = true; + using INITIALIZER = rlt::nn::layers::dense::DefaultInitializer; + + using BATCH_SAMPLING_PARAMETERS = rlt::rl::components::off_policy_runner::SequentialBatchParameters; + struct OPTIMIZER_PARAMETERS: rlt::nn::optimizers::adam::DEFAULT_PARAMETERS_TENSORFLOW{ static constexpr T ALPHA = $OPTIMIZER_ALPHA; static constexpr T BETA_1 = $OPTIMIZER_BETA_1; diff --git a/rltools/interface/inference/inference.cpp b/rltools/interface/inference/inference.cpp index 8a16447..0a6373c 100644 --- a/rltools/interface/inference/inference.cpp +++ b/rltools/interface/inference/inference.cpp @@ -19,17 +19,20 @@ namespace rlt = rl_tools; using DEVICE = rlt::devices::DEVICE_FACTORY<>; +using RNG = DEVICE::SPEC::RANDOM::ENGINE<>; DEVICE device; -auto rng = rlt::random::default_engine(typename DEVICE::SPEC::RANDOM{}); +RNG rng; using MODEL_TYPE = decltype(policy::module); using T = typename MODEL_TYPE::T; using TI = typename DEVICE::index_t; typename MODEL_TYPE::template Buffer<1> buffer; bool initialized = false; +constexpr TI seed = 0x1337; void init(){ if(!initialized){ + rlt::init(device, rng, seed); rlt::malloc(device, buffer); } } diff --git a/rltools/interface/training/training.cpp b/rltools/interface/training/training.cpp index 54b7951..d083ead 100644 --- a/rltools/interface/training/training.cpp +++ b/rltools/interface/training/training.cpp @@ -48,7 +48,7 @@ namespace RL_TOOLS_MODULE_NAME{ #ifdef RL_TOOLS_FORCE_BLAS static_assert(DEVICE::DEVICE_ID == rlt::devices::DeviceId::CPU_MKL || DEVICE::DEVICE_ID == rlt::devices::DeviceId::CPU_ACCELERATE || DEVICE::DEVICE_ID == rlt::devices::DeviceId::CPU_OPENBLAS); #endif - using RNG = decltype(rlt::random::default_engine(typename DEVICE::SPEC::RANDOM{})); + using RNG = DEVICE::SPEC::RANDOM::ENGINE<>; using TI = typename DEVICE::index_t; using T = RL_TOOLS_DTYPE; @@ -76,6 +76,7 @@ namespace RL_TOOLS_MODULE_NAME{ DEVICE device; + RNG rng; #ifdef RL_TOOLS_ENABLE_EVALUATION constexpr bool ENABLE_EVALUATION = true; @@ -124,6 +125,7 @@ namespace RL_TOOLS_MODULE_NAME{ EVALUATION_ACTOR_BUFFER evaluation_actor_buffer; bool evaluation_actor_synced = false; State(TI seed){ + rlt::init(device, rng, seed); rlt::malloc(device, static_cast(*this)); rlt::init(device, static_cast(*this), seed); rlt::malloc(device, evaluation_actor); From 60b5f2b243e60ba81c24e3cd598e911d37c005b4 Mon Sep 17 00:00:00 2001 From: Jonas Eschmann Date: Thu, 11 Sep 2025 20:39:57 -0700 Subject: [PATCH 09/13] bump rlt to 2.1 --- pyproject.toml | 2 +- rltools/external/rl_tools | 2 +- rltools/interface/inference/inference.cpp | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index bd3feba..827df69 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "rltools" -version = "2.0.0" +version = "2.1.0" authors = [ { name="Jonas Eschmann", email="jonas.eschmann@gmail.com" }, ] diff --git a/rltools/external/rl_tools b/rltools/external/rl_tools index d025a0d..94a814f 160000 --- a/rltools/external/rl_tools +++ b/rltools/external/rl_tools @@ -1 +1 @@ -Subproject commit d025a0df3a4b144a40bad57890877637e50d79f2 +Subproject commit 94a814f785fd5f537f06c349b0cebdb5d80f1d45 diff --git a/rltools/interface/inference/inference.cpp b/rltools/interface/inference/inference.cpp index 0a6373c..46fcb09 100644 --- a/rltools/interface/inference/inference.cpp +++ b/rltools/interface/inference/inference.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include #include From 164d550efbe4f6caaba3209f3a75f32467b2a14a Mon Sep 17 00:00:00 2001 From: Jonas Eschmann Date: Wed, 22 Oct 2025 20:15:01 -0700 Subject: [PATCH 10/13] bump to v2.2 --- rltools/external/rl_tools | 2 +- rltools/interface/algorithms/ppo/template.h | 13 +++++++------ rltools/interface/algorithms/sac/template.h | 13 +++++++------ rltools/interface/algorithms/td3/template.h | 15 ++++++++------- rltools/interface/inference/inference.cpp | 1 + rltools/interface/training/training.cpp | 8 ++++++-- 6 files changed, 30 insertions(+), 22 deletions(-) diff --git a/rltools/external/rl_tools b/rltools/external/rl_tools index 94a814f..9c72c78 160000 --- a/rltools/external/rl_tools +++ b/rltools/external/rl_tools @@ -1 +1 @@ -Subproject commit 94a814f785fd5f537f06c349b0cebdb5d80f1d45 +Subproject commit 9c72c7865af6eca3d0463c76f6ee01fa1e807d73 diff --git a/rltools/interface/algorithms/ppo/template.h b/rltools/interface/algorithms/ppo/template.h index c089833..3bf8206 100644 --- a/rltools/interface/algorithms/ppo/template.h +++ b/rltools/interface/algorithms/ppo/template.h @@ -7,8 +7,9 @@ namespace rlt = rl_tools; -template -struct PPO_LOOP_CORE_PARAMETERS: rlt::rl::algorithms::ppo::loop::core::DefaultParameters{ +template +struct PPO_LOOP_CORE_PARAMETERS: rlt::rl::algorithms::ppo::loop::core::DefaultParameters{ + using T = typename TYPE_POLICY::DEFAULT; static constexpr TI STEP_LIMIT = $STEP_LIMIT; static constexpr TI ACTOR_HIDDEN_DIM = $ACTOR_HIDDEN_DIM; static constexpr TI ACTOR_NUM_LAYERS = $ACTOR_NUM_LAYERS; @@ -22,7 +23,7 @@ struct PPO_LOOP_CORE_PARAMETERS: rlt::rl::algorithms::ppo::loop::core::DefaultPa static constexpr TI DATASET_SIZE = ON_POLICY_RUNNER_STEPS_PER_ENV * N_ENVIRONMENTS; static constexpr TI BATCH_SIZE = $BATCH_SIZE; - struct PPO_PARAMETERS: rlt::rl::algorithms::ppo::DefaultParameters{ + struct PPO_PARAMETERS: rlt::rl::algorithms::ppo::DefaultParameters{ static constexpr T GAMMA = $GAMMA; static constexpr T LAMBDA = $LAMBDA; static constexpr T EPSILON_CLIP = $EPSILON_CLIP; @@ -46,7 +47,7 @@ struct PPO_LOOP_CORE_PARAMETERS: rlt::rl::algorithms::ppo::loop::core::DefaultPa }; - struct OPTIMIZER_PARAMETERS: rlt::nn::optimizers::adam::DEFAULT_PARAMETERS_TENSORFLOW{ + struct OPTIMIZER_PARAMETERS: rlt::nn::optimizers::adam::DEFAULT_PARAMETERS_TENSORFLOW{ static constexpr T ALPHA = $OPTIMIZER_ALPHA; static constexpr T BETA_1 = $OPTIMIZER_BETA_1; static constexpr T BETA_2 = $OPTIMIZER_BETA_2; @@ -56,5 +57,5 @@ struct PPO_LOOP_CORE_PARAMETERS: rlt::rl::algorithms::ppo::loop::core::DefaultPa }; -template -using LOOP_CORE_CONFIG_FACTORY = rlt::rl::algorithms::ppo::loop::core::Config>; \ No newline at end of file +template +using LOOP_CORE_CONFIG_FACTORY = rlt::rl::algorithms::ppo::loop::core::Config>; \ No newline at end of file diff --git a/rltools/interface/algorithms/sac/template.h b/rltools/interface/algorithms/sac/template.h index a9e20f2..bc6ac5f 100644 --- a/rltools/interface/algorithms/sac/template.h +++ b/rltools/interface/algorithms/sac/template.h @@ -10,9 +10,10 @@ namespace RL_TOOLS_MODULE_NAME{ namespace rlt = rl_tools; // This should stay in sync with the parameters in https://github.com/rl-tools/rl-tools/blob/a98bc461f7ebfed0ba71c653216edec6d9334b78/include/rl_tools/rl/algorithms/sac/loop/core/config.h#L18 - template + template // struct SAC_LOOP_CORE_PARAMETERS: rlt::rl::algorithms::sac::loop::core::DefaultParameters{ struct SAC_LOOP_CORE_PARAMETERS{ + using T = typename TYPE_POLICY::DEFAULT; struct SAC_PARAMETERS{ static constexpr T GAMMA = $GAMMA; static constexpr TI ACTOR_BATCH_SIZE = $ACTOR_BATCH_SIZE; @@ -55,11 +56,11 @@ namespace RL_TOOLS_MODULE_NAME{ static constexpr bool SAMPLE_ENVIRONMENT_PARAMETERS = true; - using INITIALIZER = rlt::nn::layers::dense::DefaultInitializer; + using INITIALIZER = rlt::nn::layers::dense::DefaultInitializer; - using BATCH_SAMPLING_PARAMETERS = rlt::rl::components::off_policy_runner::SequentialBatchParameters; + using BATCH_SAMPLING_PARAMETERS = rlt::rl::components::off_policy_runner::SequentialBatchParameters; - struct OPTIMIZER_PARAMETERS: rlt::nn::optimizers::adam::DEFAULT_PARAMETERS_TENSORFLOW{ + struct OPTIMIZER_PARAMETERS: rlt::nn::optimizers::adam::DEFAULT_PARAMETERS_TENSORFLOW{ static constexpr T ALPHA = $OPTIMIZER_ALPHA; static constexpr T BETA_1 = $OPTIMIZER_BETA_1; static constexpr T BETA_2 = $OPTIMIZER_BETA_2; @@ -73,6 +74,6 @@ namespace RL_TOOLS_MODULE_NAME{ }; - template - using LOOP_CORE_CONFIG_FACTORY = rlt::rl::algorithms::sac::loop::core::Config>; + template + using LOOP_CORE_CONFIG_FACTORY = rlt::rl::algorithms::sac::loop::core::Config>; } \ No newline at end of file diff --git a/rltools/interface/algorithms/td3/template.h b/rltools/interface/algorithms/td3/template.h index a3e39ce..f6b235f 100644 --- a/rltools/interface/algorithms/td3/template.h +++ b/rltools/interface/algorithms/td3/template.h @@ -10,8 +10,9 @@ namespace RL_TOOLS_MODULE_NAME{ namespace rlt = rl_tools; // This should stay in sync with the parameters in https://github.com/rl-tools/rl-tools/blob/a98bc461f7ebfed0ba71c653216edec6d9334b78/include/rl_tools/rl/algorithms/td3/loop/core/config.h#L18 - template - struct TD3_LOOP_CORE_PARAMETERS: rlt::rl::algorithms::td3::loop::core::DefaultParameters{ + template + struct TD3_LOOP_CORE_PARAMETERS: rlt::rl::algorithms::td3::loop::core::DefaultParameters{ + using T = typename TYPE_POLICY::DEFAULT; struct TD3_PARAMETERS{ static constexpr T GAMMA = $GAMMA; static constexpr TI ACTOR_BATCH_SIZE = $ACTOR_BATCH_SIZE; @@ -47,11 +48,11 @@ namespace RL_TOOLS_MODULE_NAME{ static constexpr bool SHARED_BATCH = $SHARED_BATCH; static constexpr bool SAMPLE_ENVIRONMENT_PARAMETERS = true; - using INITIALIZER = rlt::nn::layers::dense::DefaultInitializer; + using INITIALIZER = rlt::nn::layers::dense::DefaultInitializer; - using BATCH_SAMPLING_PARAMETERS = rlt::rl::components::off_policy_runner::SequentialBatchParameters; + using BATCH_SAMPLING_PARAMETERS = rlt::rl::components::off_policy_runner::SequentialBatchParameters; - struct OPTIMIZER_PARAMETERS: rlt::nn::optimizers::adam::DEFAULT_PARAMETERS_TENSORFLOW{ + struct OPTIMIZER_PARAMETERS: rlt::nn::optimizers::adam::DEFAULT_PARAMETERS_TENSORFLOW{ static constexpr T ALPHA = $OPTIMIZER_ALPHA; static constexpr T BETA_1 = $OPTIMIZER_BETA_1; static constexpr T BETA_2 = $OPTIMIZER_BETA_2; @@ -60,6 +61,6 @@ namespace RL_TOOLS_MODULE_NAME{ }; }; - template - using LOOP_CORE_CONFIG_FACTORY = rlt::rl::algorithms::td3::loop::core::Config>; + template + using LOOP_CORE_CONFIG_FACTORY = rlt::rl::algorithms::td3::loop::core::Config>; } \ No newline at end of file diff --git a/rltools/interface/inference/inference.cpp b/rltools/interface/inference/inference.cpp index 46fcb09..53caff9 100644 --- a/rltools/interface/inference/inference.cpp +++ b/rltools/interface/inference/inference.cpp @@ -26,6 +26,7 @@ DEVICE device; RNG rng; using MODEL_TYPE = decltype(policy::module); using T = typename MODEL_TYPE::T; +using TYPE_POLICY = rlt::numeric_types::Policy; using TI = typename DEVICE::index_t; typename MODEL_TYPE::template Buffer<1> buffer; bool initialized = false; diff --git a/rltools/interface/training/training.cpp b/rltools/interface/training/training.cpp index d083ead..ee409e6 100644 --- a/rltools/interface/training/training.cpp +++ b/rltools/interface/training/training.cpp @@ -24,6 +24,9 @@ #include #include +#include +#include +#include #include #include #include @@ -52,6 +55,7 @@ namespace RL_TOOLS_MODULE_NAME{ using TI = typename DEVICE::index_t; using T = RL_TOOLS_DTYPE; + using TYPE_POLICY = rlt::numeric_types::Policy; #ifdef RL_TOOLS_USE_PYTHON_ENVIRONMENT @@ -71,7 +75,7 @@ namespace RL_TOOLS_MODULE_NAME{ - using LOOP_CORE_CONFIG = LOOP_CORE_CONFIG_FACTORY; + using LOOP_CORE_CONFIG = LOOP_CORE_CONFIG_FACTORY; @@ -87,7 +91,7 @@ namespace RL_TOOLS_MODULE_NAME{ constexpr TI PARAMETER_NUM_EVALUATION_EPISODES = RL_TOOLS_NUM_EVALUATION_EPISODES; template - struct LOOP_EVAL_PARAMETERS: rlt::rl::loop::steps::evaluation::Parameters{ + struct LOOP_EVAL_PARAMETERS: rlt::rl::loop::steps::evaluation::Parameters{ static constexpr TI EVALUATION_INTERVAL = PARAMETER_EVALUATION_INTERVAL; static constexpr TI NUM_EVALUATION_EPISODES = PARAMETER_NUM_EVALUATION_EPISODES; static constexpr TI N_EVALUATIONS = NEXT::CORE_PARAMETERS::STEP_LIMIT / EVALUATION_INTERVAL; From 6c6bf9f211bdc4699193b52ed33a1e2b6992838d Mon Sep 17 00:00:00 2001 From: Jonas Eschmann Date: Wed, 22 Oct 2025 21:49:10 -0700 Subject: [PATCH 11/13] bump v2.2 --- pyproject.toml | 2 +- rltools/external/rl_tools | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 827df69..958cbf0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "rltools" -version = "2.1.0" +version = "2.2.0" authors = [ { name="Jonas Eschmann", email="jonas.eschmann@gmail.com" }, ] diff --git a/rltools/external/rl_tools b/rltools/external/rl_tools index 9c72c78..56fd71a 160000 --- a/rltools/external/rl_tools +++ b/rltools/external/rl_tools @@ -1 +1 @@ -Subproject commit 9c72c7865af6eca3d0463c76f6ee01fa1e807d73 +Subproject commit 56fd71aeeedc56cb24f9db6cf40e5b9e49dd3bdd From d67d5eb432f3279e7709bac4777f70fdb7830265 Mon Sep 17 00:00:00 2001 From: Jonas Eschmann Date: Wed, 22 Oct 2025 21:58:04 -0700 Subject: [PATCH 12/13] fixing inference --- rltools/interface/inference/inference.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rltools/interface/inference/inference.cpp b/rltools/interface/inference/inference.cpp index 53caff9..33b93ad 100644 --- a/rltools/interface/inference/inference.cpp +++ b/rltools/interface/inference/inference.cpp @@ -25,7 +25,7 @@ using RNG = DEVICE::SPEC::RANDOM::ENGINE<>; DEVICE device; RNG rng; using MODEL_TYPE = decltype(policy::module); -using T = typename MODEL_TYPE::T; +using T = typename MODEL_TYPE::TYPE_POLICY::DEFAULT; using TYPE_POLICY = rlt::numeric_types::Policy; using TI = typename DEVICE::index_t; typename MODEL_TYPE::template Buffer<1> buffer; From d797a83bd62b4f77153b61365907b008a893676a Mon Sep 17 00:00:00 2001 From: Jonas Eschmann Date: Wed, 22 Oct 2025 21:58:25 -0700 Subject: [PATCH 13/13] bump --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 958cbf0..e1c2099 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "rltools" -version = "2.2.0" +version = "2.2.1" authors = [ { name="Jonas Eschmann", email="jonas.eschmann@gmail.com" }, ]