Beer Distribution Game Reinforcement Learning

Test notebook

Keywords

agent-based modeling, reinforcement-learning, beergame, beer distribution game

Beer Distribution Game Reinforcement Learning

from BPTK_Py.bptk import bptk 
bptk = bptk()
bptk.train_scenarios(
    episodes=10,
    scenario_managers=["smBeergameQlOB"],
    scenarios=["train_agents"],
    agents=["controlling"],
    agent_states=["active"],
    agent_properties=["supply_chain_reward"],
    agent_property_types=["total"],
    return_df=False,
    progress_bar=True
)

Persist Q-Tables

It takes quite some time to train q-tables and they can become quite large:

Q-Table Counts
Brewery: 3
Distributor: 4
Wholesaler: 5
Retailer: 10

from src.abm.q_learning_ob.beergame import BeergameQlOB

# Captured: marimo sends a cell's stdout to the console, not into the page.
with mo.capture_stdout() as q_table_counts:
    print("Q-Table Counts")
    print("Brewery: {}".format(BeergameQlOB.brewery_q_table.count()))
    print("Distributor: {}".format(BeergameQlOB.distributor_q_table.count()))
    print("Wholesaler: {}".format(BeergameQlOB.wholesaler_q_table.count()))
    print("Retailer: {}".format(BeergameQlOB.retailer_q_table.count()))

mo.plain_text(q_table_counts.getvalue())

Hence it makes sense to dump the trained q-tables so they can be reused:

This is the one cell on this page that is shown rather than run: it writes into data/, which a documentation build has no business doing - the q-tables there are tracked inputs of the other beergame pages. Nothing reads q_tables_10.json, so the call is here to show how it is done.

BeergameQlOB.dump_q_tables("data/q_tables_10.json", "JSON")

Load Q-Tables

Reset the q-tables:

from src.abm.q_learning_base.sparseQTable import SparseQTable
BeergameQlOB.brewery_q_table=SparseQTable(dimension=1)
BeergameQlOB.distributor_q_table=SparseQTable(dimension=1)
BeergameQlOB.wholesaler_q_table=SparseQTable(dimension=1)
BeergameQlOB.retailer_q_table=SparseQTable(dimension=1)

Load previously saved q-tables:

BeergameQlOB.load_q_tables("data/q_tables_50000.json","JSON")

Use Training Results

bptk.reset_scenario(scenario_manager="smBeergameQlOB",scenario="smart_agents")

bptk.plot_scenarios(
    scenario_managers=["smBeergameQlOB"],
    kind="area",
    scenarios=["smart_agents"],
    agents=["brewery","distributor","wholesaler","retailer"],
    agent_states=["active"],
    agent_properties=["order_balance"],
    agent_property_types=["total"], format="axes"
)

bptk.plot_scenarios(
    scenario_managers=["smBeergameQlOB"],
    kind="area",
    scenarios=["smart_agents"],
    agents=["brewery","distributor","wholesaler","retailer"],
    agent_states=["active"],
    agent_properties=["outgoing_order"],
    agent_property_types=["total"], format="axes"
)

bptk.plot_scenarios(
    scenario_managers=["smBeergameQlOB"],
    kind="area",
    scenarios=["smart_agents"],
    agents=["brewery","distributor","wholesaler","retailer"],
    agent_states=["active"],
    agent_properties=["total_cost"],
    agent_property_types=["total"], format="axes"
)

bptk.plot_scenarios(
    scenario_managers=["smBeergameQlOB"],
    kind="area",
    scenarios=["smart_agents"],
    agents=["controlling"],
    agent_states=["active"],
    agent_properties=["supply_chain_cost","target_supply_chain_cost"],
    agent_property_types=["total"], format="axes"
)

bptk.plot_scenarios(
    scenario_managers=["smBeergameQlOB"],
    kind="area",
    scenarios=["smart_agents"],
    agents=["brewery","distributor","wholesaler","retailer"],
    agent_states=["active"],
    agent_properties=["inventory"],
    agent_property_types=["total"], format="axes"
)

bptk.plot_scenarios(
    scenario_managers=["smBeergameQlOB"],
    kind="area",
    scenarios=["smart_agents"],
    agents=["brewery","distributor","wholesaler","retailer"],
    agent_states=["active"],
    agent_properties=["backorder"],
    agent_property_types=["total"], format="axes"
)