\documentclass[11pt]{article}

\usepackage[margin=1in]{geometry}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{lmodern}
\usepackage{microtype}
\usepackage{amsmath, amssymb, amsthm, mathtools}
\usepackage[dvipsnames]{xcolor}
\usepackage{graphicx}
\usepackage{booktabs}
\usepackage{array}
\usepackage{enumitem}
\usepackage{tikz}
\usetikzlibrary{arrows.meta, positioning, fit, shapes.geometric}
\usepackage[most]{tcolorbox}
\usepackage{listings}
\usepackage{caption}
\usepackage{hyperref}
\usepackage[numbers,sort&compress]{natbib}

\hypersetup{colorlinks=true, linkcolor=MidnightBlue, citecolor=MidnightBlue, urlcolor=MidnightBlue}
\graphicspath{{figures/}}

% ---------------------------------------------------------------------
% Tutorial colour macros inherited from interactive_imitation_llms.tex.
% ---------------------------------------------------------------------
\newcommand{\colsym}[2]{\textcolor{#1}{\textbf{#2}}}

% Causal-interaction symbols from the main paper.
\newcommand{\ActionSet}{\ensuremath{\textcolor{red}{\mathcal{A}}}}
\newcommand{\ObsSet}{\ensuremath{\textcolor{ForestGreen}{\mathcal{O}}}}
\newcommand{\ProgramSet}{\ensuremath{\textcolor{blue}{\mathcal{P}}}}
\newcommand{\History}{\ensuremath{\textcolor{RoyalBlue}{\mathcal{H}}}}
\newcommand{\action}{\ensuremath{\textcolor{red}{a}}}
\newcommand{\obs}{\ensuremath{\textcolor{ForestGreen}{o}}}
\newcommand{\program}{\ensuremath{\textcolor{blue}{p}}}
\newcommand{\contextsym}{\ensuremath{\textcolor{purple}{c}}}
\newcommand{\gate}{\ensuremath{\textcolor{orange}{\gamma}}}
\newcommand{\history}{\ensuremath{\textcolor{RoyalBlue}{h}}}
\newcommand{\trajectory}{\ensuremath{\textcolor{teal}{\tau}}}
\newcommand{\thetap}{\ensuremath{\textcolor{blue}{\theta}}}
\newcommand{\policy}{\ensuremath{\textcolor{MidnightBlue}{\pi}}}
\newcommand{\prob}{\ensuremath{\textcolor{MidnightBlue}{\mathbb{P}}}}
\newcommand{\E}{\ensuremath{\textcolor{MidnightBlue}{\mathbb{E}}}}
\newcommand{\KL}{\ensuremath{\textcolor{Mahogany}{D_{\mathrm{KL}}}}}
\newcommand{\doop}{\operatorname{\textcolor{orange}{do}}}
\newcommand{\intervene}[1]{\ensuremath{\doop\!\left(#1\right)}}
\newcommand{\discount}{\ensuremath{\textcolor{Orange}{\gamma_{\mathrm{RL}}}}}
\newcommand{\sftloss}{\ensuremath{\textcolor{Mahogany}{\mathcal{L}}}}
\newcommand{\obssft}{\ensuremath{\sftloss^{\mathrm{obs}}}}
\newcommand{\dosft}{\ensuremath{\sftloss^{\mathrm{do}}}}

% ---------------------------------------------------------------------
% New vNM/preference symbols.  The main convention is:
% observations are o in green; terminal outcomes/consequences are x in mahogany.
% ---------------------------------------------------------------------
\newcommand{\OutcomeSet}{\ensuremath{\textcolor{Mahogany}{\mathcal{X}}}}
\newcommand{\outcome}{\ensuremath{\textcolor{Mahogany}{x}}}
\newcommand{\outcomei}[1]{\ensuremath{\textcolor{Mahogany}{x_{#1}}}}
\newcommand{\outcomebest}{\ensuremath{\textcolor{Mahogany}{x^{+}}}}
\newcommand{\outcomeworst}{\ensuremath{\textcolor{Mahogany}{x^{-}}}}
\newcommand{\LotterySet}{\ensuremath{\textcolor{RoyalBlue}{\Delta(\OutcomeSet)}}}
\newcommand{\lottery}{\ensuremath{\textcolor{RoyalBlue}{L}}}
\newcommand{\lotteryR}{\ensuremath{\textcolor{RoyalBlue}{R}}}
\newcommand{\lotteryM}{\ensuremath{\textcolor{RoyalBlue}{M}}}
\newcommand{\lotteryN}{\ensuremath{\textcolor{RoyalBlue}{N}}}
\newcommand{\mixprob}{\ensuremath{\textcolor{Orange}{\alpha}}}
\newcommand{\pref}{\ensuremath{\textcolor{MidnightBlue}{\succeq}}}
\newcommand{\strictpref}{\ensuremath{\textcolor{MidnightBlue}{\succ}}}
\newcommand{\indiff}{\ensuremath{\textcolor{MidnightBlue}{\sim}}}
\newcommand{\utility}{\ensuremath{\textcolor{Plum}{u}}}
\newcommand{\ustar}{\ensuremath{\textcolor{Plum}{u^{\star}}}}
\newcommand{\uhat}{\ensuremath{\textcolor{Plum}{\widehat{u}}}}
\newcommand{\selfutil}{\ensuremath{\textcolor{Plum}{u^{\mathrm{self}}}}}
\newcommand{\wvec}{\ensuremath{\textcolor{cyan}{w}}}
\newcommand{\wdo}{\ensuremath{\textcolor{cyan}{\widehat{w}^{\mathrm{do}}}}}
\newcommand{\wobs}{\ensuremath{\textcolor{cyan}{\widehat{w}^{\mathrm{obs}}}}}
\newcommand{\reward}{\ensuremath{\textcolor{BrickRed}{r}}}
\newcommand{\rstar}{\ensuremath{\textcolor{BrickRed}{r^{\star}}}}
\newcommand{\Dataset}{\ensuremath{\textcolor{teal}{\mathcal{D}}}}
\newcommand{\Ddo}{\ensuremath{\textcolor{teal}{\mathcal{D}^{\mathrm{do}}}}}
\newcommand{\Dobs}{\ensuremath{\textcolor{teal}{\mathcal{D}^{\mathrm{obs}}}}}
\newcommand{\Dworld}{\ensuremath{\textcolor{teal}{\mathcal{D}^{\mathrm{world}}}}}
\newcommand{\Dself}{\ensuremath{\textcolor{teal}{\mathcal{D}^{\mathrm{self}}}}}
\newcommand{\BT}{\ensuremath{\textcolor{MidnightBlue}{\mathrm{BT}}}}
\newcommand{\Regret}{\ensuremath{\textcolor{BrickRed}{\mathrm{Regret}}}}
\newcommand{\Choice}{\ensuremath{\textcolor{RoyalBlue}{y}}}
\newcommand{\sigmoid}{\ensuremath{\textcolor{MidnightBlue}{\sigma}}}
\newcommand{\Value}{\ensuremath{\textcolor{Plum}{V}}}
\newcommand{\Qval}{\ensuremath{\textcolor{Plum}{Q}}}
\newcommand{\TraceSet}{\ensuremath{\textcolor{teal}{\mathcal{T}}}}
\newcommand{\Trace}{\ensuremath{\textcolor{teal}{T}}}

\newtcolorbox{keybox}[1]{
  colback=blue!3,
  colframe=MidnightBlue!60!black,
  title=\textbf{#1},
  fonttitle=\small,
  breakable,
  sharp corners,
  boxrule=0.6pt
}

\newtcolorbox{warningbox}[1]{
  colback=red!3,
  colframe=red!60!black,
  title=\textbf{#1},
  fonttitle=\small,
  breakable,
  sharp corners,
  boxrule=0.6pt
}

\newtcolorbox{examplebox}[1]{
  colback=ForestGreen!4,
  colframe=ForestGreen!55!black,
  title=\textbf{#1},
  fonttitle=\small,
  breakable,
  sharp corners,
  boxrule=0.6pt
}

\lstdefinestyle{tutorialcode}{
  basicstyle=\ttfamily\footnotesize,
  breaklines=true,
  columns=fullflexible,
  keepspaces=true,
  showstringspaces=false,
  frame=single,
  framerule=0.2pt,
  rulecolor=\color{black!30},
  backgroundcolor=\color{black!2}
}
\lstset{style=tutorialcode}

\title{\textbf{Emergent reward maximization}}

\author{Nando de Freitas and Pedro Ortega, assisted by GPT Codex}
\date{\today}


\begin{document}
\maketitle



\begin{abstract}
Can an interactional imitation learner, trained without scalar reward labels, recover behavior that is equivalent to expected reward maximization purely from world-written preference evidence? The answer as shown here is yes, provided the learner treats its own actions as interventions, and not as observations.  
\end{abstract}










\section{Introduction}



 Reinforcement learning (RL) usually begins with a given or \emph{engineered} scalar reward function and then studies how to maximize expected return.  Interactional agency, as discussed in our previous research notes, reverses that order: the learner is placed inside an interaction stream, where some symbols are written by the world and some symbols are written by the learner itself.  The world-written stream may contain demonstrations, corrections, comparisons, user choices, judge decisions, and other agents' choices.  These are evidence about what matters.  The learner's own choices are interventions, not evidence about what the world values.


Our hypothesis is as follows. Suppose an interaction history contains both world-written teacher actions and learner-written actions.  Suppose the teacher actions are generated by a von Neumann--Morgenstern-consistent (vNM) teacher with hidden utility \(\ustar\), while the learner's early actions are generated by a biased policy.  If learner-written action slots are correctly treated as interventions, then an interventional imitation learner should recover a utility representation aligned with \(\ustar\), and maximizing that recovered utility should maximize hidden reward in new action environments.  If learner-written action slots are incorrectly treated as ordinary evidence, then the learner can infer a self-confirming but wrong purpose.


This research note will show that interactive imitation can be enough for reward-maximizing behavior in a controlled vNM environment, provided that the interaction stream contains sufficient world-written preference evidence and the learner does not treat its own interventions as evidence.


\section{Preference evidence and vNM utility}
\label{sec:vnm-preference-experiment}

The von Neumann--Morgenstern and Savage traditions define rational agency through preferences, probabilities, and expected utility. In the vNM setting, preferences over lotteries can be represented by a utility function under suitable axioms. Savage extended this into subjective expected utility: preferences over acts reveal both subjective probabilities and utilities, and rational choice is represented as maximizing expected utility.

This is the intellectual background behind reinforcement learning, and it remains the dominant formal account of agency in AI. For this reason, this research note provides a tutorial introduction to the important vNM theorem. Anyone working in post-training LLMs, AI safety, reasoning and tool-use should be familiar with this material because it is the foundation behind RLHF.


Let
\[
  \OutcomeSet=\{\outcomei{0},\outcomei{1},\ldots,\outcomei{m-1}\}
\]
be a finite set of terminal outcomes.  A lottery is a probability distribution over \(\OutcomeSet\).  Since the outcome set is finite, we identify a lottery \(\lottery\in\LotterySet\) with a vector
\[
  \lottery=(\lottery_0,\lottery_1,\ldots,\lottery_{m-1}),
  \qquad
  \lottery_i\ge 0,
  \qquad
  \sum_{i=0}^{m-1}\lottery_i=1,
\]
where \(\lottery_i\) is the probability of terminal outcome \(\outcomei{i}\).  A preference relation \(\pref\) over lotteries is read as
\[
  \lottery\pref\lotteryM
  \quad\Longleftrightarrow\quad
  \text{the agent weakly prefers lottery }\lottery\text{ to lottery }\lotteryM.
\]
The associated strict preference and indifference relations are
\[
  \lottery\strictpref\lotteryM
  \quad\Longleftrightarrow\quad
  \lottery\pref\lotteryM\text{ and not }\lotteryM\pref\lottery,
  \qquad
  \lottery\indiff\lotteryM
  \quad\Longleftrightarrow\quad
  \lottery\pref\lotteryM\text{ and }\lotteryM\pref\lottery.
\]


\subsection{The vNM axioms in the finite lottery setting}
\label{sec:vnm-axioms}

The von Neumann--Morgenstern expected-utility theorem is usually presented in two closely related ways.  The original theory of games presentation gives an axiomatic treatment of utility under risk \citep{vonNeumannMorgenstern1944,vonNeumannMorgenstern1953}.  Later mixture-space presentations, especially Herstein and Milnor's concise formulation, express the same expected-utility idea using an order axiom, independence, and continuity \citep{hersteinMilnor1953,fishburn1982,hammond1998,ozbek2024}.  Here, we will state a six-item tutorial version: completeness, transitivity, substitutability, decomposability, monotonicity, and continuity \citep{kueck2008meu}.

In the canonical finite-lottery theorem, the core behavioral assumptions are: weak order, independence, and continuity, with compound lotteries reduced to their final outcome probabilities.  In the six-item tutorial version, weak order is split into completeness and transitivity; independence is described as substitutability; reduction of compound lotteries is stated explicitly as decomposability; monotonicity is stated as a natural property of binary best/worst lotteries; and continuity is used to calibrate intermediate outcomes against mixtures of best and worst outcomes.

These presentations are not contradictory.  They differ in what is treated as a primitive axiom, what is built into the definition of a lottery, and what is stated as a useful derived property.



\paragraph{Axiom 0: reduction of compound lotteries.}
Only final probabilities over terminal outcomes matter.  If a two-stage lottery first selects one of several lotteries \(\lottery^{(1)},\ldots,\lottery^{(k)}\) with probabilities \(\lambda_1,
\ldots,\lambda_k\), then the compound lottery is identified with the flattened lottery
\[
  \sum_{j=1}^{k}\lambda_j\lottery^{(j)}.
\]
Equivalently, if two procedures induce the same marginal probability for every terminal outcome \(\outcomei{i}\), they are the same element of \(\LotterySet\).

\begin{examplebox}{Example: flattening a compound lottery}
Suppose \(\OutcomeSet=\{\outcomei{0},\outcomei{1},\outcomei{2}\}\).  First flip a fair coin.  If heads, run \(\lottery=(1/2,1/2,0)\).  If tails, run \(\lotteryR=(0,0,1)\).  The final probabilities are
\[
  \tfrac12\lottery+\tfrac12\lotteryR
  =
  \left(\tfrac14,\tfrac14,\tfrac12\right).
\]
Reduction says that the two-stage description and the flattened vector are equivalent.  This is why lotteries can be drawn as points in a probability simplex.
\end{examplebox}

\paragraph{Axiom 1: completeness.}
Any two lotteries can be compared:
\[
  \forall \lottery,\lotteryM\in\LotterySet,
  \qquad
  \lottery\pref\lotteryM\quad\text{or}\quad\lotteryM\pref\lottery.
\]
The agent need not strictly prefer one to the other; indifference is allowed.  Completeness only rules out incomparable pairs.

\begin{examplebox}{Example: completeness}
If \(\lottery\) is ``80\% good answer, 20\% bad answer'' and \(\lotteryM\) is ``60\% good answer, 40\% medium answer'', completeness says the teacher's preference relation must rank them one way or declare indifference.  The theorem does not say which ranking is correct; it only says the relation is defined.
\end{examplebox}

\paragraph{Axiom 2: transitivity.}
Preferences do not cycle:
\[
  \lottery\pref\lotteryM\text{ and }\lotteryM\pref\lotteryN
  \quad\Longrightarrow\quad
  \lottery\pref\lotteryN.
\]
Completeness and transitivity together say that \(\pref\) is a weak order over lotteries.

\begin{examplebox}{Example: no preference cycles}
If the teacher weakly prefers plan trace \(\lottery\) to trace \(\lotteryM\), and trace \(\lotteryM\) to trace \(\lotteryN\), then a later comparison should not reveal \(\lotteryN\strictpref\lottery\).  Such cycles would block representation by a single scalar utility.
\end{examplebox}

\paragraph{Axiom 3: independence, also called substitutability.}
Mixing both sides of a comparison with the same background lottery does not change the comparison.  For any \(\lottery,\lotteryM,\lotteryN\in\LotterySet\) and any \(\mixprob\in(0,1]\),
\[
  \lottery\pref\lotteryM
  \quad\Longleftrightarrow\quad
  \mixprob\lottery+(1-\mixprob)\lotteryN
  \pref
  \mixprob\lotteryM+(1-\mixprob)\lotteryN.
\]
This is the axiom that makes the representation linear in probabilities.  The common background lottery \(\lotteryN\) is irrelevant because it is mixed into both sides in the same way.

\begin{examplebox}{Example: adding the same chance of tool failure}
Suppose a user prefers tool plan \(\lottery\) to tool plan \(\lotteryM\).  Now imagine that both plans are embedded in a system with a 10\% chance of an unrelated server outage, represented by the same background lottery \(\lotteryN\).  Independence says the ranking between the two plans should not reverse merely because both now share the same outage risk:
\[
  \lottery\strictpref\lotteryM
  \quad\Rightarrow\quad
  0.9\lottery+0.1\lotteryN
  \strictpref
  0.9\lotteryM+0.1\lotteryN.
\]
The axiom can be empirically false for humans; the Allais paradox is the classic warning.  In this notebook, however, the teacher is deliberately generated to satisfy it.
\end{examplebox}

\paragraph{Axiom 4: continuity, or the Archimedean condition.}
If one lottery is strictly better than a second, and the second is strictly better than a third, then the middle lottery can be matched by some mixture of the best and worst lotteries:
\[
  \lottery\strictpref\lotteryM\strictpref\lotteryN
  \quad\Longrightarrow\quad
  \exists\mixprob\in[0,1]
  \text{ such that }
  \mixprob\lottery+(1-\mixprob)\lotteryN\indiff\lotteryM.
\]
This rules out lexicographic or infinitely strong preferences.  It says that a sufficiently high probability of the best option can compensate for the risk of the worst option.

\begin{examplebox}{Example: calibrating a medium outcome}
Let \(\outcomebest\) be a perfect answer, \(\outcomeworst\) be a useless answer, and \(\outcome\) be a merely adequate answer.  Continuity says there is some probability \(\mixprob_{\outcome}\) such that
\[
  \outcome
  \indiff
  \mixprob_{\outcome}\outcomebest+(1-\mixprob_{\outcome})\outcomeworst.
\]
If \(\mixprob_{\outcome}=0.4\), then after the usual normalization \(\utility(\outcomeworst)=0\) and \(\utility(\outcomebest)=1\), the utility of the adequate answer is \(\utility(\outcome)=0.4\).
\end{examplebox}

\paragraph{Derived property: monotonicity between best and worst outcomes.}
If \(\outcomebest\strictpref\outcomeworst\), then more probability on \(\outcomebest\) is better:
\[
  \mixprob>\alpha'
  \quad\Longrightarrow\quad
  \mixprob\outcomebest+(1-\mixprob)\outcomeworst
  \strictpref
  \alpha'\outcomebest+(1-\alpha')\outcomeworst.
\]
Many tutorial statements list this as an axiom because it is intuitive and useful for constructing the utility scale.  In the canonical mixture-space formulation, monotonicity is also a consequence of the expected-utility representation once \(\outcomebest\) and \(\outcomeworst\) are ordered.

\subsection{The vNM representation theorem}
\label{sec:vnm-theorem}

\begin{keybox}{Theorem: finite von Neumann--Morgenstern representation}
Let \(\pref\) be a preference relation over \(\LotterySet\).  If compound lotteries are reduced to their final outcome probabilities and \(\pref\) satisfies completeness, transitivity, independence, and continuity, then there exists a utility function
\[
  \utility:\OutcomeSet\to\mathbb{R}
\]
such that, for all lotteries \(\lottery,\lotteryM\in\LotterySet\),
\[
  \lottery\pref\lotteryM
  \quad\Longleftrightarrow\quad
  \sum_{i=0}^{m-1}\lottery_i\utility(\outcomei{i})
  \ge
  \sum_{i=0}^{m-1}\lotteryM_i\utility(\outcomei{i}).
\]
Conversely, any preference relation represented by the expected value of some \(\utility\) satisfies the vNM axioms.  The representing utility is unique only up to positive affine transformation: if \(\utility\) represents \(\pref\), then so does
\[
  \widetilde{\utility}=a\utility+b,
  \qquad a>0,
  \qquad b\in\mathbb{R}.
\]
\end{keybox}

The proof idea is constructive.  Pick a best outcome \(\outcomebest\) and a worst outcome \(\outcomeworst\).  Normalize
\[
  \utility(\outcomeworst)=0,
  \qquad
  \utility(\outcomebest)=1.
\]
For every intermediate outcome \(\outcome\), use continuity to find the probability \(\mixprob_{\outcome}\) such that
\[
  \outcome\indiff
  \mixprob_{\outcome}\outcomebest+(1-\mixprob_{\outcome})\outcomeworst.
\]
Define \(\utility(\outcome)=\mixprob_{\outcome}\).  Independence and reduction then extend the construction from sure outcomes to arbitrary lotteries, forcing lottery values to be expectations.

\begin{examplebox}{Example: expected utility is not expected money}
Suppose
\[
  \utility(\outcomei{0})=0,
  \qquad
  \utility(\outcomei{1})=0.4,
  \qquad
  \utility(\outcomei{2})=1.
\]
Compare
\[
  \lottery=(0.5,0,0.5),
  \qquad
  \lotteryM=(0,1,0).
\]
Then
\[
  \E_{\lottery}[\utility]=0.5\cdot0+0.5\cdot1=0.5,
  \qquad
  \E_{\lotteryM}[\utility]=0.4.
\]
The vNM representation predicts \(\lottery\strictpref\lotteryM\).  If we transform the utility by \(\widetilde{\utility}=10\utility-3\), the numerical expected utilities change, but the ranking does not.  This is why the notebook compares recovered utilities to \(\ustar\) only after affine alignment.
\end{examplebox}

\subsection{From utility to reward maximization}
\label{sec:vnm-utility-to-reward}

The theorem supplies the bridge from interactional preference learning to reward maximization.  If an action \(\action\in\ActionSet\) at history \(\history\in\History\) induces a lottery over terminal outcomes,
\[
  \outcome\sim \prob(\cdot\mid \history,\intervene{\action}),
\]
then choosing the most preferred action is equivalent to
\[
  \action^{\star}(\history)
  \in
  \arg\max_{\action\in\ActionSet}
  \E[\utility(\outcome)\mid \history,\intervene{\action}]
  =
  \arg\max_{\action\in\ActionSet}
  \sum_{\outcome\in\OutcomeSet}
  \prob(\outcome\mid\history,\intervene{\action})\utility(\outcome).
\]
If we define the terminal reward by
\[
  \reward(\outcome)=\utility(\outcome),
\]
then the same choice rule is expected reward maximization.  The reward was not primitive in the interactional setup.  It was recovered as a representation of learned preferences.

\begin{warningbox}{The trajectory caveat}
vNM gives expected utility over lotteries.  To obtain the usual reinforcement-learning return
\[
  U(\trajectory)=\sum_{t=0}^{T}\discount^t\reward(s_t,\action_t,s_{t+1}),
\]
one needs additional separability assumptions saying that trajectory utility decomposes additively over time.  The notebook uses terminal outcomes, so the identification \(\reward(\outcome)=\utility(\outcome)\) is enough.  Sequential Markov rewards require more structure.
\end{warningbox}

\section{Interactional agency}
\label{sec:vnm-natural-test}

In interactional agency, the learner's environment can include other agents.  Their choices, demonstrations, corrections, and comparisons are world-written symbols.  They are evidence.  The learner's own actions are different: they are interventions made from the first-person action channel.  Therefore the preference-learning version of the intervention/evidence rule is
\[
\begin{aligned}
  \gate_i=0 &: \text{teacher or world choice; use as evidence,}\\
  \gate_i=1 &: \text{learner choice; condition on it but do not learn from it as evidence.}
\end{aligned}
\]

The experiment tests whether this rule is enough to turn preference evidence into reward-maximizing behavior.  It is a useful test because the hidden reward \(\ustar\) is known to us as experimenters but never shown to the learner.  Thus we can check, after training, whether the learner inferred the correct utility representation from choices alone.

\begin{examplebox}{The causal reading of a preference comparison}
A transcript line such as
\begin{quote}
Teacher chooses lottery \(\lottery\) over lottery \(\lotteryR\).
\end{quote}
is a world-written observation, so it has \(\gate=0\).  It is evidence about which lotteries the environment's teacher prefers.  By contrast, a line such as
\begin{quote}
Learner chooses lottery \(\lottery\) over lottery \(\lotteryR\).
\end{quote}
is generated by the learner's own action channel, so it has \(\gate=1\).  It may change the future interaction, and it remains in the context, but it is not evidence that the teacher or environment prefers \(\lottery\).
\end{examplebox}

\subsection{Experiment 1 setup}
\label{sec:vnm-synthetic-environment}

The data is an explicit interaction stream.  Each trajectory is a time-ordered sequence of slots.  At each step, the world first writes an observation slot showing two lotteries, and then either the teacher or the learner writes an action slot choosing one of those lotteries:
\[
  \trajectory_k
  =
  \big((\obs_{k,0},\action_{k,0}),
        (\obs_{k,1},\action_{k,1}),
        \ldots,
        (\obs_{k,T-1},\action_{k,T-1})\big).
\]
Here the observation slot is not a terminal outcome.  It is a world-written description of the choice problem:
\[
  \obs_{k,t}=(\lottery_{k,t},\lotteryR_{k,t}).
\]
The terminal outcomes remain
\[
  \OutcomeSet=
  \{\outcomei{0},\outcomei{1},\outcomei{2},\outcomei{3},\outcomei{4},\outcomei{5}\}.
\]


\paragraph{Hidden utilities.}
The teacher has a hidden vNM utility, used by the simulator but never given to the learner:
\[
  \ustar=(0.00,0.25,0.75,1.00,0.50,0.00).
\]
The learner's early own-action policy is generated from a different utility-like vector,
\[
  \selfutil=(1.00,0.70,0.45,0.25,0.10,0.00).
\]
This vector is deliberately biased toward outcomes that the teacher does not value highly. 

\paragraph{Lottery sampling.}
At every step, the world samples two lotteries over \(\OutcomeSet\):
\[
  \lottery_{k,t},\lotteryR_{k,t}
  \sim
  \operatorname{Dirichlet}(0.6\mathbf{1}).
\]
The vector \(\lottery_{k,t}\) gives the probabilities of \(\outcomei{0},\ldots,\outcomei{5}\), and similarly for \(\lotteryR_{k,t}\).  The action slot then records a binary choice
\[
  \Choice_{k,t}=1 \quad\text{if the writer chooses }\lottery_{k,t},
  \qquad
  \Choice_{k,t}=0 \quad\text{if the writer chooses }\lotteryR_{k,t}.
\]

\paragraph{Teacher and agent action slots.}
With probability \(p_{\mathrm{teacher}}=0.40\), the action slot is written by the teacher (we ablate this parameter later).  It has provenance gate \(\gate=0\) and is evidence.  The teacher chooses the better lottery under \(\ustar\), except that with probability \(p_{\mathrm{wrong}}=0.05\) the teacher deliberately chooses the lower-utility lottery.  Thus teacher mistakes are noisy evidence, not interventions.

With probability \(1-p_{\mathrm{teacher}}=0.60\), the action slot is written by the learner's early policy.  It has \(\gate=1\) and is an intervention.  These choices are sampled according to a Bradley--Terry/logistic rule centered on \(\selfutil\):
\[
  \prob(\Choice=1\mid\lottery,\lotteryR,\selfutil)
  =
  \sigmoid\!\left(14(\lottery-\lotteryR)^\top\selfutil\right),
  \qquad
  \sigmoid(z)=\frac{1}{1+e^{-z}}.
\]

\subsection{Generated trajectory data}
\label{sec:new-generated-trajectory-data}

The accompanying notebook run generates 30 trajectories of length 24.  Each step has one observation slot and one action slot, so the full interaction stream contains \(30\times24\times2=1440\) slots.  The realized action split for Experiment 1 is 298 teacher action slots and 422 learner action slots.

\begin{table}[htbp]
\centering
\small
\begin{tabular}{lr}
\toprule
Quantity & Value\\
\midrule
Trajectories & 30\\
Steps per trajectory & 24\\
Total slots & 1440\\
Observation slots & 720\\
Teacher action slots & 298\\
Agent action slots & 422\\
Configured teacher wrong-choice probability & 0.050000\\
Realized teacher wrong-choice rate & 0.060403\\
\bottomrule
\end{tabular}
\caption{Trajectory summary for Experiment 1.  The randomness in the teacher/agent writer choice means that 298 of the 720 action slots are teacher-written evidence and 422 are learner-written interventions.}
\label{tab:new-trajectory-summary}
\end{table}

The notebook then extracts the action slots.  For each action slot it records the writer, the gate \(\gate\), the chosen side, whether the slot should be learned from, and the hidden-utility advantage of the chosen lottery:
\[
  \Delta^{\star}_{k,t}
  =
  \ustar(\text{chosen lottery})
  -
  \ustar(\text{unchosen lottery}).
\]
Here \(\ustar(\lottery)\) abbreviates the expected utility \(\lottery^\top\ustar\).  Positive \(\Delta^{\star}\) means that the action chose the better lottery under the teacher's hidden reward; negative \(\Delta^{\star}\) means it chose the worse lottery.


Figure~\ref{fig:trajectory-prefix} shows the rendered trajectory view from the notebook.  Each step contains one observation slot, in which the world reveals two lotteries, followed by one action slot, which is either teacher-written or agent-written.  Blue teacher panels are evidence with \(\gate=0\); orange agent panels are interventions with \(\gate=1\) and are therefore masked out by the interventional learner.  The figure makes the causal distinction visually explicit: the two kinds of action slot live in the same history, but they play different epistemic roles.

\begin{figure}[htbp]
  \centering
  \includegraphics[width=\textwidth]{vnm_extracted/trajectory_top.png}
  \caption{A rendered prefix of one simulated interaction trajectory from the notebook.  World observation slots display the pair of lotteries shown at each step.  Teacher action slots are world-written evidence used for learning; agent action slots are self-written interventions and are masked out by the interventional learner.  The diagnostic line in each observation slot reports the hidden expected utility of the left and right lotteries under \(\ustar\), which is shown only for analysis and never provided to the learner.}
  \label{fig:trajectory-prefix}
\end{figure}

\subsection{The observational and interventional learners}
\label{sec:new-two-trajectory-learners}

Both learners fit the same scalar expected-utility preference model.  Given a pair of lotteries \((\lottery,\lotteryR)\), the model predicts
\[
  \prob_{\wvec}(\Choice=1\mid \lottery,\lotteryR)
  =
  \sigmoid\!\left((\lottery-\lotteryR)^\top\wvec\right),
\]
where \(\wvec\in\mathbb{R}^6\) is a learned utility-weight vector.  The additive constant of \(\wvec\) is not identifiable, since \((\lottery-\lotteryR)^\top\mathbf{1}=0\), so the notebook recenters \(\wvec\) after each gradient update.

For a set of action slots \(\Dataset\), the fitted objective is
\[
\begin{aligned}
  J(\wvec;\Dataset)
  &=-\frac{1}{|\Dataset|}\sum_{j\in\Dataset}
  \Big[
  \Choice_j\log\sigmoid(X_j^\top\wvec)
  +(1-\Choice_j)\log\big(1-\sigmoid(X_j^\top\wvec)\big)
  \Big]
  +\frac{10^{-3}}{2}\|\wvec\|_2^2,\\
  X_j&=\lottery_j-\lotteryR_j.
\end{aligned}
\]
The main fit uses learning rate 0.25 and 2500 gradient steps.

\paragraph{Interventional learner.}
The interventional learner uses only the teacher action slots:
\[
  \Ddo
  =
  \{(\lottery_j,\lotteryR_j,\Choice_j):\texttt{learn\_from\_slot}=\texttt{True}\}.
\]
In the main run, \(|\Ddo|=298\).  These are exactly the action slots with \(\gate=0\) that carry world-written preference evidence.

\paragraph{Observational learner.}
The observational learner uses all action slots:
\[
  \Dobs
  =
  \{(\lottery_j,\lotteryR_j,\Choice_j):\texttt{slot\_type}\in
  \{\texttt{teacher\_action},\texttt{agent\_action}\}\}.
\]
In the main run, \(|\Dobs|=720\).  This learner has more data, but 422 of its rows are the learner's own interventions generated by the biased early policy.

\begin{warningbox}{Why this is a trajectory-level test}
A pure comparison dataset can hide the fact that preference evidence and learner actions are interleaved in one history.  This notebook puts both kinds of events inside the same interaction stream.  The learner has to respect provenance at the slot level: teacher action slots are evidence, while agent action slots are interventions.  This is the preference-learning analogue of the intervention/evidence distinction in interactive imitation.
\end{warningbox}

\section{Results}

\subsection{Utility recovery from one interaction history}
\label{sec:new-main-utility-recovery}

Since vNM utility is unique only up to positive affine transformation, the notebook evaluates a learned vector \(\widehat{\wvec}\) by fitting the least-squares affine alignment
\[
  \widetilde{u}(\outcomei{i})=a\widehat{w}_i+b
\]
to the hidden teacher utility \(\ustar\).  A positive slope \(a>0\) is preference-preserving; a high correlation and low aligned MSE indicate that the recovered utility has the same vNM content as \(\ustar\).

\begin{table}[htbp]
\centering
\small
\resizebox{\textwidth}{!}{%
\begin{tabular}{lrrrr}
\toprule
Learner & Training action slots & Correlation with \(\ustar\) & Affine slope & Aligned MSE\\
\midrule
Interventional: teacher actions only & 298 & 0.988000 & 0.116886 & 0.003313\\
Observational: all actions & 720 & 0.366445 & 0.092898 & 0.120239\\
\bottomrule
\end{tabular}%
}
\caption{Main utility-recovery result.  The interventional learner uses fewer rows, but those rows are the right rows: world-written teacher actions.  The observational learner uses more rows but corrupts the inferred purpose by treating agent interventions as evidence.}
\label{tab:new-fit-summary}
\end{table}

Figure~\ref{fig:main-fit-plots} visualizes the main fitting result.  The left panel shows that the interventional training loss continues to decrease as the learner extracts a coherent signal from the teacher-written preference actions, whereas the observational loss quickly plateaus at a much worse value because the dataset mixes teacher evidence with the agent's own biased interventions.  The right panel compares the recovered utility vectors after affine alignment.  The interventional curve almost coincides with the hidden teacher utility, while the observational curve is flattened and misordered.

\begin{figure}[htbp]
  \centering
  \includegraphics[width=\textwidth]{vnm_extracted/cell12_out0.png}
  \caption{Main result of Experiment 1.  Left: training loss as a function of gradient steps for the interventional and observational learners.  Right: hidden teacher utility together with the recovered utility vectors after affine alignment.  The interventional learner recovers the correct preference ordering and spacing much more faithfully.}
  \label{fig:main-fit-plots}
\end{figure}

The interventional learner places high utility on \(\outcomei{2}\) and \(\outcomei{3}\), low utility on \(\outcomei{0}\) and \(\outcomei{5}\), and an intermediate value on \(\outcomei{4}\), matching the teacher.  The observational learner is much flatter and substantially misordered because it tries to explain both the teacher's choices and the biased agent's choices with one utility vector.



\subsection{Learning over time from trajectory prefixes}
\label{sec:new-learning-over-time}

The notebook next replays the history from left to right.  For a checkpoint slot \(s\), let \(\Dataset_{\le s}\) be the action slots observed up to that point.  The interventional learner fits only the teacher-written subset of \(\Dataset_{\le s}\); the observational learner fits every action slot in \(\Dataset_{\le s}\).  At each checkpoint, the recovered utility is also evaluated in fresh action environments.

A fresh action environment contains four possible actions.  Each action induces a lottery over terminal outcomes:
\[
  \prob(\outcome\mid \history,\intervene{\action}),
  \qquad
  \action\in\{1,2,3,4\}.
\]
Given policy utility \(\wvec\), the agent chooses
\[
  \action_{\wvec}(\history)
  \in
  \arg\max_{\action}
  \Qval_{\wvec}(\history,\action),
  \qquad
  \Qval_{\wvec}(\history,\action)
  =
  \sum_{\outcome\in\OutcomeSet}
  \prob(\outcome\mid\history,\intervene{\action})\wvec(\outcome).
\]
The choice is scored under the hidden reward \(\rstar(\outcome)=\ustar(\outcome)\).  The regret is
\[
  \Regret(\history)
  =
  \max_{\action}
  \sum_{\outcome}
  \prob(\outcome\mid\history,\intervene{\action})\ustar(\outcome)
  -
  \sum_{\outcome}
  \prob(\outcome\mid\history,
        \intervene{\action_{\wvec}(\history)})\ustar(\outcome).
\]
The notebook uses 1200 sampled states, four action lotteries per state, and a Dirichlet concentration of 0.7 for these evaluations.


\subsection{Experiment 2: Teacher mistakes}
\label{sec:new-teacher-mistakes}

We vary the teacher wrong-choice probability.  This is important conceptually.  A teacher mistake is a bad label, but it is still world-written evidence.  A learner action is not evidence at all; it is an intervention.  The experiment therefore predicts that teacher noise should degrade the interventional learner gradually, while the observational learner remains additionally burdened by self-action contamination.

For this sweep, each run uses 24 trajectories of length 24, teacher action probability \(p_{\mathrm{teacher}}=0.60\), and five random repeats per teacher-noise level.  
Figure~\ref{fig:teacher-noise-sweep} depicts the results.  The left panel shows utility recovery degrading smoothly as teacher noise increases, but the interventional learner stays well above the observational learner throughout the sweep.  The middle panel shows the resulting reward-optimal action rate, and the right panel shows hidden-reward regret.  The lesson is subtle but important: noisy teacher evidence is still evidence, whereas the learner's own actions are not evidence at all.

\begin{figure}[t]
  \centering
  \includegraphics[width=\textwidth]{vnm_extracted/cell18_out0.png}
  \caption{Teacher-noise sweep from the notebook.  Left: utility recovery as a function of teacher wrong-choice probability.  Middle: rate of choosing the hidden-reward-optimal action in fresh environments.  Right: mean hidden-reward regret.  Even with substantial teacher noise, the interventional learner remains dramatically better than the observational learner.}
  \label{fig:teacher-noise-sweep}
\end{figure}

At the notebook's default mistake level of 0.05, the sweep gives the interventional learner mean correlation 0.993355, mean optimal action rate 0.933667, and mean regret 0.001200.  The observational learner at the same noise level has mean correlation 0.795759, mean optimal action rate 0.709167, and mean regret 0.027085.  Even when the teacher is perfectly reliable, the observational learner is worse because it still mixes teacher evidence with biased self-interventions.

\subsection{Experiment 3: How much teacher evidence is needed?}
\label{sec:new-teacher-frequency-sweep}

The final experimental sweep varies \(p_{\mathrm{teacher}}\), the probability that an action slot is written by the teacher rather than the agent.  This directly tests the amount of world-written preference evidence available to the interventional learner.  At \(p_{\mathrm{teacher}}=0\), there are no teacher action slots; the interventional learner has no preference evidence and its entries are intentionally undefined.  At \(p_{\mathrm{teacher}}=1\), all action slots are teacher evidence; the interventional and observational learners become identical because there are no self-interventions to contaminate the dataset.

The sweep uses 24 trajectories of length 24, teacher wrong-choice probability 0.05, and five repeats per value of \(p_{\mathrm{teacher}}\).  

Figure~\ref{fig:teacher-frequency-sweep} presents the results graphically.  The left panel shows that the interventional learner recovers the hidden utility almost as soon as even a modest amount of teacher evidence is available.  The middle panel translates this into reward-optimal action choice.  The right panel shows the performance gap, measured as observational regret minus interventional regret.  The gap closes only when almost all action slots are teacher-written, because only then does the observational learner stop being contaminated by self-generated interventions.

\begin{figure}[t]
  \centering
  \includegraphics[width=\textwidth]{vnm_extracted/cell20_out2.png}
  \caption{Teacher-action-frequency sweep from the notebook.  Left: utility recovery as the teacher-action probability \(p_{\mathrm{teacher}}\) increases.  Middle: reward-optimal action rate in fresh environments.  Right: the regret gap between the observational and interventional learners.  The interventional learner needs some teacher evidence, but it remains strong even when teacher slots are a minority of the history.}
  \label{fig:teacher-frequency-sweep}
\end{figure}

The qualitative pattern is the central result of the experiment.  With only about 56.6 teacher action slots on average, the interventional learner already reaches correlation 0.981 and optimal action rate 0.888.  The observational learner at the same point has correlation \(-0.184\), optimal action rate 0.127, and mean regret 0.229413.  As \(p_{\mathrm{teacher}}\) increases, the observational learner gradually recovers because the fraction of contaminated self-action rows decreases.  When \(p_{\mathrm{teacher}}=1\), the two learners coincide. This result is remarkable for the interventional agent in terms of sample complexity.

\section{Discussion}
\label{sec:new-did-recover-vnm}

Did the interactional agent recover the vNM result? Yes, in the controlled sense tested by the trajectory notebook.  The vNM theorem itself is not an empirical result: if a preference relation over lotteries satisfies the axioms, then an expected-utility representation exists.  What the notebook tests is whether a learner embedded in an interaction history can recover the teacher's expected-utility representation from preference-revealing world-written actions while ignoring its own interventions as evidence.

The interventional learner does recover that representation.  In the main full-history fit, it obtains correlation 0.988000 with \(\ustar\) and aligned MSE 0.003313 using only 298 teacher action slots.  In the prefix replay, at the final history slot, it obtains correlation 0.988501, reward-optimal action rate 0.903333, and mean regret 0.002073.  In the teacher-frequency sweep at \(p_{\mathrm{teacher}}=0.4\), close to the main setting, it obtains mean correlation 0.992664, mean optimal action rate 0.935167, and mean regret 0.001307.

The observational learner is the negative control.  It also fits a scalar expected-utility model, but it fits the wrong one.  In the main full-history fit, its correlation with \(\ustar\) is only 0.366445 and its aligned MSE is 0.120239.  At the final prefix checkpoint, its optimal action rate is 0.363333 and its mean hidden-reward regret is 0.137797.  In the teacher-frequency sweep at \(p_{\mathrm{teacher}}=0.4\), its mean optimal action rate is only 0.329667 and its mean regret is 0.141122.



Does this show that interactive imitation is enough for reward maximization? It shows a conditional sufficiency result in a clean trajectory-level interactional setting.  The condition is not merely ``imitate everything in the transcript.''  The condition is: imitate from an interaction stream while preserving the causal distinction between world-written evidence and agent-written interventions.  When the environment supplies enough vNM-consistent teacher preference evidence, and when the learner masks its own action slots from the preference-learning objective, the recovered utility is good enough that maximizing its expectation is almost the same as maximizing the hidden teacher reward in new action environments.

The teacher-frequency sweep clarifies the limits.  With \(p_{\mathrm{teacher}}=0\), the interventional learner has no teacher preference evidence and cannot infer the teacher's purpose.  With \(p_{\mathrm{teacher}}=1\), there are no agent interventions, so observational and interventional learning coincide.  The interesting region is the mixed case, where the same history contains both teacher evidence and self-generated actions.  There the intervention mask is decisive.

The result should therefore be stated carefully:
\begin{quote}
\emph{Interactive imitation can be enough for reward-maximizing behavior in this controlled vNM environment, provided that the interaction stream contains sufficient world-written preference evidence and the learner does not treat its own interventions as evidence.}
\end{quote}
It does not show that arbitrary behavioral cloning yields reward maximization.  It does not show that all human preferences satisfy vNM axioms.  It also does not show that every utility over whole trajectories decomposes into a Markovian per-step reward.  But it does show the conceptual bridge needed by the interactional account: reward maximization can emerge as a representation of learned preferences, rather than being inserted as a primitive scalar training signal.


\begin{thebibliography}{99}

\bibitem[Allais(1953)]{allais1953}
Allais, M. (1953).
Le comportement de l'homme rationnel devant le risque: critique des postulats et axiomes de l'\'{e}cole am\'{e}ricaine.
\emph{Econometrica}, 21(4), 503--546.

\bibitem[Fishburn(1982)]{fishburn1982}
Fishburn, P. C. (1982).
\emph{The Foundations of Expected Utility}.
Dordrecht: D. Reidel.

\bibitem[Hammond(1998)]{hammond1998}
Hammond, P. J. (1998).
Objective expected utility.
In S. Barber\`{a}, P. J. Hammond, and C. Seidl (eds.), \emph{Handbook of Utility Theory}, Vol. 1.

\bibitem[Herstein and Milnor(1953)]{hersteinMilnor1953}
Herstein, I. N., and Milnor, J. (1953).
An axiomatic approach to measurable utility.
\emph{Econometrica}, 21(2), 291--297.

\bibitem[Kueck et al.(2008)]{kueck2008meu}
Kueck, H., Hoffman, M., Doucet, A., and de Freitas, N. (2008).
Inference and learning for active sensing, experimental design and control.

\bibitem[Nakano et al.(2021)]{nakano2021webgpt}
Nakano, R., Hilton, J., Balaji, S., Wu, J., Ouyang, L., Kim, C., et al. (2021).
WebGPT: Browser-assisted question-answering with human feedback.
\emph{arXiv:2112.09332}.

\bibitem[Ouyang et al.(2022)]{ouyang2022instructgpt}
Ouyang, L., Wu, J., Jiang, X., Almeida, D., Wainwright, C., Mishkin, P., et al. (2022).
Training language models to follow instructions with human feedback.
\emph{NeurIPS}.

\bibitem[Ozbek(2024)]{ozbek2024}
Ozbek, K. (2024).
Expected utility, independence, and continuity.
\emph{Theory and Decision}, 97, 1--22.

\bibitem[Qin et al.(2024)]{qin2024toolllm}
Qin, Y., Liang, S., Ye, Y., Zhu, K., Yan, L., Lu, Y., et al. (2024).
ToolLLM: Facilitating large language models to master 16000+ real-world APIs.
\emph{ICLR}.

\bibitem[Rafailov et al.(2023)]{rafailov2023dpo}
Rafailov, R., Sharma, A., Mitchell, E., Ermon, S., Manning, C. D., and Finn, C. (2023).
Direct Preference Optimization: Your language model is secretly a reward model.
\emph{arXiv:2305.18290}.

\bibitem[Schick et al.(2023)]{schick2023toolformer}
Schick, T., Dwivedi-Yu, J., Dess\`{i}, R., Raileanu, R., Lomeli, M., Hambro, E., et al. (2023).
Toolformer: Language models can teach themselves to use tools.
\emph{NeurIPS}.

\bibitem[Stiennon et al.(2020)]{stiennon2020summarize}
Stiennon, N., Ouyang, L., Wu, J., Ziegler, D., Lowe, R., Voss, C., et al. (2020).
Learning to summarize from human feedback.
\emph{NeurIPS}.

\bibitem[von Neumann and Morgenstern(1944)]{vonNeumannMorgenstern1944}
von Neumann, J., and Morgenstern, O. (1944).
\emph{Theory of Games and Economic Behavior}.
Princeton University Press.

\bibitem[von Neumann and Morgenstern(1953)]{vonNeumannMorgenstern1953}
von Neumann, J., and Morgenstern, O. (1953).
\emph{Theory of Games and Economic Behavior}, 3rd ed.
Princeton University Press.

\bibitem[Yao et al.(2022)]{yao2022react}
Yao, S., Zhao, J., Yu, D., Du, N., Shafran, I., Narasimhan, K., and Cao, Y. (2022).
ReAct: Synergizing reasoning and acting in language models.
\emph{arXiv:2210.03629}.

\bibitem[OpenReview APOLLO submission(2026)]{apollo2026}
Anonymous authors. (2026).
Towards preference following in tool calling language agents.
OpenReview ACL ARR January 2026 submission.

\bibitem[Yehudai et al.(2026)]{agentEvalSurvey2026}
Yehudai, A., Eden, L., Li, A., Uziel, G., Zhao, Y., Bar-Haim, R., Cohan, A., and Shmueli-Scheuer, M. (2026).
A survey on evaluation of LLM-based agents.
\emph{arXiv:2503.16416v2}.

\end{thebibliography}

\end{document}
