\section{Proofs}
\label{app:proofs}

\subsection{Proof of Theorem~\ref{thm:convergence}}
\label{app:convergence}

\begin{proof}
Write $e_t=\theta_t-\theta^*$, $g_t=\nabla F(\theta_t)$, and
$V_t=\|e_t\|^2$. We first bound the historical error, then establish a
conditional contraction and use it separately for the quantitative
mean-square bound and almost-sure convergence.

\paragraph{Historical error.}
Independence and centering of the innovations imply
\[
\mathbb E\left\|\sum_{s=1}^tW_{t,s}\varepsilon_s\right\|^2
=\sum_{s=1}^t\operatorname{tr}(W_{t,s}QW_{t,s}^{\top}).
\]
Using $\|u+v\|^2\le2\|u\|^2+2\|v\|^2$ and
\eqref{eq:weight_bounds} gives
\begin{equation}\label{eq:b_second_moment}
\mathbb E\|b_t\|^2\le\frac{B}{t},\qquad B=2(B_W+B_\delta).
\end{equation}
If $\delta_t\equiv0$, the identity above gives $B=B_W$ directly.

\paragraph{Conditional contraction.}
For a convex $L$-smooth function, the gradient is $1/L$-cocoercive.
Since $\nabla F(\theta^*)=0$, this and strong convexity give
\[
\|g_t\|^2\le L\langle e_t,g_t\rangle,
\qquad \langle e_t,g_t\rangle\ge\mu V_t.
\]
Consequently, whenever $\eta_t\le1/L$,
\begin{equation}\label{eq:gradient_contraction}
\begin{aligned}
\|e_t-\eta_tg_t\|^2
&\le V_t-(2\eta_t-L\eta_t^2)\langle e_t,g_t\rangle\\
&\le(1-\mu\eta_t)V_t.
\end{aligned}
\end{equation}
In particular, $\|e_t-\eta_tg_t\|\le\|e_t\|$.
Young's inequality yields
\[
2\eta_t\bigl|\langle e_t-\eta_tg_t,b_t\rangle\bigr|
\le\frac{\mu\eta_t}{2}V_t+\frac{2\eta_t}{\mu}\|b_t\|^2.
\]
Conditioning the squared update on $\mathcal F_t$ eliminates the
cross term with $\xi_{t+1}$. Since $\eta_t\le1/L\le1/\mu$, we obtain
\begin{equation}\label{eq:conditional_recursion}
\begin{aligned}
\mathbb E[V_{t+1}\mid\mathcal F_t]
&\le\left(1-\frac{\mu\eta_t}{2}\right)V_t
 +\left(\frac{2\eta_t}{\mu}+\eta_t^2\right)\|b_t\|^2
 +\sigma^2\eta_t^2\\
&\le\left(1-\frac{\mu\eta_t}{2}\right)V_t
 +\frac{3\eta_t}{\mu}\|b_t\|^2+\sigma^2\eta_t^2.
\end{aligned}
\end{equation}
All second moments are finite, including before the small-step regime.
Indeed, Lipschitz continuity of the gradient implies
$\|e_t-\eta_tg_t\|\le(1+\eta_tL)\|e_t\|$.
Since $\eta_t\le D$, the inequality
$\|u+v+w\|^2\le3(\|u\|^2+\|v\|^2+\|w\|^2)$ gives, for every $t\ge1$,
\[
a_{t+1}\le A a_t+3D^2(\sigma^2+B),\qquad A=3(1+DL)^2>1.
\]
Thus the initial interval is controlled explicitly by
\begin{equation}\label{eq:initial_bound}
C_{\mathrm{init}}=a_n
\le A^{n-1}a_1+
3D^2(\sigma^2+B)\frac{A^{n-1}-1}{A-1}.
\end{equation}
The right-hand side may replace $C_{\mathrm{init}}$ in
\eqref{eq:mse_bound}.

\paragraph{Explicit mean-square bound.}
Taking expectations in \eqref{eq:conditional_recursion} and using
\eqref{eq:b_second_moment} gives, for $t\ge n$,
\begin{equation}\label{eq:scalar_recursion}
a_{t+1}\le(1-ct^{-\alpha})a_t
+q t^{-2\alpha}+h t^{-\alpha-1},
\quad
c=\frac{\mu D}{2},\quad q=D^2\sigma^2,\quad h=\frac{3DB}{\mu}.
\end{equation}
By the definition of $n$, $\eta_t\le1/L$ and
$t^{1-\alpha}\ge2/c$ for all $t\ge n$.
For $p\in\{\alpha,1\}$, convexity of $x\mapsto x^{-p}$ implies
\[
(t+1)^{-p}\ge t^{-p}(1-p/t)
\ge t^{-p}\left(1-\frac c2t^{-\alpha}\right).
\]
Set
\[
f_t=\frac{2q}{c}t^{-\alpha}+\frac{2h}{c}t^{-1}
=C_\xi t^{-\alpha}+C_b t^{-1}.
\]
The preceding inequality shows that
\[
f_{t+1}\ge(1-ct^{-\alpha})f_t
+q t^{-2\alpha}+h t^{-\alpha-1}.
\]
Let $P_{n,n}=1$ and
$P_{n,t}=\prod_{j=n}^{t-1}(1-cj^{-\alpha})$ for $t>n$.
The factors lie in $[1/2,1]$. Comparing with \eqref{eq:scalar_recursion}
and inducting from $t=n$ yields
\[
a_t\le a_nP_{n,t}+f_t.
\]
Finally, $1-x\le e^{-x}$ and an integral comparison give
\[
P_{n,t}\le\exp\!\left(-c\sum_{j=n}^{t-1}j^{-\alpha}\right)
\le\exp\!\left[-\frac{c}{1-\alpha}
\bigl(t^{1-\alpha}-n^{1-\alpha}\bigr)\right].
\]
This proves \eqref{eq:mse_bound}, with the constants in
\eqref{eq:constants}, and hence mean-square convergence.

\paragraph{Almost-sure convergence.}
Define the nonnegative $\mathcal F_t$-measurable remainder
\[
u_t=\frac{3\eta_t}{\mu}\|b_t\|^2+\sigma^2\eta_t^2.
\]
Because $\alpha>1/2$ and \eqref{eq:b_second_moment} holds,
\[
\mathbb E\sum_{t=n}^{\infty}u_t
\le\frac{3DB}{\mu}\sum_{t=n}^{\infty}t^{-\alpha-1}
 +D^2\sigma^2\sum_{t=n}^{\infty}t^{-2\alpha}<\infty.
\]
Tonelli's theorem therefore gives $\sum_{t=n}^{\infty}u_t<\infty$
almost surely. Applying the Robbins--Siegmund almost-supermartingale
theorem~\cite{robbins1971} to \eqref{eq:conditional_recursion}, $V_t$ converges to a finite
nonnegative limit and
\[
\sum_{t=n}^{\infty}\eta_tV_t<\infty\qquad\text{almost surely}.
\]
Since $\sum_t\eta_t=\infty$, a positive limit of $V_t$ would contradict
this summability. Thus $V_t\to0$ almost surely, completing the proof.
\end{proof}


\subsection{Proof of Theorem~\ref{thm:pr}}
\label{app:pr}

\begin{proof}
Throughout the proof, write $\|X\|_{L^2}=(\mathbb E\|X\|^2)^{1/2}$.
We first prove the classification, which only uses
\eqref{eq:beta_bound}, and then establish the PR central limit theorem.

\paragraph{Classification of cumulative errors.}
Minkowski's inequality yields
\begin{equation}\label{eq:cumulative_l2}
\|B_T\|_{L^2}
\le\frac{\sqrt B}{\sqrt T}\sum_{t=1}^Tt^{-\beta/2}.
\end{equation}
Since $\|\operatorname{Cov}(B_T)\|_{\mathrm{op}}
\le\mathbb E\|B_T\|^2$, integral comparisons give
\begin{equation}\label{eq:covariance_rates}
\|\Gamma_T^{(b)}\|_{\mathrm{op}}\le
\begin{cases}
O(T^{1-\beta}), & 0<\beta<2,\\
O(T^{-1}\log^2 T), & \beta=2,\\
O(T^{-1}), & \beta>2.
\end{cases}
\end{equation}
Thus $B_T\to0$ in $L^2$ for $\beta>1$, whereas the covariance is bounded
for $\beta=1$.

The remaining possibilities are realized by scalar historical errors.
Let $U$ be centered with variance one, measurable at time one, and set
$\delta_t=0$. Each example below has the form $b_t=w_tU$, obtained by
using only the first innovation in the historical representation.
For $0<\beta<1$, the choices $w_t=0$ and $w_t=t^{-1/2}$ give limiting
cumulative variances zero and four, respectively, and both satisfy
\eqref{eq:beta_bound}. Taking $w_t=t^{-\beta/2}$ gives
\[
\operatorname{Var}(B_T)\sim
\frac{T^{1-\beta}}{(1-\beta/2)^2}\longrightarrow\infty.
\]
For $\beta=1$, take $w_t=t^{-1/2}[1+\sin(\log t)]$. Integral comparison
with a bounded sum--integral error gives
\[
\frac1{\sqrt T}\sum_{t=1}^Tw_t
=2+\operatorname{Im}\!\left(\frac{T^{i}}{1/2+i}\right)+o(1).
\]
Its square is bounded and does not converge. This example also satisfies
\eqref{eq:beta_bound} for every $0<\beta<1$.
These examples establish the possibilities in the table; they are not
assertions that all of the additional CLT conditions hold.

\paragraph{Mean-square control for a general exponent.}
Now assume all the PR conditions. Independence of $Z_{t+1}$ from
$\mathcal F_t$ implies the gradient-noise conditions of
Theorem~\ref{thm:convergence}, with $\sigma^2=\operatorname{tr}(S)$.
The conditional contraction \eqref{eq:conditional_recursion} only
requires adaptedness of $b_t$. Inserting \eqref{eq:beta_bound} gives,
for all sufficiently large $t$,
\[
a_{t+1}\le(1-ct^{-\alpha})a_t
+q t^{-2\alpha}+h t^{-\alpha-\beta},
\qquad c=\frac{\mu D}{2},\quad
q=D^2\operatorname{tr}(S),\quad h=\frac{3DB}{\mu}.
\]
For $t^{1-\alpha}\ge2\max\{\alpha,\beta\}/c$ and
$p\in\{\alpha,\beta\}$,
\[
(t+1)^{-p}\ge t^{-p}(1-p/t)
\ge t^{-p}\left(1-\frac c2t^{-\alpha}\right).
\]
The comparison argument from Appendix~\ref{app:convergence}, now with
$f_t=(2q/c)t^{-\alpha}+(2h/c)t^{-\beta}$, therefore proves
\begin{equation}\label{eq:general_beta_mse}
a_t=O(t^{-\alpha}+t^{-\beta}).
\end{equation}
All finite initial moments are controlled as before, since
$\mathbb E\|b_t\|^2\le B$. The same almost-supermartingale argument also
gives almost-sure convergence, since $\alpha+\beta>1$ under
$\beta>\max\{1/2,2\alpha-1\}$.

\paragraph{PR linearization.}
Assumption~\ref{ass:objective}(b) and the integral form of Taylor's formula
give, with $e=\theta-\theta^*$,
\begin{equation}\label{eq:quadratic_remainder}
\begin{aligned}
\|\nabla F(\theta)-He\|
&=\left\|\int_0^1
[\nabla^2F(\theta^*+ue)-H]e\,du\right\|\\
&\le\int_0^1L_Hu\|e\|^2\,du
=\frac{L_H}{2}\|e\|^2.
\end{aligned}
\end{equation}
Thus the quadratic remainder constant is $M=L_H/2$.

Write $e_t=\theta_t-\theta^*$ and
$\rho_t=\nabla F(\theta_t)-He_t$. The update gives the exact identity
\begin{equation}\label{eq:pr_identity}
H\frac1{\sqrt T}\sum_{t=1}^Te_t
=A_T-\frac1{\sqrt T}\sum_{t=1}^T(\xi_{t+1}+b_t)
-\frac1{\sqrt T}\sum_{t=1}^T\rho_t,
\end{equation}
where summation by parts yields
\[
A_T=\frac1{\sqrt T}\sum_{t=1}^T\frac{e_t-e_{t+1}}{\eta_t}
=\frac1{\sqrt T}\left[
\frac{e_1}{\eta_1}-\frac{e_{T+1}}{\eta_T}
+\sum_{t=2}^T(\eta_t^{-1}-\eta_{t-1}^{-1})e_t\right].
\]
By \eqref{eq:general_beta_mse},
$\|e_t\|_{L^2}=O(t^{-\alpha/2}+t^{-\beta/2})$.
Also $\eta_t^{-1}-\eta_{t-1}^{-1}\le C t^{\alpha-1}$ for $t\ge2$,
where $C$ depends only on $D,\alpha$.
Consequently,
\begin{equation}\label{eq:boundary_control}
\|A_T\|_{L^2}
\le O(T^{-1/2})
+O\!\left(T^{(\alpha-1)/2}+T^{\alpha-(1+\beta)/2}\right)
+\frac C{\sqrt T}\sum_{t=2}^T
\left(t^{\alpha/2-1}+t^{\alpha-\beta/2-1}\right)
\longrightarrow0.
\end{equation}
Indeed, the first power sum is $O(T^{\alpha/2})$, and the second is
$O(T^{\alpha-\beta/2})$ if $\beta<2\alpha$, $O(\log T)$ if
$\beta=2\alpha$, and $O(1)$ if $\beta>2\alpha$.
The resulting terms vanish because $\alpha<1$ and $\beta>2\alpha-1$.
Moreover, \eqref{eq:quadratic_remainder} and
\eqref{eq:general_beta_mse} imply
\[
\mathbb E\left\|\frac1{\sqrt T}\sum_{t=1}^T\rho_t\right\|
\le\frac M{\sqrt T}\sum_{t=1}^Ta_t
=\frac1{\sqrt T}\sum_{t=1}^TO(t^{-\alpha}+t^{-\beta})\longrightarrow0,
\]
since $\alpha,\beta>1/2$. Hence \eqref{eq:pr_identity} becomes
\begin{equation}\label{eq:pr_linearization}
\sqrt T(\bar\theta_T-\theta^*)
=-H^{-1}\frac1{\sqrt T}\sum_{t=1}^T(\xi_{t+1}+b_t)+o_p(1).
\end{equation}
The index shift in the noise is harmless: its difference from the sum
of $\xi_t$ is $(\xi_{T+1}-\xi_1)/\sqrt T$, which converges to zero in
$L^2$ by the finite second moment assumption.

\paragraph{Joint triangular-array central limit theorem.}
Exchange the finite historical sums and use \eqref{eq:beta_bound} to get
\[
B_T=\frac1{\sqrt T}\sum_{s=1}^TK_{T,s}\varepsilon_s+o_p(1).
\]
Define the independent triangular-array summands
\[
Y_{T,s}=\frac1{\sqrt T}
\begin{pmatrix}\xi_s\\K_{T,s}\varepsilon_s\end{pmatrix}.
\]
Their total covariance converges, by \eqref{eq:weight_clt}, to
\begin{equation}\label{eq:joint_limit_covariance}
\sum_{s=1}^T\mathbb E[Y_{T,s}Y_{T,s}^\top]
\longrightarrow
\begin{pmatrix}S&C_0K^\top\\KC_0^\top&\Gamma\end{pmatrix}.
\end{equation}
For completeness, the finite-second-moment assumption and the maximal
weight condition imply Lindeberg even if $\mathcal Q$ is singular.
If $\mathcal Q$ has positive rank, choose a full-column-rank matrix $P$
with $\mathcal Q=PP^\top$. There are i.i.d. centered vectors $U_s$ with
identity covariance and finite second moment such that $Z_s=PU_s$
almost surely: take $U_s=P^\dagger Z_s$, where $P^\dagger$ is the left
pseudoinverse. The equality follows because the component of $Z_s$
orthogonal to the range of $P$ has zero second moment. Write
\[
Y_{T,s}=A_{T,s}U_s,\qquad
A_{T,s}=\frac1{\sqrt T}
\begin{pmatrix}I_p&0\\0&K_{T,s}\end{pmatrix}P.
\]
Then $m_T:=\max_s\|A_{T,s}\|_{\mathrm{op}}\to0$ and
$\sum_s\|A_{T,s}\|_{\mathrm F}^2=O(1)$ by
\eqref{eq:joint_limit_covariance}. For any $\epsilon>0$,
\[
\sum_{s=1}^T\mathbb E\!\left[
\|Y_{T,s}\|^2\mathbf1_{\{\|Y_{T,s}\|>\epsilon\}}\right]
\le\left(\sum_{s=1}^T\|A_{T,s}\|_{\mathrm F}^2\right)
\mathbb E\!\left[\|U_1\|^2
\mathbf1_{\{\|U_1\|>\epsilon/m_T\}}\right]\longrightarrow0.
\]
Rows with $m_T=0$ are identically zero. If $\mathcal Q=0$, the whole
array is identically zero. The multivariate Lindeberg--Feller theorem
therefore gives the joint Gaussian limit with covariance
\eqref{eq:joint_limit_covariance}. Adding its two blocks yields
\[
\frac1{\sqrt T}\sum_{t=1}^T(\xi_{t+1}+b_t)
\Rightarrow N(0,S+\Gamma+C_0K^\top+KC_0^\top).
\]
Combining this with \eqref{eq:pr_linearization} and Slutsky's theorem
proves \eqref{eq:pr_clt}--\eqref{eq:effective_covariance}.

Finally, if $\beta>1$, \eqref{eq:cumulative_l2} gives $B_T\to0$ in
$L^2$. The ordinary i.i.d. CLT for the gradient noise and
\eqref{eq:pr_linearization} then yield the standard covariance
$H^{-1}SH^{-\top}$ without requiring \eqref{eq:weight_clt}.
\end{proof}


\subsection{Derivations for the examples}
\label{app:examples}

\paragraph{Running average.}
Independence and centering of $\varepsilon_s$ give
$\mathbb E[b_tb_t^\top]=Q/t$. Set $k_{T,s}=\sum_{t=s}^Tt^{-1}$.
Exchanging sums yields
\[
\sum_{s=1}^Tk_{T,s}=\sum_{t=1}^T\frac1t\sum_{s=1}^t1=T.
\]
For the squared weights, counting the innovations common to times
$t$ and $u$ gives
\[
\begin{aligned}
\sum_{s=1}^Tk_{T,s}^2
&=\sum_{t=1}^T\sum_{u=1}^T\frac{\min(t,u)}{tu}\\
&=h_T+2\sum_{u=2}^T\frac{u-1}{u}=2T-h_T.
\end{aligned}
\]
This proves \eqref{eq:running_weight_identities}. Since $h_T=O(\log T)$,
all weight conditions of Theorem~\ref{thm:pr} hold and
\eqref{eq:running_pr_covariance} follows.

\paragraph{Recursive representation and product estimates.}
Fix $c>1/2$. Iteration of \eqref{eq:recursive_average} proves
\eqref{eq:recursive_weights}. Write $W_{t,s}=w_{t,s}I$ and
$P_{t,s}=\prod_{j=s+1}^t(1-c/j)$.
Choose a fixed integer $m>2c+1$. Uniformly for $t\ge s\ge m$,
Taylor expansion of the logarithm and a harmonic-sum comparison give
\[
\log P_{t,s}=-c\sum_{j=s+1}^t\frac1j+O\!\left(\sum_{j=s+1}^t\frac1{j^2}\right)
=-c\log(t/s)+O(s^{-1}).
\]
The constants in the $O$ terms depend only on $c$. Hence
\begin{equation}\label{eq:recursive_product_asymptotics}
P_{t,s}=(s/t)^c\bigl(1+O(s^{-1})\bigr),
\qquad
|w_{t,s}|\le C s^{c-1}t^{-c}.
\end{equation}
The finitely many indices $s<m$ satisfy the same upper bound after
increasing $C$: their products factor into a fixed finite prefix and
$P_{t,m}$. Possible zero factors cause no difficulty. The same argument
gives $\|\delta_t\|=O(t^{-c})$. Thus
\[
\left\|\frac1{\sqrt T}\sum_{t=1}^T\delta_t\right\|
\le\begin{cases}
O(T^{1/2-c}),&1/2<c<1,\\
O(T^{-1/2}\log T),&c=1,\\
O(T^{-1/2}),&c>1,
\end{cases}
\]
which tends to zero.

\paragraph{Single-step covariance.}
The initial term is deterministic, so
\[
\mathbb E[b_tb_t^\top]
=\left(\sum_{s=1}^tw_{t,s}^2\right)Q+\delta_t\delta_t^\top.
\]
We show that $t\sum_{s=1}^tw_{t,s}^2\to c^2/(2c-1)$.
For every fixed $\epsilon\in(0,1)$,
\eqref{eq:recursive_product_asymptotics} holds with relative error
$O((\epsilon t)^{-1})$ uniformly for $\epsilon t\le s\le t$.
Consequently, a Riemann sum gives
\[
t\sum_{s=\lceil\epsilon t\rceil}^tw_{t,s}^2
\longrightarrow c^2\int_\epsilon^1x^{2c-2}\,dx.
\]
For the omitted indices, the product bound gives
\[
t\sum_{s<\epsilon t}w_{t,s}^2
\le C t^{1-2c}\sum_{s<\epsilon t}s^{2c-2}
\le C'\epsilon^{2c-1}.
\]
The last bound follows from $2c-2>-1$; an empty sum is zero.
Letting $\epsilon\downarrow0$ yields
\[
t\sum_{s=1}^tw_{t,s}^2\longrightarrow
c^2\int_0^1x^{2c-2}\,dx=\frac{c^2}{2c-1}.
\]
The initial term satisfies $t\|\delta_t\|^2\to0$, proving
\eqref{eq:recursive_single_covariance}. The bounds also give
\[
\sup_{t\ge1}t\,\mathbb E\|b_t\|^2<\infty.
\]
Thus the direct error condition holds with $\beta=1$. The weight and
remainder bounds in Theorem~\ref{thm:convergence} likewise hold for
finite constants.

\paragraph{Cumulative kernel and boundary control.}
Write $K_{T,s}=k_{T,s}I$, where $k_{T,s}=\sum_{t=s}^Tw_{t,s}$.
For $s/T\to x>0$, \eqref{eq:recursive_product_asymptotics} and a
Riemann-sum approximation imply
\[
k_{T,s}
=\sum_{t=s}^T\frac cs(s/t)^c\bigl(1+O(s^{-1})\bigr)
\longrightarrow c x^{c-1}\int_x^1v^{-c}\,dv=k_c(x).
\]
The convergence is uniform when $s/T\in[\epsilon,1]$ for fixed
$\epsilon>0$. To justify passage to the first and second moment
integrals, including near $x=0$, sum the product bound to obtain
\begin{equation}\label{eq:recursive_kernel_envelope}
|k_{T,s}|\le C\begin{cases}
(T/s)^{1-c},&1/2<c<1,\\
1+\log(T/s),&c=1,\\
1,&c>1.
\end{cases}
\end{equation}
Thus the small-index contribution to $T^{-1}\sum_s k_{T,s}^2$ has
limsup bounded by $C\epsilon^{2c-1}$ when $c<1$, by
$C\int_0^\epsilon[1+\log(1/x)]^2\,dx$ when $c=1$, and by $C\epsilon$
when $c>1$. Each bound tends to zero with $\epsilon$.
Cauchy--Schwarz gives the corresponding vanishing bound for the
first moments. Combining these estimates with the uniform interior
convergence proves
\[
\frac1T\sum_{s=1}^Tk_{T,s}\to\int_0^1k_c(x)\,dx,
\qquad
\frac1T\sum_{s=1}^Tk_{T,s}^2\to\int_0^1k_c(x)^2\,dx.
\]
The envelope also gives
\[
\max_{s\le T}\frac{|k_{T,s}|}{\sqrt T}
\le\begin{cases}
O(T^{1/2-c}),&1/2<c<1,\\
O(T^{-1/2}\log T),&c=1,\\
O(T^{-1/2}),&c>1,
\end{cases}
\longrightarrow0.
\]
This verifies the maximal weight condition, including the fixed early
innovations.

\paragraph{Evaluation of the covariance integrals.}
For $c\ne1$, direct integration of \eqref{eq:recursive_kernel} gives
\[
\int_0^1k_c(x)\,dx=\frac c{c-1}\left(1-\frac1c\right)=1,
\]
\[
\int_0^1k_c(x)^2\,dx
=\frac{c^2}{(c-1)^2}\left(1-\frac2c+\frac1{2c-1}\right)
=\frac{2c}{2c-1}.
\]
For $c=1$, the corresponding integrals are
$\int_0^1\log(1/x)\,dx=1$ and
$\int_0^1\log^2(1/x)\,dx=2$. This establishes
\eqref{eq:recursive_weight_limits} in every case. Applying
Theorem~\ref{thm:pr} with $C_0=0$ proves
\eqref{eq:recursive_pr_covariance}.

Finally, differentiating \eqref{eq:gain_objectives} proves the stated
parameter comparison. Since $H^{-1}QH^{-\top}$ is positive semidefinite,
the scalar monotonicity of $f_{\mathrm{PR}}$ implies monotonicity of
the downstream covariance in the positive semidefinite order. If
$Q\ne0$, its trace strictly decreases, whereas the single-step
asymptotic MSE has its unique minimum at $c=1$.


\subsection{Proof of Proposition~\ref{prop:lower_bound}}
\label{app:lower_bound}

\begin{proof}
Independence and centering give
\[
\Gamma_T=\frac QT\sum_{s=1}^Tk_{T,s}^2.
\]
Normalization and exchange of the finite sums imply
\[
\sum_{s=1}^Tk_{T,s}
=\sum_{t=1}^T\sum_{s=1}^tw_{t,s}=T.
\]
Therefore
\[
\frac1T\sum_{s=1}^T(k_{T,s}-1)^2
=\frac1T\sum_{s=1}^Tk_{T,s}^2-1,
\]
which proves \eqref{eq:lower_bound_identity} and sufficiency in
\eqref{eq:attainment_condition}. If $Q\ne0$, taking traces gives
\[
\operatorname{tr}(\Gamma_T-Q)
=\operatorname{tr}(Q)\frac1T\sum_{s=1}^T(k_{T,s}-1)^2,
\]
so the condition is necessary as well. Finally, if the PR covariance is
$H^{-1}(S+\Gamma)H^{-\top}$ with $\Gamma_T\to\Gamma$, closedness of the
positive semidefinite cone gives $\Gamma\succeq Q$. Congruence by
$H^{-1}$ proves \eqref{eq:downstream_lower_bound}.
\end{proof}

\subsection{Proof of Proposition~\ref{prop:attainability}}
\label{app:attainability}

\begin{proof}
Set $\gamma_1=1$. Expanding \eqref{eq:polynomial_gain} gives
\[
w_{t,s}=\gamma_s\prod_{j=s+1}^t(1-\gamma_j),\qquad 1\le s\le t.
\]
Since $0<\gamma_t\le1$, the weights are nonnegative. Their row sum is
one at $t=1$ and remains one by induction, because the next row has
sum $(1-\gamma_t)\cdot1+\gamma_t=1$.

\paragraph{Single-step error.}
To control the deterministic weights without imposing $Q\ne0$, let
$u_s$ be independent centered scalar random variables of variance one,
and define the auxiliary scalar recursion
\[
z_1=u_1,\qquad z_t=(1-\gamma_t)z_{t-1}+\gamma_tu_t.
\]
Then $z_t=\sum_s w_{t,s}u_s$ and
$v_t:=\mathbb E z_t^2=\sum_s w_{t,s}^2$ satisfies
\begin{equation}\label{eq:polynomial_variance_recursion}
v_t=(1-\gamma_t)^2v_{t-1}+\gamma_t^2
\le(1-at^{-\rho})v_{t-1}+a^2t^{-2\rho}.
\end{equation}
For completeness, $v_t\le Ct^{-\rho}$ follows by a direct comparison.
For all sufficiently large $t$,
\[
(t-1)^{-\rho}-t^{-\rho}\le C_1t^{-\rho-1}
\le\frac a2t^{-2\rho},
\]
where the last inequality uses $\rho<1$. If $v_{t-1}\le C(t-1)^{-\rho}$,
then \eqref{eq:polynomial_variance_recursion} implies
\[
v_t\le Ct^{-\rho}
+\left(-\frac{Ca}{2}+a^2\right)t^{-2\rho}
\le Ct^{-\rho}
\]
for $C\ge2a$. Increase $C$ to cover the finite initial interval and
apply induction. Consequently,
\[
\mathbb E\|b_t\|^2=\operatorname{tr}(Q)v_t=O(t^{-\rho}).
\]
The rate is genuinely of order $t^{-\rho}$ when $Q\ne0$.
Indeed, for the last $\lfloor t^\rho\rfloor$ indices $s\le t$, one has
$\gamma_s\ge at^{-\rho}$ and, for sufficiently large $t$,
\[
\prod_{j=s+1}^t(1-\gamma_j)
\ge\exp\!\left(-2\sum_{j=s+1}^t\gamma_j\right)\ge c_*>0,
\]
where $c_*$ is independent of $t,s$. Here $\gamma_j\le1/2$ on that
interval, and its length is $O(t^\rho)$ with $j\asymp t$.
Thus $v_t\ge c_*^2a^2t^{-2\rho}\lfloor t^\rho\rfloor$, proving
$v_t=\Theta(t^{-\rho})$.

\paragraph{Cumulative weights approach one.}
The key estimate follows from summation by parts, without approximating
the individual historical products. Define
$d_t=\gamma_t^{-1}-1$, so $d_1=0$. Rearranging the scalar recursion gives
\[
u_t=z_t+d_t(z_t-z_{t-1}),
\]
where $z_0$ can be set to zero since $d_1=0$. Summing yields
\begin{equation}\label{eq:polynomial_abel_identity}
\sum_{t=1}^T(z_t-u_t)
=-d_Tz_T+\sum_{t=1}^{T-1}(d_{t+1}-d_t)z_t.
\end{equation}
For $t\ge2$, $d_t=a^{-1}t^\rho-1$, so
$d_t=O(t^\rho)$ and $d_{t+1}-d_t=O(t^{\rho-1})$; the first increment is
finite. Using $\|z_t\|_{L^2}=O(t^{-\rho/2})$ and Minkowski's inequality,
\[
\begin{aligned}
\left\|\frac1{\sqrt T}\sum_{t=1}^T(z_t-u_t)\right\|_{L^2}
&\le O(T^{(\rho-1)/2})+O(T^{-1/2})
+\frac C{\sqrt T}\sum_{t=2}^{T-1}t^{\rho/2-1}\\
&=O(T^{(\rho-1)/2}).
\end{aligned}
\]
On the other hand, exchanging sums gives the exact identity
\[
\frac1{\sqrt T}\sum_{t=1}^T(z_t-u_t)
=\frac1{\sqrt T}\sum_{s=1}^T(k_{T,s}-1)u_s.
\]
Independence and unit variance therefore imply
\begin{equation}\label{eq:polynomial_flatness_rate}
\frac1T\sum_{s=1}^T(k_{T,s}-1)^2
=\mathbb E\left|\frac1{\sqrt T}\sum_{t=1}^T(z_t-u_t)\right|^2
=O(T^{\rho-1})\longrightarrow0.
\end{equation}
This is a deterministic weight estimate, valid for any innovation
covariance $Q$. Proposition~\ref{prop:lower_bound} now gives
\[
\Gamma_T=Q\bigl[1+O(T^{\rho-1})\bigr]\longrightarrow Q.
\]

\paragraph{Downstream PR limit.}
Here $W_{t,s}=w_{t,s}I$, $\delta_t=0$, and $\beta=\rho$ in
Theorem~\ref{thm:pr}. Normalization gives
$T^{-1}\sum_sK_{T,s}=I$ exactly, and the covariance limit is $Q$.
Moreover, \eqref{eq:polynomial_flatness_rate} implies
\[
\max_{s\le T}\frac{|k_{T,s}|}{\sqrt T}
\le T^{-1/2}
+\left(\frac1T\sum_{s=1}^T(k_{T,s}-1)^2\right)^{1/2}
\longrightarrow0.
\]
Thus every cumulative-weight condition holds. Since $\rho>1/2$, the
remaining exponent condition is exactly $\rho>2\alpha-1$.
Theorem~\ref{thm:pr} with $C_0=0$ proves \eqref{eq:optimal_pr_covariance}.
\end{proof}