\documentclass[numbers=enddot,12pt,final,onecolumn,notitlepage]{scrartcl}%
\usepackage[headsepline,footsepline,manualmark]{scrlayer-scrpage}
\usepackage[all,cmtip]{xy}
\usepackage{amssymb}
\usepackage{amsmath}
\usepackage{amsthm}
\usepackage{framed}
\usepackage{comment}
\usepackage{color}
\usepackage{hyperref}
\usepackage[sc]{mathpazo}
\usepackage[T1]{fontenc}
\usepackage{tikz}
\usepackage{needspace}
\usepackage{tabls}
\usepackage{wasysym}
\usepackage{easytable}
\usepackage{pythonhighlight}
%TCIDATA{OutputFilter=latex2.dll}
%TCIDATA{Version=5.50.0.2960}
%TCIDATA{LastRevised=Tuesday, January 04, 2022 21:12:23}
%TCIDATA{SuppressPackageManagement}
%TCIDATA{<META NAME="GraphicsSave" CONTENT="32">}
%TCIDATA{<META NAME="SaveForMode" CONTENT="1">}
%TCIDATA{BibliographyScheme=Manual}
%TCIDATA{Language=American English}
%BeginMSIPreambleData
\providecommand{\U}[1]{\protect\rule{.1in}{.1in}}
%EndMSIPreambleData
\usetikzlibrary{arrows.meta}
\usetikzlibrary{chains}
\newcounter{exer}
\newcounter{exera}
\numberwithin{exer}{subsection}
\theoremstyle{definition}
\newtheorem{theo}{Theorem}[subsection]
\newenvironment{theorem}[1][]
{\begin{theo}[#1]\begin{leftbar}}
{\end{leftbar}\end{theo}}
\newtheorem{lem}[theo]{Lemma}
\newenvironment{lemma}[1][]
{\begin{lem}[#1]\begin{leftbar}}
{\end{leftbar}\end{lem}}
\newtheorem{prop}[theo]{Proposition}
\newenvironment{proposition}[1][]
{\begin{prop}[#1]\begin{leftbar}}
{\end{leftbar}\end{prop}}
\newtheorem{defi}[theo]{Definition}
\newenvironment{definition}[1][]
{\begin{defi}[#1]\begin{leftbar}}
{\end{leftbar}\end{defi}}
\newtheorem{remk}[theo]{Remark}
\newenvironment{remark}[1][]
{\begin{remk}[#1]\begin{leftbar}}
{\end{leftbar}\end{remk}}
\newtheorem{coro}[theo]{Corollary}
\newenvironment{corollary}[1][]
{\begin{coro}[#1]\begin{leftbar}}
{\end{leftbar}\end{coro}}
\newtheorem{conv}[theo]{Convention}
\newenvironment{convention}[1][]
{\begin{conv}[#1]\begin{leftbar}}
{\end{leftbar}\end{conv}}
\newtheorem{quest}[theo]{Question}
\newenvironment{question}[1][]
{\begin{quest}[#1]\begin{leftbar}}
{\end{leftbar}\end{quest}}
\newtheorem{warn}[theo]{Warning}
\newenvironment{warning}[1][]
{\begin{warn}[#1]\begin{leftbar}}
{\end{leftbar}\end{warn}}
\newtheorem{conj}[theo]{Conjecture}
\newenvironment{conjecture}[1][]
{\begin{conj}[#1]\begin{leftbar}}
{\end{leftbar}\end{conj}}
\newtheorem{exam}[theo]{Example}
\newenvironment{example}[1][]
{\begin{exam}[#1]\begin{leftbar}}
{\end{leftbar}\end{exam}}
\newtheorem{exmp}[exer]{Exercise}
\newenvironment{exercise}[1][]
{\begin{exmp}[#1]\begin{leftbar}}
{\end{leftbar}\end{exmp}}
\newenvironment{statement}{\begin{quote}}{\end{quote}}
\newenvironment{fineprint}{\medskip \begin{small}}{\end{small} \medskip}
\iffalse
\newenvironment{proof}[1][Proof]{\noindent\textbf{#1.} }{\ \rule{0.5em}{0.5em}}
\newenvironment{question}[1][Question]{\noindent\textbf{#1.} }{\ \rule{0.5em}{0.5em}}
\newenvironment{warning}[1][Warning]{\noindent\textbf{#1.} }{\ \rule{0.5em}{0.5em}}
\newenvironment{teachingnote}[1][Teaching note]{\noindent\textbf{#1.} }{\ \rule{0.5em}{0.5em}}
\fi
\let\sumnonlimits\sum
\let\prodnonlimits\prod
\let\cupnonlimits\bigcup
\let\capnonlimits\bigcap
\renewcommand{\sum}{\sumnonlimits\limits}
\renewcommand{\prod}{\prodnonlimits\limits}
\renewcommand{\bigcup}{\cupnonlimits\limits}
\renewcommand{\bigcap}{\capnonlimits\limits}
\setlength\tablinesep{3pt}
\setlength\arraylinesep{3pt}
\setlength\extrarulesep{3pt}
\voffset=0cm
\hoffset=-0.7cm
\setlength\textheight{22.5cm}
\setlength\textwidth{15.5cm}
\newcommand\arxiv[1]{\href{http://www.arxiv.org/abs/#1}{\texttt{arXiv:#1}}}
\newenvironment{verlong}{}{}
\newenvironment{vershort}{}{}
\newenvironment{noncompile}{}{}
\newenvironment{teachingnote}{}{}
\excludecomment{verlong}
\includecomment{vershort}
\excludecomment{noncompile}
\excludecomment{teachingnote}
\newcommand{\CC}{\mathbb{C}}
\newcommand{\RR}{\mathbb{R}}
\newcommand{\QQ}{\mathbb{Q}}
\newcommand{\NN}{\mathbb{N}}
\newcommand{\ZZ}{\mathbb{Z}}
\newcommand{\KK}{\mathbb{K}}
\newcommand{\id}{\operatorname{id}}
\newcommand{\lcm}{\operatorname{lcm}}
\newcommand{\rev}{\operatorname{rev}}
\newcommand{\powset}[2][]{\ifthenelse{\equal{#2}{}}{\mathcal{P}\left(#1\right)}{\mathcal{P}_{#1}\left(#2\right)}}
\newcommand{\set}[1]{\left\{ #1 \right\}}
\newcommand{\abs}[1]{\left| #1 \right|}
\newcommand{\tup}[1]{\left( #1 \right)}
\newcommand{\ive}[1]{\left[ #1 \right]}
\newcommand{\floor}[1]{\left\lfloor #1 \right\rfloor}
\newcommand{\lf}[2]{#1^{\underline{#2}}}
\newcommand{\underbrack}[2]{\underbrace{#1}_{\substack{#2}}}
\newcommand{\horrule}[1]{\rule{\linewidth}{#1}}
\newcommand{\are}{\ar@{-}}
\newcommand{\nnn}{\nonumber\\}
\newcommand{\sslash}{\mathbin{/\mkern-6mu/}}
\newcommand{\numboxed}[2]{\underbrace{\boxed{#1}}_{\text{box } #2}}
\newcommand{\ig}[2]{\includegraphics[scale=#1]{#2.png}}
\newcommand{\UNFINISHED}{\begin{center} \Huge{\textbf{Unfinished material begins here.}} \end{center} }
\iffalse
\NOEXPAND{\today}{\today}
\NOEXPAND{\sslash}{\sslash}
\NOEXPAND{\numboxed}[2]{\numboxed}
\NOEXPAND{\UNFINISHED}{\UNFINISHED}
\fi
\ihead{Math 504 notes}
\ohead{page \thepage}
\cfoot{\today}
\begin{document}

\title{Math 504: Advanced Linear Algebra}
\author{Hugo Woerdeman and Darij Grinberg\thanks{Drexel University, Korman Center, 15
S 33rd Street, Philadelphia PA, 19104, USA}}
\date{\today\ (unfinished!)}
\maketitle
\tableofcontents

\section*{Preface}

These are lecture notes originally written by Hugo Woerdeman and edited by
myself for the Math 504 (Advanced Linear Algebra) class at Drexel University
in Fall 2021. The website of this class can be found at%
\[
\text{\url{http://www.cip.ifi.lmu.de/~grinberg/t/21fala}\ \ .}%
\]


\textbf{This document is a work in progress.}

\textbf{Please report any errors you find to
\texttt{\href{mailto:darijgrinberg@gmail.com}{darijgrinberg@gmail.com}} .}

\subsection*{What is this?}

This is a second course on linear algebra, meant for (mostly graduate)
students that are already familiar with matrices, determinants and vector
spaces. Much of the prerequisites (but also some of our material, and even
some content that goes beyond our course) is covered by textbooks like
\cite{Heffer20}, \cite{LaNaSc16}, \cite{Taylor20}, \cite{Treil15},
\cite{Strick20}, \cite[Part I]{GalQua20}, \cite{Loehr14}, \cite{Woerde16}%
\footnote{This list is nowhere near complete. (It is biased towards freely
available sources, but even in that category it is probably far from
comprehensive.)}. The text we will follow the closest is \cite{HorJoh13}.

We will freely use the basic theory of complex numbers, including the
Fundamental Theorem of Algebra. See \cite[Chapters 2--3]{LaNaSc16} or
\cite[Chapters 9--10]{Korner20} for an introduction to these matters.

\subsection*{Notations}

\begin{itemize}
\item We let $\mathbb{N}:=\left\{  0,1,2,\ldots\right\}  $.

\item For any $n\in\mathbb{N}$, we let $\left[  n\right]  $ denote the
$n$-element set $\left\{  1,2,\ldots,n\right\}  $.

\item If $\mathbb{F}$ is a field, and $n,m\in\mathbb{N}$, then $\mathbb{F}%
^{n\times m}$ denotes the set (actually, an $\mathbb{F}$-vector space) of all
$n\times m$-matrices over $\mathbb{F}$.

\item If $\mathbb{F}$ is a field, and $n\in\mathbb{N}$, then the space
$\mathbb{F}^{n\times1}$ of all $n\times1$-matrices over $\mathbb{F}$ (that is,
column vectors of size $n$) is also denoted by $\mathbb{F}^{n}$.

\item The $n\times n$ identity matrix is denoted by $I_{n}$ or by $I$ if the
$n$ is clear from the context.

\item The transpose of a matrix $A$ is denoted by $A^{T}$.

\item Zero vectors and zero matrices will be denoted by $0$, no matter what
their sizes or ambient spaces are.

\item If $A$ is an $n\times m$-matrix, and if $i\in\left[  n\right]  $ and
$j\in\left[  m\right]  $, then:

\begin{itemize}
\item we let $A_{i,j}$ denote the $\left(  i,j\right)  $-th entry of $A$ (that
is, the entry of $A$ in the $i$-th row and the $j$-th column);

\item we let $A_{i,\bullet}$ denote the $i$-th row of $A$;

\item we let $A_{\bullet,j}$ denote the $j$-th column of $A$.
\end{itemize}

\item The letter $i$ usually denotes the complex number $\sqrt{-1}$. Sometimes
(e.g. in the bullet point just above) it also stands for something else
(usually an \textbf{i}ndex that is an \textbf{i}nteger). I'll do my best to
avoid the latter meaning when there is any realistic chance that it be
confused for the former.

\item We use the notation $\operatorname*{diag}\left(  \lambda_{1},\lambda
_{2},\ldots,\lambda_{n}\right)  $ for the diagonal matrix with diagonal
entries $\lambda_{1},\lambda_{2},\ldots,\lambda_{n}$.
\end{itemize}

\subsection{Remark on exercises}

Each exercise gives a number of \textquotedblleft experience
points\textquotedblright, which roughly corresponds to its difficulty (with
some adjustment for its relevance). This is the number in the square (like
\fbox{3} or \fbox{5}). The harder or more important the exercise, the larger
is the number in the square. A \fbox{1} is a warm-up question whose solution
you will probably see right after reading; a \fbox{3} typically requires some
thinking or work; a \fbox{5} requires both; higher values tend to involve some
creativity or research.

\subsection{Scribes}

Parts of these notes were scribed by Math 504 students. I thank the following
students for their help:%
\[%
\begin{tabular}
[c]{c|c}%
scribe & sections\\\hline\hline
Hunter Wages & proof of Theorem \ref{thm.schurtri.syl.equivalence}%
\end{tabular}
\]


\newpage

\section{Unitary matrices (\cite[\S 2.1]{HorJoh13})}

In this chapter, $n$ will usually denote a nonnegative integer.

\begin{noncompile}%
\[%
\begin{tabular}
[c]{||l||}\hline\hline
\textbf{Old lecture 1 starts here.}\\\hline\hline
\end{tabular}
\ \
\]

\end{noncompile}

%

\[%
\begin{tabular}
[c]{||l||}\hline\hline
\textbf{Lecture 1 starts here.}\\\hline\hline
\end{tabular}
\ \
\]


\subsection{Inner products}

We recall a basic definition regarding complex numbers:

\begin{definition}
Let $z\in\mathbb{C}$ be a complex number. Then, the \emph{complex conjugate}
of $z$ means the complex number $a-bi$, where $z$ is written in the form
$z=a+bi$ for some $a,b\in\mathbb{R}$. In other words, the complex conjugate of
$z$ is obtained from $z$ by keeping the real part unchanged but flipping the
sign of the imaginary part.

The complex conjugate of $z$ is denoted by $\overline{z}$.
\end{definition}

Complex conjugation is known to preserve all arithmetic operations: i.e., for
any complex numbers $z$ and $w$, we have%
\begin{align*}
\overline{z+w}  &  =\overline{z}+\overline{w}\ \ \ \ \ \ \ \ \ \ \text{and}%
\ \ \ \ \ \ \ \ \ \ \overline{z-w}=\overline{z}-\overline{w}%
\ \ \ \ \ \ \ \ \ \ \text{and}\\
\overline{z\cdot w}  &  =\overline{z}\cdot\overline{w}%
\ \ \ \ \ \ \ \ \ \ \text{and}\ \ \ \ \ \ \ \ \ \ \overline{z/w}=\overline
{z}/\overline{w}.
\end{align*}
Also, a complex number $z$ satisfies $\overline{z}=z$ if and only if
$z\in\mathbb{R}$. Finally, if $z$ is any complex number, then $z\overline
{z}=\left\vert z\right\vert ^{2}$ is a nonnegative real.

\begin{definition}
\label{def.unitary.innerprod.innerprod}For any two vectors $x=\left(
\begin{array}
[c]{c}%
x_{1}\\
x_{2}\\
\vdots\\
x_{n}%
\end{array}
\right)  \in\mathbb{C}^{n}$ and $y=\left(
\begin{array}
[c]{c}%
y_{1}\\
y_{2}\\
\vdots\\
y_{n}%
\end{array}
\right)  \in\mathbb{C}^{n}$, we define the scalar%
\begin{equation}
\left\langle x,y\right\rangle :=x_{1}\overline{y_{1}}+x_{2}\overline{y_{2}%
}+\cdots+x_{n}\overline{y_{n}}\in\mathbb{C}
\label{eq.def.unitary.innerprod.innerprod.def}%
\end{equation}
(where $\overline{z}$ denotes the complex conjugate of a $z\in\mathbb{C}$).
This scalar $\left\langle x,y\right\rangle $ is called the \emph{inner
product} (or \emph{dot product}) of $x$ and $y$.
\end{definition}

\begin{example}
If $x=\left(
\begin{array}
[c]{c}%
1+i\\
2+3i
\end{array}
\right)  \in\mathbb{C}^{2}$ and $y=\left(
\begin{array}
[c]{c}%
-i\\
4+i
\end{array}
\right)  \in\mathbb{C}^{2}$, then%
\begin{align*}
\left\langle x,y\right\rangle  &  =\left(  1+i\right)  \left(  \overline
{-i}\right)  +\left(  2+3i\right)  \left(  \overline{4+i}\right) \\
&  =\left(  1+i\right)  i+\left(  2+3i\right)  \left(  4-i\right) \\
&  =i-1+8-2i+12i+3=10+11i.
\end{align*}

\end{example}

Some warnings about the literature are in order:

\begin{itemize}
\item Some authors (e.g., Treil in \cite{Treil15}) write $\left(  x,y\right)
$ instead of $\left\langle x,y\right\rangle $ for the inner product of $x$ and
$y$. This can be rather confusing, since $\left(  x,y\right)  $ also means the
pair consisting of $x$ and $y$.

\item The notation $\left\langle x,y\right\rangle $, too, can mean something
different in certain texts (namely, the span of $x$ and $y$); however, it
won't have this second meaning in our course.

\item If I am not mistaken, Definition \ref{def.unitary.innerprod.innerprod}
is also not the only game in town. Some authors follow a competing standard,
which causes their $\left\langle x,y\right\rangle $ to be what we would denote
$\left\langle y,x\right\rangle $.

\item Finally, the word \textquotedblleft dot product\textquotedblright\ often
means the analogue of $\left\langle x,y\right\rangle $ that does not use
complex conjugation (i.e., that replaces
(\ref{eq.def.unitary.innerprod.innerprod.def}) by $\left\langle
x,y\right\rangle :=x_{1}y_{1}+x_{2}y_{2}+\cdots+x_{n}y_{n}$). This convention
is used mostly in abstract algebra, where complex conjugation is not
considered intrinsic to the number system. We will not use this convention.
For vectors with real entries, the distinction disappears, since
$\overline{\lambda}=\lambda$ for any $\lambda\in\mathbb{R}$.
\end{itemize}

\begin{definition}
\label{def.unitary.innerprod.ystar}For any column vector $y=\left(
\begin{array}
[c]{c}%
y_{1}\\
y_{2}\\
\vdots\\
y_{n}%
\end{array}
\right)  \in\mathbb{C}^{n}$, we define the row vector
\[
y^{\ast}:=\left(
\begin{array}
[c]{cccc}%
\overline{y_{1}} & \overline{y_{2}} & \cdots & \overline{y_{n}}%
\end{array}
\right)  \in\mathbb{C}^{1\times n}.
\]

\end{definition}

\begin{proposition}
\label{prop.unitary.innerprod.props}Let $x\in\mathbb{C}^{n}$ and
$y\in\mathbb{C}^{n}$. Then: \medskip

\textbf{(a)} We have $\left\langle x,y\right\rangle =y^{\ast}x$. \medskip

\textbf{(b)} We have $\left\langle x,y\right\rangle =\overline{\left\langle
y,x\right\rangle }$. \medskip

\textbf{(c)} We have $\left\langle x+x^{\prime},y\right\rangle =\left\langle
x,y\right\rangle +\left\langle x^{\prime},y\right\rangle $ for any $x^{\prime
}\in\mathbb{C}^{n}$. \medskip

\textbf{(d)} We have $\left\langle x,y+y^{\prime}\right\rangle =\left\langle
x,y\right\rangle +\left\langle x,y^{\prime}\right\rangle $ for any $y^{\prime
}\in\mathbb{C}^{n}$. \medskip

\textbf{(e)} We have $\left\langle \lambda x,y\right\rangle =\lambda
\left\langle x,y\right\rangle $ for any $\lambda\in\mathbb{C}$. \medskip

\textbf{(f)} We have $\left\langle x,\lambda y\right\rangle =\overline
{\lambda}\left\langle x,y\right\rangle $ for any $\lambda\in\mathbb{C}$.
\medskip

\textbf{(g)} We have $\left\langle x-x^{\prime},y\right\rangle =\left\langle
x,y\right\rangle -\left\langle x^{\prime},y\right\rangle $ for any $x^{\prime
}\in\mathbb{C}^{n}$. \medskip

\textbf{(h)} We have $\left\langle x,y-y^{\prime}\right\rangle =\left\langle
x,y\right\rangle -\left\langle x,y^{\prime}\right\rangle $ for any $y^{\prime
}\in\mathbb{C}^{n}$. \medskip

\textbf{(i)} We have $\left\langle \sum_{i=1}^{k}\lambda_{i}x_{i}%
,y\right\rangle =\sum_{i=1}^{k}\lambda_{i}\left\langle x_{i},y\right\rangle $
for any $k\in\mathbb{N}$, any $x_{1},x_{2},\ldots,x_{k}\in\mathbb{C}^{n}$ and
any $\lambda_{1},\lambda_{2},\ldots,\lambda_{k}\in\mathbb{C}$. \medskip

\textbf{(j)} We have $\left\langle x,\sum_{i=1}^{k}\lambda_{i}y_{i}%
\right\rangle =\sum_{i=1}^{k}\overline{\lambda_{i}}\left\langle x,y_{i}%
\right\rangle $ for any $k\in\mathbb{N}$, any $y_{1},y_{2},\ldots,y_{k}%
\in\mathbb{C}^{n}$ and any $\lambda_{1},\lambda_{2},\ldots,\lambda_{k}%
\in\mathbb{C}$.
\end{proposition}

\begin{proof}
Parts \textbf{(a)} till \textbf{(h)} are straightforward computations using
Definition \ref{def.unitary.innerprod.innerprod}, since

\begin{itemize}
\item the multiplication in $\mathbb{C}$ is commutative;

\item we have $\overline{\overline{z}}=z$ for any $z\in\mathbb{C}$.
\end{itemize}

Parts \textbf{(i)} and \textbf{(j)} follow from parts \textbf{(c)},
\textbf{(d)}, \textbf{(e)} and \textbf{(f)} by induction on $k$.
\end{proof}

\begin{proposition}
\label{prop.unitary.innerprod.pos}Let $x\in\mathbb{C}^{n}$. Then: \medskip

\textbf{(a)} The number $\left\langle x,x\right\rangle $ is a nonnegative
real. \medskip

\textbf{(b)} We have $\left\langle x,x\right\rangle >0$ whenever $x\neq0$.
\end{proposition}

\begin{proof}
Write $x$ as $x=\left(
\begin{array}
[c]{cccc}%
x_{1} & x_{2} & \cdots & x_{n}%
\end{array}
\right)  ^{T}$. Then, the definition of $\left\langle x,x\right\rangle $
yields%
\begin{align}
\left\langle x,x\right\rangle  &  =x_{1}\overline{x_{1}}+x_{2}\overline{x_{2}%
}+\cdots+x_{n}\overline{x_{n}}\nonumber\\
&  =\left\vert x_{1}\right\vert ^{2}+\left\vert x_{2}\right\vert ^{2}%
+\cdots+\left\vert x_{n}\right\vert ^{2},
\label{pf.prop.unitary.innerprod.pos.1}%
\end{align}
since any complex number $z$ satisfies $z\overline{z}=\left\vert z\right\vert
^{2}$. Since all the absolute values $\left\vert x_{1}\right\vert ,\left\vert
x_{2}\right\vert ,\ldots,\left\vert x_{n}\right\vert $ are real, this yields
immediately that $\left\langle x,x\right\rangle $ is a nonnegative real. Thus,
Proposition \ref{prop.unitary.innerprod.pos} \textbf{(a)} is proved. \medskip

\textbf{(b)} Assume that $x\neq0$. Thus, at least one $i\in\left[  n\right]  $
satisfies $x_{i}\neq0$ and therefore $\left\vert x_{i}\right\vert ^{2}>0$.
This entails $\left\langle x,x\right\rangle =\left\vert x_{1}\right\vert
^{2}+\left\vert x_{2}\right\vert ^{2}+\cdots+\left\vert x_{n}\right\vert
^{2}>0$ (because a sum of nonnegative reals that has at least one positive
addend is always $>0$). In view of (\ref{pf.prop.unitary.innerprod.pos.1}),
this rewrites as $\left\langle x,x\right\rangle >0$. This proves Proposition
\ref{prop.unitary.innerprod.pos} \textbf{(b)}.
\end{proof}

\begin{definition}
\label{def.unitary.innerprod.len}Let $x\in\mathbb{C}^{n}$. We define the
\emph{length} of $x$ to be the nonnegative real number
\[
\left\vert \left\vert x\right\vert \right\vert :=\sqrt{\left\langle
x,x\right\rangle }.
\]
This is well-defined, since Proposition \ref{prop.unitary.innerprod.pos}
\textbf{(a)} says that $\left\langle x,x\right\rangle $ is a nonnegative real.
\end{definition}

\begin{example}
If $x=\left(
\begin{array}
[c]{c}%
1+i\\
3-2i
\end{array}
\right)  \in\mathbb{C}^{2}$, then%
\begin{align*}
\left\langle x,x\right\rangle  &  =\left(  1+i\right)  \left(  \overline
{1+i}\right)  +\left(  3-2i\right)  \left(  \overline{3+2i}\right)  =\left(
1+i\right)  \left(  1-i\right)  +\left(  3-2i\right)  \left(  3+2i\right) \\
&  =1+1+9+4=15
\end{align*}
and thus $\left\vert \left\vert x\right\vert \right\vert =\sqrt{\left\langle
x,x\right\rangle }=\sqrt{15}$.
\end{example}

The length $\left\vert \left\vert x\right\vert \right\vert $ of a vector
$x\in\mathbb{C}^{n}$ is sometimes also called the \emph{norm} of $x$ (but
beware that other things are called \textquotedblleft norms\textquotedblright%
\ as well).

A vector $x\in\mathbb{C}^{n}$ has zero length if and only if it is $0$:

\begin{proposition}
Let $x\in\mathbb{C}^{n}$. Then, $\left\vert \left\vert x\right\vert
\right\vert =0$ if and only if $x=0$.
\end{proposition}

\begin{proof}
The \textquotedblleft if\textquotedblright\ part follows from $\left\vert
\left\vert 0\right\vert \right\vert =0$, which is obvious. To prove the
\textquotedblleft only if\textquotedblright\ part, we assume that $\left\vert
\left\vert x\right\vert \right\vert =0$. Thus, $\left\langle x,x\right\rangle
=0$ (since $\left\vert \left\vert x\right\vert \right\vert =\sqrt{\left\langle
x,x\right\rangle }$). However, if we had $x\neq0$, then Proposition
\ref{prop.unitary.innerprod.pos} \textbf{(b)} would yield $\left\langle
x,x\right\rangle >0$, which would contradict $\left\langle x,x\right\rangle
=0$. Thus, we cannot have $x\neq0$. Hence, $x=0$. Thus, the \textquotedblleft
only if\textquotedblright\ part is proven.
\end{proof}

\begin{proposition}
For any $\lambda\in\mathbb{C}$ and $x\in\mathbb{C}^{n}$, we have $\left\vert
\left\vert \lambda x\right\vert \right\vert =\left\vert \lambda\right\vert
\cdot\left\vert \left\vert x\right\vert \right\vert $.
\end{proposition}

\begin{proof}
Straightforward.
\end{proof}

\begin{exercise}
\label{exe.unitary.innerprod.x+y}\fbox{3} Let $x\in\mathbb{C}^{n}$ and
$y\in\mathbb{C}^{n}$. Prove that
\[
\left\vert \left\vert x+y\right\vert \right\vert ^{2}-\left\vert \left\vert
x\right\vert \right\vert ^{2}-\left\vert \left\vert y\right\vert \right\vert
^{2}=\left\langle x,y\right\rangle +\left\langle y,x\right\rangle
=2\cdot\operatorname*{Re}\left\langle x,y\right\rangle .
\]
Here, $\operatorname*{Re}z$ denotes the real part of any complex number $z$.
\end{exercise}

One of the most famous properties of the inner product is the
\emph{Cauchy--Schwarz inequality} (see \cite{Steele04} for various applications):

\begin{theorem}
[Cauchy--Schwarz inequality]\label{thm.unitary.innerprod.cs}Let $x\in
\mathbb{C}^{n}$ and $y\in\mathbb{C}^{n}$ be two vectors. Then: \medskip

\textbf{(a)} The inequality%
\[
\left\vert \left\vert x\right\vert \right\vert \cdot\left\vert \left\vert
y\right\vert \right\vert \geq\left\vert \left\langle x,y\right\rangle
\right\vert
\]
holds. \medskip

\textbf{(b)} This inequality becomes an equality if and only if the pair
$\left(  x,y\right)  $ of vectors is linearly dependent.
\end{theorem}

\begin{proof}
[Proof of Theorem \ref{thm.unitary.innerprod.cs}.]If $x=0$, then Theorem
\ref{thm.unitary.innerprod.cs} is obvious (because the inequality in part
\textbf{(a)} simplifies to $0\geq0$, and since the pair $\left(  0,y\right)  $
of vectors is always linearly dependent). Hence, for the rest of this proof,
we WLOG assume that $x\neq0$.

Thus, Proposition \ref{prop.unitary.innerprod.pos} \textbf{(a)} yields that
$\left\langle x,x\right\rangle $ is a nonnegative real, and Proposition
\ref{prop.unitary.innerprod.pos} \textbf{(b)} yields $\left\langle
x,x\right\rangle >0$. Let $a:=\left\langle x,x\right\rangle $. Then,
$a=\left\langle x,x\right\rangle >0$. Furthermore, let $b:=\left\langle
y,x\right\rangle \in\mathbb{C}$. Thus, $\overline{b}=\overline{\left\langle
y,x\right\rangle }=\left\langle x,y\right\rangle $ (by Proposition
\ref{prop.unitary.innerprod.props} \textbf{(b)}).

Now, Proposition \ref{prop.unitary.innerprod.pos} \textbf{(a)} (applied to
$bx-ay$ instead of $x$) yields that
\begin{equation}
\left\langle bx-ay,bx-ay\right\rangle \geq0.
\label{pf.thm.unitary.innerprod.cs.main}%
\end{equation}
Since%
\begin{align*}
&  \left\langle bx-ay,bx-ay\right\rangle \\
&  =\underbrace{\left\langle bx,bx-ay\right\rangle }_{\substack{=\left\langle
bx,bx\right\rangle -\left\langle bx,ay\right\rangle \\\text{(by Proposition
\ref{prop.unitary.innerprod.props} \textbf{(h)})}}}-\underbrace{\left\langle
ay,bx-ay\right\rangle }_{\substack{=\left\langle ay,bx\right\rangle
-\left\langle ay,ay\right\rangle \\\text{(by Proposition
\ref{prop.unitary.innerprod.props} \textbf{(h)})}}}\ \ \ \ \ \ \ \ \ \ \left(
\text{by Proposition \ref{prop.unitary.innerprod.props} \textbf{(g)}}\right)
\\
&  =\left\langle bx,bx\right\rangle -\left\langle bx,ay\right\rangle -\left(
\left\langle ay,bx\right\rangle -\left\langle ay,ay\right\rangle \right) \\
&  =\underbrace{\left\langle bx,bx\right\rangle }_{\substack{=b\left\langle
x,bx\right\rangle \\\text{(by Proposition \ref{prop.unitary.innerprod.props}
\textbf{(e)})}}}+\underbrace{\left\langle ay,ay\right\rangle }%
_{\substack{=a\left\langle y,ay\right\rangle \\\text{(by Proposition
\ref{prop.unitary.innerprod.props} \textbf{(e)})}}}\\
&  \ \ \ \ \ \ \ \ \ \ -\underbrace{\left\langle bx,ay\right\rangle
}_{\substack{=b\left\langle x,ay\right\rangle \\\text{(by Proposition
\ref{prop.unitary.innerprod.props} \textbf{(e)})}}}-\underbrace{\left\langle
ay,bx\right\rangle }_{\substack{=a\left\langle y,bx\right\rangle \\\text{(by
Proposition \ref{prop.unitary.innerprod.props} \textbf{(e)})}}}\\
&  =b\underbrace{\left\langle x,bx\right\rangle }_{\substack{=\overline
{b}\left\langle x,x\right\rangle \\\text{(by Proposition
\ref{prop.unitary.innerprod.props} \textbf{(f)})}}}+a\underbrace{\left\langle
y,ay\right\rangle }_{\substack{=\overline{a}\left\langle y,y\right\rangle
\\\text{(by Proposition \ref{prop.unitary.innerprod.props} \textbf{(f)})}}}\\
&  \ \ \ \ \ \ \ \ \ \ -b\underbrace{\left\langle x,ay\right\rangle
}_{\substack{=\overline{a}\left\langle x,y\right\rangle \\\text{(by
Proposition \ref{prop.unitary.innerprod.props} \textbf{(f)})}}%
}-a\underbrace{\left\langle y,bx\right\rangle }_{\substack{=\overline
{b}\left\langle y,x\right\rangle \\\text{(by Proposition
\ref{prop.unitary.innerprod.props} \textbf{(f)})}}}\\
&  =b\overline{b}\underbrace{\left\langle x,x\right\rangle }_{=a}%
+a\underbrace{\overline{a}}_{\substack{=a\\\text{(since }a\in\mathbb{R}%
\text{)}}}\left\langle y,y\right\rangle -b\underbrace{\overline{a}%
}_{\substack{=a\\\text{(since }a\in\mathbb{R}\text{)}}%
}\underbrace{\left\langle x,y\right\rangle }_{=\overline{b}}-a\overline
{b}\underbrace{\left\langle y,x\right\rangle }_{=b}\\
&  =b\overline{b}a+\underbrace{aa}_{=a^{2}}\left\langle y,y\right\rangle
-\underbrace{ba}_{=ab}\overline{b}-\underbrace{a\overline{b}b}_{=b\overline
{b}a}\\
&  =b\overline{b}a+a^{2}\left\langle y,y\right\rangle -ab\overline
{b}-b\overline{b}a=a^{2}\left\langle y,y\right\rangle -ab\overline{b}=a\left(
a\left\langle y,y\right\rangle -b\overline{b}\right)  ,
\end{align*}
we can rewrite this as
\[
a\left(  a\left\langle y,y\right\rangle -b\overline{b}\right)  \geq0.
\]
We can divide both sides of this inequality by $a$ (since $a>0$). Thus, we
obtain
\[
a\left\langle y,y\right\rangle -b\overline{b}\geq0.
\]
In other words,%
\[
a\left\langle y,y\right\rangle \geq b\overline{b}.
\]
In view of
\[
a=\left\langle x,x\right\rangle =\left\vert \left\vert x\right\vert
\right\vert ^{2}\ \ \ \ \ \ \ \ \ \ \left(  \text{since }\left\vert \left\vert
x\right\vert \right\vert =\sqrt{\left\langle x,x\right\rangle }\text{ (by the
definition of }\left\vert \left\vert x\right\vert \right\vert \text{)}\right)
\]
and%
\[
\left\langle y,y\right\rangle =\left\vert \left\vert y\right\vert \right\vert
^{2}\ \ \ \ \ \ \ \ \ \ \left(  \text{since }\left\vert \left\vert
y\right\vert \right\vert =\sqrt{\left\langle y,y\right\rangle }\text{ (by the
definition of }\left\vert \left\vert y\right\vert \right\vert \text{)}\right)
\]
and%
\[
b\overline{b}=\overline{b}\underbrace{b}_{=\overline{\overline{b}}}%
=\overline{b}\overline{\overline{b}}=\left\vert \overline{b}\right\vert
^{2}\ \ \ \ \ \ \ \ \ \ \left(  \text{because }z\overline{z}=\left\vert
z\right\vert ^{2}\text{ for any }z\in\mathbb{C}\right)  ,
\]
we can rewrite this as $\left\vert \left\vert x\right\vert \right\vert
^{2}\left\vert \left\vert y\right\vert \right\vert ^{2}\geq\left\vert
\overline{b}\right\vert ^{2}$. Since $\left\vert \left\vert x\right\vert
\right\vert $ and $\left\vert \left\vert y\right\vert \right\vert $ and
$\left\vert \overline{b}\right\vert $ are nonnegative reals, we can take
square roots on both sides of this inequality, and obtain $\left\vert
\left\vert x\right\vert \right\vert \cdot\left\vert \left\vert y\right\vert
\right\vert \geq\left\vert \overline{b}\right\vert $. In other words,
$\left\vert \left\vert x\right\vert \right\vert \cdot\left\vert \left\vert
y\right\vert \right\vert \geq\left\vert \left\langle x,y\right\rangle
\right\vert $ (since $\overline{b}=\left\langle x,y\right\rangle $). This
proves Theorem \ref{thm.unitary.innerprod.cs} \textbf{(a)}. \medskip

\textbf{(b)} Our above proof of the inequality $\left\vert \left\vert
x\right\vert \right\vert \cdot\left\vert \left\vert y\right\vert \right\vert
\geq\left\vert \left\langle x,y\right\rangle \right\vert $ shows that this
inequality can only become an equality if $\left\langle
bx-ay,bx-ay\right\rangle =0$ (since it was obtained by a chain of reversible
transformations from the inequality (\ref{pf.thm.unitary.innerprod.cs.main})).
But this happens if and only if $bx-ay=0$ (since Proposition
\ref{prop.unitary.innerprod.pos} \textbf{(b)} shows that $\left\langle
bx-ay,bx-ay\right\rangle >0$ in any other case). In turn, $bx-ay=0$ entails
that the pair $\left(  x,y\right)  $ is linearly dependent (since $a>0$).
Thus, the inequality $\left\vert \left\vert x\right\vert \right\vert
\cdot\left\vert \left\vert y\right\vert \right\vert \geq\left\vert
\left\langle x,y\right\rangle \right\vert $ can only become an equality if the
pair $\left(  x,y\right)  $ is linearly dependent. Conversely, it is easy to
see that if the pair $\left(  x,y\right)  $ is linearly dependent, then the
inequality $\left\vert \left\vert x\right\vert \right\vert \cdot\left\vert
\left\vert y\right\vert \right\vert \geq\left\vert \left\langle
x,y\right\rangle \right\vert $ indeed becomes an equality (because in light of
$x\neq0$, the linear dependence of the pair $\left(  x,y\right)  $ yields that
$y=\lambda x$ for some $\lambda\in\mathbb{C}$). Thus, Theorem
\ref{thm.unitary.innerprod.cs} \textbf{(b)} is proven.
\end{proof}

Using Theorem \ref{thm.unitary.innerprod.cs} and Exercise
\ref{exe.unitary.innerprod.x+y}, we can easily obtain the following:

\begin{theorem}
[triangle inequality]\label{thm.unitary.innerprod.norm-tria} Let
$x\in\mathbb{C}^{n}$ and $y\in\mathbb{C}^{n}$. Then: \medskip

\textbf{(a)} The inequality $\left\vert \left\vert x\right\vert \right\vert
+\left\vert \left\vert y\right\vert \right\vert \geq\left\vert \left\vert
x+y\right\vert \right\vert $ holds. \medskip

\textbf{(b)} This inequality becomes an equality if and only if we have $y=0$
or $x=\lambda y$ for some nonnegative real $\lambda$.
\end{theorem}

\begin{exercise}
\label{exe.unitary.innerprod.norm-tria}\fbox{3} Prove Theorem
\ref{thm.unitary.innerprod.norm-tria}.
\end{exercise}

Theorem \ref{thm.unitary.innerprod.norm-tria} \textbf{(a)} is the reason why
the map $\mathbb{C}^{n}\rightarrow\mathbb{R},\ x\mapsto\left\vert \left\vert
x\right\vert \right\vert $ is called a \textquotedblleft
norm\textquotedblright.

\subsection{Orthogonality and orthonormality}

We shall now define orthogonality first for two vectors, then for any tuple of vectors.

\begin{definition}
\label{def.unitary.innerprod.orth}Let $x\in\mathbb{C}^{n}$ and $y\in
\mathbb{C}^{n}$ be two vectors. We say that $x$ is \emph{orthogonal} to $y$ if
and only if $\left\langle x,y\right\rangle =0$. The shorthand notation for
this is \textquotedblleft$x\perp y$\textquotedblright.
\end{definition}

The relation $\perp$ is symmetric:

\begin{proposition}
\label{prop.unitary.innerprod.orth-symm}Let $x\in\mathbb{C}^{n}$ and
$y\in\mathbb{C}^{n}$ be two vectors. Then, $x\perp y$ holds if and only if
$y\perp x$.
\end{proposition}

\begin{proof}
Follows from Proposition \ref{prop.unitary.innerprod.props} \textbf{(b)}.
\end{proof}

\begin{definition}
\label{def.unitary.innerprod.orth-n}Let $\left(  u_{1},u_{2},\ldots
,u_{k}\right)  $ be a tuple of vectors in $\mathbb{C}^{n}$. Then: \medskip

\textbf{(a)} We say that the tuple $\left(  u_{1},u_{2},\ldots,u_{k}\right)  $
is \emph{orthogonal} if we have
\[
u_{p}\perp u_{q}\ \ \ \ \ \ \ \ \ \ \text{whenever }p\neq q.
\]


\textbf{(b)} We say that the tuple $\left(  u_{1},u_{2},\ldots,u_{k}\right)  $
is \emph{orthonormal} if it is orthogonal \textbf{and} satisfies%
\[
\left\vert \left\vert u_{1}\right\vert \right\vert =\left\vert \left\vert
u_{2}\right\vert \right\vert =\cdots=\left\vert \left\vert u_{k}\right\vert
\right\vert =1.
\]


\textbf{(c)} We note that the orthogonality and the orthonormality of a tuple
are preserved when the entries of the tuple are permuted. Thus, we can extend
both notions (\textquotedblleft orthogonal\textquotedblright\ and
\textquotedblleft orthonormal\textquotedblright) to finite sets of vectors in
$\mathbb{C}^{n}$: A set $\left\{  u_{1},u_{2},\ldots,u_{k}\right\}  $ of
vectors in $\mathbb{C}^{n}$ (with $u_{1},u_{2},\ldots,u_{k}$ being distinct)
is said to be \emph{orthogonal} (or \emph{orthonormal}, respectively) if and
only if the tuple $\left(  u_{1},u_{2},\ldots,u_{k}\right)  $ is orthogonal
(resp., orthonormal). \medskip

\textbf{(d)} Sometimes, we (sloppily) say \textquotedblleft the vectors
$u_{1},u_{2},\ldots,u_{k}$ are orthogonal\textquotedblright\ when we mean
\textquotedblleft the tuple $\left(  u_{1},u_{2},\ldots,u_{k}\right)  $ is
orthogonal\textquotedblright. The same applies to \textquotedblleft
orthonormal\textquotedblright.
\end{definition}

\begin{example}
\textbf{(a)} The tuple $\left(  \left(
\begin{array}
[c]{c}%
1\\
0\\
0
\end{array}
\right)  ,\left(
\begin{array}
[c]{c}%
0\\
1\\
0
\end{array}
\right)  ,\left(
\begin{array}
[c]{c}%
0\\
0\\
1
\end{array}
\right)  \right)  $ of vectors in $\mathbb{C}^{3}$ is orthonormal. It is also
a basis of $\mathbb{C}^{3}$, and known as the \emph{standard basis}. \medskip

\textbf{(b)} More generally: Let $n\in\mathbb{N}$. Let $e_{1},e_{2}%
,\ldots,e_{n}\in\mathbb{C}^{n}$ be the vectors defined by%
\[
e_{i}=\underbrace{\left(
\begin{array}
[c]{ccccccccc}%
0 & 0 & \cdots & 0 & 1 & 0 & 0 & \cdots & 0
\end{array}
\right)  ^{T}}_{\substack{\text{the }1\text{ is in the }i\text{-th
position;}\\\text{all other entries are }0}}.
\]
Then, $\left(  e_{1},e_{2},\ldots,e_{n}\right)  $ is an orthonormal basis of
$\mathbb{C}^{n}$, and is known as the \emph{standard basis} of $\mathbb{C}%
^{n}$. \medskip

\textbf{(c)} The pair $\left(  \left(
\begin{array}
[c]{c}%
1\\
-i\\
2
\end{array}
\right)  ,\left(
\begin{array}
[c]{c}%
0\\
2i\\
1
\end{array}
\right)  \right)  $ of vectors in $\mathbb{C}^{3}$ is orthogonal (but not
orthonormal). Indeed,%
\[
\left\langle \left(
\begin{array}
[c]{c}%
1\\
-i\\
2
\end{array}
\right)  ,\left(
\begin{array}
[c]{c}%
0\\
2i\\
1
\end{array}
\right)  \right\rangle =1\cdot\overline{0}+\left(  -i\right)  \cdot
\overline{2i}+2\cdot\overline{1}=0-2+2=0.
\]


\textbf{(d)} The pair $\left(  \dfrac{1}{\sqrt{6}}\left(
\begin{array}
[c]{c}%
1\\
-i\\
2
\end{array}
\right)  ,\dfrac{1}{\sqrt{5}}\left(
\begin{array}
[c]{c}%
0\\
2i\\
1
\end{array}
\right)  \right)  $ of vectors in $\mathbb{C}^{3}$ is orthonormal. (This is
just the previous pair, with each vector scaled so that its length becomes $1$.)
\end{example}

\begin{proposition}
\label{prop.unitary.innerprod.orth-norm}Let $\left(  u_{1},u_{2},\ldots
,u_{k}\right)  $ be an orthogonal tuple of nonzero vectors in $\mathbb{C}^{n}%
$. Then, the tuple%
\[
\left(  \dfrac{1}{\left\vert \left\vert u_{1}\right\vert \right\vert }%
u_{1},\ \ \dfrac{1}{\left\vert \left\vert u_{2}\right\vert \right\vert }%
u_{2},\ \ \ldots,\ \ \dfrac{1}{\left\vert \left\vert u_{k}\right\vert
\right\vert }u_{k}\right)
\]
is orthonormal.
\end{proposition}

\begin{proof}
Straightforward. (Observe that any two orthogonal vectors remain orthogonal
when they are scaled by scalars.)
\end{proof}

\begin{proposition}
\label{prop.unitary.orthog.indep}Any orthogonal tuple of nonzero vectors in
$\mathbb{C}^{n}$ is linearly independent.
\end{proposition}

\begin{proof}
Let $\left(  u_{1},u_{2},\ldots,u_{k}\right)  $ be an orthogonal tuple of
nonzero vectors in $\mathbb{C}^{n}$. We must prove that it is linearly independent.

Indeed, for any $i\in\left[  k\right]  $ and any $\lambda_{1},\lambda
_{2},\ldots,\lambda_{k}\in\mathbb{C}$, we have
\begin{align}
&  \left\langle \lambda_{1}u_{1}+\lambda_{2}u_{2}+\cdots+\lambda_{k}%
u_{k},u_{i}\right\rangle \nonumber\\
&  =\lambda_{1}\left\langle u_{1},u_{i}\right\rangle +\lambda_{2}\left\langle
u_{2},u_{i}\right\rangle +\cdots+\lambda_{k}\left\langle u_{k},u_{i}%
\right\rangle \nonumber\\
&  \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \left(  \text{by parts \textbf{(c)}
and \textbf{(e)} of Proposition \ref{prop.unitary.innerprod.props}}\right)
\nonumber\\
&  =\lambda_{i}\left\langle u_{i},u_{i}\right\rangle +\sum_{\substack{j\in
\left[  k\right]  ;\\j\neq i}}\lambda_{j}\underbrace{\left\langle u_{j}%
,u_{i}\right\rangle }_{\substack{=0\\\text{(since }u_{j}\perp u_{i}%
\\\text{(because }\left(  u_{1},u_{2},\ldots,u_{k}\right)  \text{
is}\\\text{an orthogonal tuple))}}}\nonumber\\
&  =\lambda_{i}\left\langle u_{i},u_{i}\right\rangle .
\label{pf.prop.unitary.orthog.indep.1}%
\end{align}


For any $i\in\left[  k\right]  $, we have $u_{i}\neq0$ (since $\left(
u_{1},u_{2},\ldots,u_{k}\right)  $ is a tuple of nonzero vectors) and thus
\begin{equation}
\left\langle u_{i},u_{i}\right\rangle >0
\label{pf.prop.unitary.orthog.indep.pos}%
\end{equation}
(by Proposition \ref{prop.unitary.innerprod.pos} \textbf{(b)}, applied to
$x=u_{i}$).

Now, let $\lambda_{1},\lambda_{2},\ldots,\lambda_{k}\in\mathbb{C}$ be such
that $\lambda_{1}u_{1}+\lambda_{2}u_{2}+\cdots+\lambda_{k}u_{k}=0$. Then, for
each $i\in\left[  k\right]  $, we have%
\begin{align*}
\lambda_{i}\left\langle u_{i},u_{i}\right\rangle  &  =\left\langle
\underbrace{\lambda_{1}u_{1}+\lambda_{2}u_{2}+\cdots+\lambda_{k}u_{k}}%
_{=0},u_{i}\right\rangle \ \ \ \ \ \ \ \ \ \ \left(  \text{by
(\ref{pf.prop.unitary.orthog.indep.1})}\right) \\
&  =\left\langle 0,u_{i}\right\rangle =0
\end{align*}
and therefore $\lambda_{i}=0$ (indeed, we can divide by $\left\langle
u_{i},u_{i}\right\rangle $, because of (\ref{pf.prop.unitary.orthog.indep.pos})).

Forget that we fixed $\lambda_{1},\lambda_{2},\ldots,\lambda_{k}$. We thus
have shown that if $\lambda_{1},\lambda_{2},\ldots,\lambda_{k}\in\mathbb{C}$
are such that $\lambda_{1}u_{1}+\lambda_{2}u_{2}+\cdots+\lambda_{k}u_{k}=0$,
then we have $\lambda_{i}=0$ for each $i\in\left[  k\right]  $. In other
words, $\left(  u_{1},u_{2},\ldots,u_{k}\right)  $ is linearly independent.
This proves Proposition \ref{prop.unitary.orthog.indep}.
\end{proof}

The following simple lemma will be used further below:

\begin{lemma}
\label{lem.unitary.orthog.one-more}Let $k<n$. Let $a_{1},a_{2},\ldots,a_{k}$
be $k$ vectors in $\mathbb{C}^{n}$. Then, there exists a nonzero vector
$b\in\mathbb{C}^{n}$ that is orthogonal to each of $a_{1},a_{2},\ldots,a_{k}$.
\end{lemma}

\begin{proof}
Write each vector $a_{i}$ as $a_{i}=\left(
\begin{array}
[c]{cccc}%
a_{i,1} & a_{i,2} & \cdots & a_{i,n}%
\end{array}
\right)  ^{T}$. Now, consider an arbitrary vector $b=\left(
\begin{array}
[c]{cccc}%
b_{1} & b_{2} & \cdots & b_{n}%
\end{array}
\right)  ^{T}\in\mathbb{C}^{n}$, whose entries $b_{1},b_{2},\ldots,b_{n}$ are
so far undetermined. This new vector $b$ is orthogonal to each of $a_{1}%
,a_{2},\ldots,a_{k}$ if and only if it satisfies%
\[
\left\langle b,a_{i}\right\rangle =0\ \ \ \ \ \ \ \ \ \ \text{for all }%
i\in\left[  k\right]  .
\]
In other words, this new vector $b$ is orthogonal to each of $a_{1}%
,a_{2},\ldots,a_{k}$ if and only if it satisfies%
\[
b_{1}\overline{a_{i,1}}+b_{2}\overline{a_{i,2}}+\cdots+b_{n}\overline{a_{i,n}%
}=0\ \ \ \ \ \ \ \ \ \ \text{for all }i\in\left[  k\right]
\]
(since $\left\langle b,a_{i}\right\rangle =b_{1}\overline{a_{i,1}}%
+b_{2}\overline{a_{i,2}}+\cdots+b_{n}\overline{a_{i,n}}$ for each $i\in\left[
k\right]  $). In other words, this new vector $b$ is orthogonal to each of
$a_{1},a_{2},\ldots,a_{k}$ if and only if it satisfies the system of equations%
\[
\left\{
\begin{array}
[c]{c}%
b_{1}\overline{a_{1,1}}+b_{2}\overline{a_{1,2}}+\cdots+b_{n}\overline{a_{1,n}%
}=0;\\
b_{1}\overline{a_{2,1}}+b_{2}\overline{a_{2,2}}+\cdots+b_{n}\overline{a_{2,n}%
}=0;\\
\cdots;\\
b_{1}\overline{a_{k,1}}+b_{2}\overline{a_{k,2}}+\cdots+b_{n}\overline{a_{k,n}%
}=0.
\end{array}
\right.
\]
But this is a system of $k$ homogeneous linear equations in the $n$ unknowns
$b_{1},b_{2},\ldots,b_{n}$, and thus (by a classical fact in linear
algebra\footnote{The fact we are using here is the following: If $p$ and $q$
are two integers such that $0\leq p<q$, then any system of $p$ homogeneous
linear equations in $q$ unknowns has at least one nonzero solution. Rewritten
in terms of matrices, this is saying that if $p$ and $q$ are two integers such
that $0\leq p<q$, then any $p\times q$-matrix has a nonzero vector in its
kernel (= nullspace). For a proof, see, e.g., \cite[Remark 8.9]{Strick20} or
(rewritten in the language of linear maps) \cite[Corollary 6.5.3 item
1]{LaNaSc16}.}) has at least one nonzero solution (since $k<n$). In other
words, there exists at least one nonzero vector $b=\left(
\begin{array}
[c]{cccc}%
b_{1} & b_{2} & \cdots & b_{n}%
\end{array}
\right)  ^{T}\in\mathbb{C}^{n}$ that is orthogonal to each of $a_{1}%
,a_{2},\ldots,a_{k}$. This proves Lemma \ref{lem.unitary.orthog.one-more}.
\medskip

\begin{fineprint}
Here is a neater way to state the same argument: We define a map
$f:\mathbb{C}^{n}\rightarrow\mathbb{C}^{k}$ by setting%
\[
f\left(  w\right)  =\left(
\begin{array}
[c]{c}%
\left\langle w,a_{1}\right\rangle \\
\left\langle w,a_{2}\right\rangle \\
\vdots\\
\left\langle w,a_{k}\right\rangle
\end{array}
\right)  \ \ \ \ \ \ \ \ \ \ \text{for each }w\in\mathbb{C}^{n}.
\]
It is easy to see that this map $f$ is $\mathbb{C}$-linear. (Indeed,
Proposition \ref{prop.unitary.innerprod.props} \textbf{(c)} shows that every
two vectors $x,x^{\prime}\in\mathbb{C}^{n}$ and every $i\in\left[  k\right]  $
satisfy $\left\langle x+x^{\prime},a_{i}\right\rangle =\left\langle
x,a_{i}\right\rangle +\left\langle x^{\prime},a_{i}\right\rangle $; therefore,
every two vectors $x,x^{\prime}\in\mathbb{C}^{n}$ satisfy $f\left(
x+x^{\prime}\right)  =f\left(  x\right)  +f\left(  x^{\prime}\right)  $.
Similarly, Proposition \ref{prop.unitary.innerprod.props} \textbf{(e)} can be
used to show that $f\left(  \lambda x\right)  =\lambda f\left(  x\right)  $
for each $\lambda\in\mathbb{C}$ and $x\in\mathbb{C}^{n}$. Hence, $f$ is
$\mathbb{C}$-linear.)

Now, we know that $f$ is a $\mathbb{C}$-linear map from $\mathbb{C}^{n}$ to
$\mathbb{C}^{k}$. Hence, the rank-nullity theorem (see, e.g., \cite[Chapter 2,
Theorem 7.2]{Treil15} or \cite[Chapter II, Corollary 2.15]{Knapp1} or
\cite[Proposition 3.3.35]{Goodman}) yields that%
\[
n=\dim\left(  \operatorname*{Ker}f\right)  +\dim\left(  \operatorname{Im}%
f\right)  ,
\]
where $\operatorname*{Ker}f$ denotes the kernel of $f$ (that is, the subspace
of $\mathbb{C}^{n}$ that consists of all vectors $v\in\mathbb{C}^{n}$
satisfying $f\left(  v\right)  =0$), and where $\operatorname{Im}f$ denotes
the image\footnote{also known as \textquotedblleft range\textquotedblright} of
$f$ (that is, the subspace of $\mathbb{C}^{k}$ consisting of all vectors of
the form $f\left(  v\right)  $ with $v\in\mathbb{C}^{n}$). Therefore,%
\[
\dim\left(  \operatorname*{Ker}f\right)  =n-\dim\left(  \operatorname{Im}%
f\right)  .
\]
However, $\operatorname{Im}f$ is a vector subspace of $\mathbb{C}^{k}$, and
thus has dimension $\leq k$. Thus, $\dim\left(  \operatorname{Im}f\right)
\leq k<n$, so that%
\[
\dim\left(  \operatorname*{Ker}f\right)  =n-\underbrace{\dim\left(
\operatorname{Im}f\right)  }_{<n}>n-n=0.
\]
This shows that the vector space $\operatorname*{Ker}f$ contains at least one
nonzero vector $b$. Consider this $b$. Thus, $b\in\operatorname*{Ker}%
f\subseteq\mathbb{C}^{n}$.

However, $b\in\operatorname*{Ker}f$ shows that $f\left(  b\right)  =0$. But
the definition of $f$ yields $f\left(  b\right)  =\left(
\begin{array}
[c]{c}%
\left\langle b,a_{1}\right\rangle \\
\left\langle b,a_{2}\right\rangle \\
\vdots\\
\left\langle b,a_{k}\right\rangle
\end{array}
\right)  $. Thus, $\left(
\begin{array}
[c]{c}%
\left\langle b,a_{1}\right\rangle \\
\left\langle b,a_{2}\right\rangle \\
\vdots\\
\left\langle b,a_{k}\right\rangle
\end{array}
\right)  =f\left(  b\right)  =0$. In other words, each $i\in\left[  k\right]
$ satisfies $\left\langle b,a_{i}\right\rangle =0$. In other words, each
$i\in\left[  k\right]  $ satisfies $b\perp a_{i}$. In other words, $b$ is
orthogonal to each of $a_{1},a_{2},\ldots,a_{k}$. Thus, we have found a
nonzero vector $b\in\mathbb{C}^{n}$ that is orthogonal to each of $a_{1}%
,a_{2},\ldots,a_{k}$. This proves Lemma \ref{lem.unitary.orthog.one-more}.
\end{fineprint}
\end{proof}

\begin{corollary}
\label{cor.unitary.orthog-extend}Let $\left(  u_{1},u_{2},\ldots,u_{k}\right)
$ be an orthogonal $k$-tuple of nonzero vectors in $\mathbb{C}^{n}$. Then, we
have $k\leq n$, and we can find $n-k$ further nonzero vectors $u_{k+1}%
,u_{k+2},\ldots,u_{n}$ such that $\left(  u_{1},u_{2},\ldots,u_{n}\right)  $
is an orthogonal basis of $\mathbb{C}^{n}$.
\end{corollary}

\begin{exercise}
\label{exe.unitary.orthog-extend}\fbox{2} Prove Corollary
\ref{cor.unitary.orthog-extend}.
\end{exercise}

\begin{corollary}
\label{cor.unitary.orthon-extend}Let $\left(  u_{1},u_{2},\ldots,u_{k}\right)
$ be an orthonormal $k$-tuple of vectors in $\mathbb{C}^{n}$. Then, we have
$k\leq n$, and we can find $n-k$ further nonzero vectors $u_{k+1}%
,u_{k+2},\ldots,u_{n}$ such that $\left(  u_{1},u_{2},\ldots,u_{n}\right)  $
is an orthonormal basis of $\mathbb{C}^{n}$.
\end{corollary}

\begin{proof}
The $k$-tuple $\left(  u_{1},u_{2},\ldots,u_{k}\right)  $ is an orthogonal
tuple of nonzero vectors (since it is orthonormal). Hence, Corollary
\ref{cor.unitary.orthog-extend} yields that we can find $n-k$ further nonzero
vectors $u_{k+1},u_{k+2},\ldots,u_{n}$ such that $\left(  u_{1},u_{2}%
,\ldots,u_{n}\right)  $ is an orthogonal basis of $\mathbb{C}^{n}$. Consider
these $n-k$ vectors $u_{k+1},u_{k+2},\ldots,u_{n}$, and replace them by%
\[
\dfrac{1}{\left\vert \left\vert u_{k+1}\right\vert \right\vert }%
u_{k+1},\ \ \dfrac{1}{\left\vert \left\vert u_{k+2}\right\vert \right\vert
}u_{k+2},\ \ \ldots,\ \ \dfrac{1}{\left\vert \left\vert u_{n}\right\vert
\right\vert }u_{n},
\]
respectively. Then, the orthogonal basis $\left(  u_{1},u_{2},\ldots
,u_{n}\right)  $ becomes an orthonormal basis (since an orthogonal basis
remains orthogonal when we scale its entries, and since the first $k$ vectors
$u_{1},u_{2},\ldots,u_{k}$ already have length $1$ by assumption). Thus,
Corollary \ref{cor.unitary.orthon-extend} is proved.
\end{proof}

\subsection{Conjugate transposes}

The following definition generalizes Definition
\ref{def.unitary.innerprod.ystar}:

\begin{definition}
\label{def.unitary.innerprod.A*}Let $A=\left(
\begin{array}
[c]{ccc}%
a_{1,1} & \cdots & a_{1,m}\\
\vdots & \ddots & \vdots\\
a_{n,1} & \cdots & a_{n,m}%
\end{array}
\right)  \in\mathbb{C}^{n\times m}$ be any $n\times m$-matrix. Then, we define
the $m\times n$-matrix%
\[
A^{\ast}:=\left(
\begin{array}
[c]{ccc}%
\overline{a_{1,1}} & \cdots & \overline{a_{n,1}}\\
\vdots & \ddots & \vdots\\
\overline{a_{1,m}} & \cdots & \overline{a_{n,m}}%
\end{array}
\right)  \in\mathbb{C}^{m\times n}.
\]
This matrix $A^{\ast}$ is called the \emph{conjugate transpose} of $A$.
\end{definition}

This conjugate transpose $A^{\ast}$ can thus be obtained from the usual
transpose $A^{T}$ by conjugating all entries.

\begin{example}
$\left(
\begin{array}
[c]{ccc}%
1+i & 2-3i & 5i\\
6 & 2+4i & 10-i
\end{array}
\right)  ^{\ast}=\left(
\begin{array}
[c]{cc}%
1-i & 6\\
2+3i & 2-4i\\
-5i & 10+i
\end{array}
\right)  $.
\end{example}

In the olden days, the conjugate transpose of a matrix was also known as the
\textquotedblleft adjoint\textquotedblright\ of $A$. Unsurprisingly, this word
has at least one other meaning, which opens the door to a lot of unwanted
confusion; thus we will speak of the \textquotedblleft conjugate
transpose\textquotedblright\ instead.

Some authors use the alternative notation $A^{\dag}$ (read \textquotedblleft%
$A$ dagger\textquotedblright) for $A^{\ast}$. (The Wikipedia suggests calling
it the \textquotedblleft bedaggered matrix $A$\textquotedblright, although I
am not aware of anyone using this terminology outside of the Wikipedia.)
\medskip

The following rules for conjugate transposes are straightforward to check:

\begin{proposition}
\label{prop.unitary.(AB)*}\textbf{(a)} If $A\in\mathbb{C}^{n\times m}$ and
$B\in\mathbb{C}^{n\times m}$ are two matrices, then $\left(  A+B\right)
^{\ast}=A^{\ast}+B^{\ast}$. \medskip

\textbf{(b)} If $A\in\mathbb{C}^{n\times m}$ and $\lambda\in\mathbb{C}$, then
$\left(  \lambda A\right)  ^{\ast}=\overline{\lambda}A^{\ast}$. \medskip

\textbf{(c)} If $A\in\mathbb{C}^{n\times m}$ and $B\in\mathbb{C}^{m\times k}$
are two matrices, then $\left(  AB\right)  ^{\ast}=B^{\ast}A^{\ast}$. \medskip

\textbf{(d)} If $A\in\mathbb{C}^{n\times m}$, then $\left(  A^{\ast}\right)
^{\ast}=A$.
\end{proposition}

\subsection{Isometries}

\begin{definition}
\label{def.unitary.innerprod.isometry}An $n\times k$-matrix $A$ is said to be
an \emph{isometry} if $A^{\ast}A=I_{k}$.
\end{definition}

\begin{proposition}
\label{prop.unitary.innerprod.isometry.2}An $n\times k$-matrix $A$ is an
isometry if and only if its columns form an orthonormal tuple of vectors.
\end{proposition}

\begin{proof}
Let $A$ be an $n\times k$-matrix with columns $a_{1},a_{2},\ldots,a_{k}$ from
left to right. Therefore,%
\[
A=\left(
\begin{array}
[c]{ccc}%
\mid &  & \mid\\
a_{1} & \cdots & a_{k}\\
\mid &  & \mid
\end{array}
\right)  \ \ \ \ \ \ \ \ \ \ \text{and thus}\ \ \ \ \ \ \ \ \ \ A^{\ast
}=\left(
\begin{array}
[c]{ccc}%
\text{---} & a_{1}^{\ast} & \text{---}\\
& \vdots & \\
\text{---} & a_{k}^{\ast} & \text{---}%
\end{array}
\right)  .
\]
Hence,%
\begin{align*}
A^{\ast}A  &  =\left(
\begin{array}
[c]{cccc}%
a_{1}^{\ast}a_{1} & a_{1}^{\ast}a_{2} & \cdots & a_{1}^{\ast}a_{k}\\
a_{2}^{\ast}a_{1} & a_{2}^{\ast}a_{2} & \cdots & a_{2}^{\ast}a_{k}\\
\vdots & \vdots & \ddots & \vdots\\
a_{k}^{\ast}a_{1} & a_{k}^{\ast}a_{2} & \cdots & a_{k}^{\ast}a_{k}%
\end{array}
\right) \\
&  =\left(
\begin{array}
[c]{cccc}%
\left\vert \left\vert a_{1}\right\vert \right\vert ^{2} & \left\langle
a_{1},a_{2}\right\rangle  & \cdots & \left\langle a_{1},a_{k}\right\rangle \\
\left\langle a_{2},a_{1}\right\rangle  & \left\vert \left\vert a_{2}%
\right\vert \right\vert ^{2} & \cdots & \left\langle a_{2},a_{k}\right\rangle
\\
\vdots & \vdots & \ddots & \vdots\\
\left\langle a_{k},a_{1}\right\rangle  & \left\langle a_{k},a_{2}\right\rangle
& \cdots & \left\vert \left\vert a_{k}\right\vert \right\vert ^{2}%
\end{array}
\right)  .
\end{align*}
On the other hand,%
\[
I_{k}=\left(
\begin{array}
[c]{cccc}%
1 & 0 & \cdots & 0\\
0 & 1 & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & 1
\end{array}
\right)  .
\]
Thus, $A^{\ast}A=I_{k}$ holds if and only if we have%
\[
\left\langle a_{p},a_{q}\right\rangle =0\ \ \ \ \ \ \ \ \ \ \text{for all
}p\neq q
\]
and%
\[
\left\vert \left\vert a_{p}\right\vert \right\vert ^{2}%
=1\ \ \ \ \ \ \ \ \ \ \text{for each }p.
\]
In other words, $A^{\ast}A=I_{k}$ holds if and only if we have%
\[
a_{p}\perp a_{q}\ \ \ \ \ \ \ \ \ \ \text{for all }p\neq q
\]
and%
\[
\left\vert \left\vert a_{1}\right\vert \right\vert =\left\vert \left\vert
a_{2}\right\vert \right\vert =\cdots=\left\vert \left\vert a_{k}\right\vert
\right\vert =1.
\]
In other words, $A$ is an isometry if and only if $\left(  a_{1},a_{2}%
,\ldots,a_{k}\right)  $ is orthonormal. This proves Proposition
\ref{prop.unitary.innerprod.isometry.2}.
\end{proof}

Isometries are called isometries because they preserve lengths:

\begin{proposition}
\label{prop.unitary.innerprod.isometry.len}Let $A\in\mathbb{C}^{n\times k}$ be
an isometry. Then, each $x\in\mathbb{C}^{k}$ satisfies $\left\vert \left\vert
Ax\right\vert \right\vert =\left\vert \left\vert x\right\vert \right\vert $.
\end{proposition}

\begin{proof}
We have $A^{\ast}A=I_{k}$ (since $A$ is an isometry). Let $x\in\mathbb{C}^{k}%
$. Then, the definition of $\left\vert \left\vert Ax\right\vert \right\vert $
yields $\left\vert \left\vert Ax\right\vert \right\vert =\sqrt{\left\langle
Ax,Ax\right\rangle }$. Hence,%
\begin{align}
\left\vert \left\vert Ax\right\vert \right\vert ^{2}  &  =\left\langle
Ax,Ax\right\rangle \nonumber\\
&  =\underbrace{\left(  Ax\right)  ^{\ast}}_{\substack{=x^{\ast}A^{\ast
}\\\text{(by Proposition \ref{prop.unitary.(AB)*} \textbf{(c)})}%
}}Ax\ \ \ \ \ \ \ \ \ \ \left(  \text{by Proposition
\ref{prop.unitary.innerprod.props} \textbf{(a)}}\right) \nonumber\\
&  =x^{\ast}A^{\ast}Ax\label{pf.prop.unitary.innerprod.isometry.len.1}\\
&  =x^{\ast}x\ \ \ \ \ \ \ \ \ \ \left(  \text{since }A^{\ast}A=I_{k}\right)
\nonumber\\
&  =\left\langle x,x\right\rangle \ \ \ \ \ \ \ \ \ \ \left(  \text{by
Proposition \ref{prop.unitary.innerprod.props} \textbf{(a)}}\right)
\nonumber\\
&  =\left\vert \left\vert x\right\vert \right\vert ^{2}\nonumber
\end{align}
(since the definition of $\left\vert \left\vert x\right\vert \right\vert $
yields $\left\vert \left\vert x\right\vert \right\vert =\sqrt{\left\langle
x,x\right\rangle }$). In other words, we have $\left\vert \left\vert
Ax\right\vert \right\vert =\left\vert \left\vert x\right\vert \right\vert $
(since $\left\vert \left\vert Ax\right\vert \right\vert $ and $\left\vert
\left\vert x\right\vert \right\vert $ are nonnegative reals). This proves
Proposition \ref{prop.unitary.innerprod.isometry.len}.
\end{proof}

\begin{remark}
Another warning on terminology: Some authors (e.g., Conrad in
\cite[\textquotedblleft Isometries\textquotedblright]{Conrad}) use the word
\textquotedblleft isometry\textquotedblright\ in a wider sense than we do.
Namely, they use it for arbitrary maps from $\mathbb{C}^{k}$ to $\mathbb{C}%
^{n}$ that preserve distances. Our isometries can be viewed as \textbf{linear}
isometries in this wider sense, because a matrix $A\in\mathbb{C}^{n\times k}$
corresponds to a linear map from $\mathbb{C}^{k}$ to $\mathbb{C}^{n}$.
However, not all isometries in this wider sense are linear.
\end{remark}

\subsection{Unitary matrices}

\subsubsection{Definition, examples, basic properties}

\begin{definition}
\label{def.unitary.unitary.unitary}A matrix $U\in\mathbb{C}^{n\times k}$ is
said to be \emph{unitary} if and only if both $U$ and $U^{\ast}$ are isometries.
\end{definition}

%

\[%
\begin{tabular}
[c]{||l||}\hline\hline
\textbf{Lecture 2 starts here.}\\\hline\hline
\end{tabular}
\]


\begin{example}
\label{exa.unitary.unitary.exas}\textbf{(a)} The matrix $A=\dfrac{1}{\sqrt{2}%
}\left(
\begin{array}
[c]{cc}%
1 & 1\\
1 & -1
\end{array}
\right)  $ is unitary. Indeed, it is easy to see that $A^{\ast}A=I_{2}$, so
that $A$ is an isometry. Thus, $A^{\ast}$ is an isometry as well, since
$A^{\ast}=A$. Hence, $A$ is unitary. \medskip

\textbf{(b)} A $1\times1$-matrix $\left(
\begin{array}
[c]{c}%
\lambda
\end{array}
\right)  \in\mathbb{C}^{1\times1}$ is unitary if and only if $\left\vert
\lambda\right\vert =1$. \medskip

\textbf{(c)} For any $n\in\mathbb{N}$, the identity matrix $I_{n}$ is unitary.
\medskip

\textbf{(d)} Let $n\in\mathbb{N}$, and let $\sigma$ be a permutation of
$\left[  n\right]  $ (that is, a bijective map from $\left[  n\right]  $ to
$\left[  n\right]  $). Let $P_{\sigma}$ be the \emph{permutation matrix} of
$\sigma$; this is the $n\times n$-matrix whose $\left(  \sigma\left(
j\right)  ,j\right)  $-th entry is $1$ for each $j\in\left[  n\right]  $, and
whose all other entries are $0$. For instance, if $n=3$ and if $\sigma$ is the
cyclic permutation sending $1,2,3$ to $2,3,1$ (respectively), then%
\[
P_{\sigma}=\left(
\begin{array}
[c]{ccc}%
0 & 0 & 1\\
1 & 0 & 0\\
0 & 1 & 0
\end{array}
\right)  .
\]


The permutation matrix $P_{\sigma}$ is always unitary (for any $n$ and any
permutation $\sigma$). Indeed, its conjugate transpose $\left(  P_{\sigma
}\right)  ^{\ast}$ is easily seen to be the permutation matrix $P_{\sigma
^{-1}}$ of the inverse permutation $\sigma^{-1}$; but this latter permutation
matrix $P_{\sigma^{-1}}$ is also the inverse of $P_{\sigma}$. \medskip

\textbf{(e)} A diagonal matrix $\operatorname*{diag}\left(  \lambda
_{1},\lambda_{2},\ldots,\lambda_{n}\right)  \in\mathbb{C}^{n\times n}$ is
unitary if and only if its diagonal entries $\lambda_{1},\lambda_{2}%
,\ldots,\lambda_{n}$ lie on the unit circle (i.e., their absolute values
$\left\vert \lambda_{1}\right\vert ,\left\vert \lambda_{2}\right\vert
,\ldots,\left\vert \lambda_{n}\right\vert $ all equal $1$).
\end{example}

Unitary matrices can be characterized in many other ways:

\begin{theorem}
\label{thm.unitary.unitary.eqs}Let $U\in\mathbb{C}^{n\times k}$ be a matrix.
The following six statements are equivalent:

\begin{itemize}
\item $\mathcal{A}$: The matrix $U$ is unitary.

\item $\mathcal{B}$: The matrices $U$ and $U^{\ast}$ are isometries.

\item $\mathcal{C}$: We have $UU^{\ast}=I_{n}$ and $U^{\ast}U=I_{k}$.

\item $\mathcal{D}$: The matrix $U$ is square (that is, $n=k$) and invertible
and satisfies $U^{-1}=U^{\ast}$.

\item $\mathcal{E}$: The columns of $U$ form an orthonormal basis of
$\mathbb{C}^{n}$.

\item $\mathcal{F}$: The matrix $U$ is square (that is, $n=k$) and is an isometry.
\end{itemize}
\end{theorem}

\begin{proof}
The equivalence $\mathcal{A}\Longleftrightarrow\mathcal{B}$ follows
immediately from Definition \ref{def.unitary.unitary.unitary}. The equivalence
$\mathcal{B}\Longleftrightarrow\mathcal{C}$ follows immediately from the
definition of an isometry (since $\left(  U^{\ast}\right)  ^{\ast}=U$). The
implication $\mathcal{D}\Longrightarrow\mathcal{C}$ is obvious. The
implication $\mathcal{C}\Longrightarrow\mathcal{D}$ follows from the known
fact (see, e.g., \cite[Chapter 2, Corollary 3.7]{Treil15}) that every
invertible matrix is square. Let us now prove some of the other implications:

\begin{itemize}
\item $\mathcal{D}\Longrightarrow\mathcal{E}$\textit{:} Assume that statement
$\mathcal{D}$ holds. Then, $U^{\ast}U=I_{k}$ (since $U^{-1}=U^{\ast}$), and
therefore $U$ is an isometry. Hence, Proposition
\ref{prop.unitary.innerprod.isometry.2} shows that the tuple of columns of $U$
is orthonormal. However, the columns of $U$ form a basis of $\mathbb{C}^{n}$
(because $U$ is invertible), and this basis is orthonormal (since we have just
shown that the tuple of columns of $U$ is orthonormal). Thus, statement
$\mathcal{E}$ holds. We have thus proved the implication $\mathcal{D}%
\Longrightarrow\mathcal{E}$.

\item $\mathcal{E}\Longrightarrow\mathcal{D}$\textit{:} Assume that statement
$\mathcal{E}$ holds. Then, the columns of $U$ form an orthonormal basis, hence
an orthonormal tuple. Thus, Proposition
\ref{prop.unitary.innerprod.isometry.2} shows that $U$ is an isometry, so that
$U^{\ast}U=I_{k}$. However, $U$ is invertible because the columns of $U$ form
a basis of $\mathbb{C}^{n}$. Therefore, from $U^{\ast}U=I_{k}$, we obtain
$U^{-1}=U^{\ast}$. Finally, the matrix $U$ is square, since any invertible
matrix is square. Thus, statement $\mathcal{D}$ holds. We have thus proved the
implication $\mathcal{E}\Longrightarrow\mathcal{D}$.

\item $\mathcal{D}\Longrightarrow\mathcal{F}$\textit{:} The implication
$\mathcal{D}\Longrightarrow\mathcal{F}$ is easy (since $U^{-1}=U^{\ast}$
entails $U^{\ast}U=I_{k}$, which shows that $U$ is an isometry).

\item $\mathcal{F}\Longrightarrow\mathcal{D}$\textit{:} Assume that statement
$\mathcal{F}$ holds. Thus, $U$ is an isometry; that is, we have $U^{\ast
}U=I_{k}=I_{n}$ (since $k=n$). However, it is known\footnote{This is one part
of the infamous \textquotedblleft inverse matrix theorem\textquotedblright%
\ that lists many equivalent conditions for invertibility. See, for example,
\cite[Chapter 2, Proposition 3.8]{Treil15}.} that a square matrix $A$ that has
a left inverse (i.e., a further square matrix $B$ satisfying $BA=I$) must be
invertible. We can apply this to the square matrix $U$ (which has a left
inverse, since $U^{\ast}U=I_{n}$), and thus conclude that $U$ is invertible.
Hence, from $U^{\ast}U=I_{n}$, we obtain $U^{-1}=U^{\ast}$. Therefore,
statement $\mathcal{D}$ holds. We have thus proved the implication
$\mathcal{F}\Longrightarrow\mathcal{D}$.
\end{itemize}

Altogether, we have thus proved that all six statements $\mathcal{A}%
,\mathcal{B},\mathcal{C},\mathcal{D},\mathcal{E},\mathcal{F}$ are equivalent.
\end{proof}

Note that Theorem \ref{thm.unitary.unitary.eqs} (specifically, the implication
$\mathcal{A}\Longrightarrow\mathcal{D}$) shows that any unitary matrix is
square. In contrast, an isometry can be rectangular -- but only tall, not
wide, as the following exercise shows:

\begin{exercise}
\label{exe.unitary.isometry-tall}\fbox{1} Let $A\in\mathbb{C}^{n\times k}$ be
an isometry. Show that $n\geq k$.
\end{exercise}

\begin{exercise}
\label{exe.unitary.group}\fbox{3} \textbf{(a)} Prove that the product $AB$ of
two isometries $A\in\mathbb{C}^{n\times m}$ and $B\in\mathbb{C}^{m\times k}$
is always an isometry. \medskip

\textbf{(b)} Prove that the product $AB$ of two unitary matrices
$A\in\mathbb{C}^{n\times n}$ and $B\in\mathbb{C}^{n\times n}$ is always
unitary. \medskip

\textbf{(c)} Prove that the inverse of a unitary matrix $A\in\mathbb{C}%
^{n\times n}$ is always unitary.
\end{exercise}

Exercise \ref{exe.unitary.group} shows that the set of all unitary $n\times
n$-matrices over $\mathbb{C}$ (for a given $n\in\mathbb{N}$) is a group under
multiplication. This group is known as the $n$\emph{-th unitary group}, and is
denoted by $\operatorname*{U}\nolimits_{n}\left(  \mathbb{C}\right)  $.

\begin{exercise}
\label{exe.unitary.det-eval}\fbox{2} Let $U\in\mathbb{C}^{n\times n}$ be a
unitary matrix. \medskip

\textbf{(a)} Prove that $\left\vert \det U\right\vert =1$. \medskip

\textbf{(b)} Prove that any eigenvalue $\lambda$ of $U$ satisfies $\left\vert
\lambda\right\vert =1$.
\end{exercise}

\subsubsection{Various constructions of unitary matrices}

The next two exercises show some ways to generate unitary matrices:

\begin{exercise}
\label{exe.unitary.house}\fbox{3} Let $w\in\mathbb{C}^{n}$ be a nonzero
vector. Then, $w^{\ast}w=\left\langle w,w\right\rangle >0$ (by Proposition
\ref{prop.unitary.innerprod.pos} \textbf{(b)}). Thus, we can define an
$n\times n$-matrix
\[
U_{w}:=I_{n}-2\left(  w^{\ast}w\right)  ^{-1}ww^{\ast}\in\mathbb{C}^{n\times
n}.
\]
This is called a \emph{Householder matrix}.

Show that this matrix $U_{w}$ is unitary and satisfies $U_{w}^{\ast}=U_{w}$.
\end{exercise}

The next exercise uses the notion of a skew-Hermitian matrix:

\begin{definition}
\label{def.unitary.skew-herm}A matrix $S\in\mathbb{C}^{n\times n}$ is said to
be \emph{skew-Hermitian} if and only if $S^{\ast}=-S$.
\end{definition}

For instance, the matrix $\left(
\begin{array}
[c]{cc}%
i & 1\\
-1 & 0
\end{array}
\right)  $ is skew-Hermitian.

\begin{exercise}
\fbox{5} \label{exe.unitary.skew-herm.1}Let $S\in\mathbb{C}^{n\times n}$ be a
skew-Hermitian matrix. \medskip

\textbf{(a)} Prove that the matrix $I_{n}-S$ is invertible. \medskip

[\textbf{Hint:} Show first that the matrix $I_{n}+S^{\ast}S$ is invertible,
since each nonzero vector $v\in\mathbb{C}^{n}$ satisfies $v^{\ast}\left(
I_{n}+S^{\ast}S\right)  v=\underbrace{\left\langle v,v\right\rangle }%
_{>0}+\underbrace{\left\langle Sv,Sv\right\rangle }_{\geq0}>0$. Then, expand
the product $\left(  I_{n}-S^{\ast}\right)  \left(  I_{n}-S\right)  $.]
\medskip

\textbf{(b)} Prove that the matrices $I_{n}+S$ and $\left(  I_{n}-S\right)
^{-1}$ commute (i.e., satisfy $\left(  I_{n}+S\right)  \cdot\left(
I_{n}-S\right)  ^{-1}=\left(  I_{n}-S\right)  ^{-1}\cdot\left(  I_{n}%
+S\right)  $). \medskip

\textbf{(c)} Prove that the matrix $U:=\left(  I_{n}-S\right)  ^{-1}%
\cdot\left(  I_{n}+S\right)  $ is unitary. \medskip

\textbf{(d)} Prove that the matrix $U+I_{n}$ is invertible. \medskip

\textbf{(e)} Prove that $S=\left(  U-I_{n}\right)  \cdot\left(  U+I_{n}%
\right)  ^{-1}$.
\end{exercise}

Exercise \ref{exe.unitary.skew-herm.1} constructs a map\footnote{Recall that
$\operatorname*{U}\nolimits_{n}\left(  \mathbb{C}\right)  $ denotes the $n$-th
unitary group (i.e., the set of all unitary $n\times n$-matrices).}%
\begin{align*}
\left\{  \text{skew-Hermitian matrices in }\mathbb{C}^{n\times n}\right\}   &
\rightarrow\left\{  U\in\operatorname*{U}\nolimits_{n}\left(  \mathbb{C}%
\right)  \text{ }\mid\ U+I_{n}\text{ is invertible}\right\}  ,\\
S  &  \mapsto\left(  I_{n}-S\right)  ^{-1}\cdot\left(  I_{n}+S\right)  .
\end{align*}
This map is known as the \emph{Cayley parametrization} of the unitary matrices
(and can be seen as an $n$-dimensional generalization of the stereographic
projection from the imaginary axis to the unit circle -- which is what it does
for $n=1$). Exercise \ref{exe.unitary.skew-herm.1} \textbf{(e)} shows that it
is injective. It is not hard to check that it is surjective, too.

How close is the set $\left\{  U\in\operatorname*{U}\nolimits_{n}\left(
\mathbb{C}\right)  \text{ }\mid\ U+I_{n}\text{ is invertible}\right\}  $ to
the whole unitary group $\operatorname*{U}\nolimits_{n}\left(  \mathbb{C}%
\right)  $ ? The answer is that it is almost the entire group
$\operatorname*{U}\nolimits_{n}\left(  \mathbb{C}\right)  $. Here is a
rigorous way to state this:

\begin{exercise}
\label{exe.unitary.skew-herm.2}\fbox{3} Let $A\in\mathbb{C}^{n\times n}$ be a
matrix. Prove the following: \medskip

\textbf{(a)} If $A$ is unitary, then the matrix $\lambda A$ is unitary for
each $\lambda\in\mathbb{C}$ satisfying $\left\vert \lambda\right\vert =1$.
\medskip

\textbf{(b)} The matrix $\lambda A+I_{n}$ is invertible for all but finitely
many $\lambda\in\mathbb{C}$. \medskip

[\textbf{Hint:} The determinant $\det\left(  \lambda A+I_{n}\right)  $ is a
polynomial function in $\lambda$.] \medskip

\textbf{(c)} The set $\left\{  U\in\operatorname*{U}\nolimits_{n}\left(
\mathbb{C}\right)  \text{ }\mid\ U+I_{n}\text{ is invertible}\right\}  $ is
dense in $\operatorname*{U}\nolimits_{n}\left(  \mathbb{C}\right)  $. (That
is, each unitary matrix in $\operatorname*{U}\nolimits_{n}\left(
\mathbb{C}\right)  $ can be written as a limit $\lim\limits_{k\rightarrow
\infty}U_{k}$ of a sequence of unitary matrices $U_{k}$ such that $U_{k}%
+I_{n}$ is invertible for each $k$.)
\end{exercise}

Thus, if the Cayley parametrization does not hit a unitary matrix, then at
least it comes arbitrarily close.

\begin{remark}
A square matrix $A\in\mathbb{C}^{n\times n}$ satisfying $AA^{T}=A^{T}A=I_{n}$
is called \emph{orthogonal}. Thus, unitary matrices differ from orthogonal
matrices only in the use of the conjugate transpose $A^{\ast}$ instead of the
transpose $A^{T}$. In particular, a matrix $A\in\mathbb{R}^{n\times n}$ (with
real entries) is orthogonal if and only if it is unitary.
\end{remark}

\begin{exercise}
\label{exe.unitary.skew-herm.pyth}\fbox{5} A \emph{Pythagorean triple} is a
triple $\left(  p,q,r\right)  $ of positive integers satisfying $p^{2}%
+q^{2}=r^{2}$. (In other words, it is a triple of positive integers that are
the sides of a right-angled triangle.) Two famous Pythagorean triples are
$\left(  3,4,5\right)  $ and $\left(  5,12,13\right)  $. \medskip

\textbf{(a)} Prove that a triple $\left(  p,q,r\right)  $ of positive integers
is Pythagorean if and only if the matrix $\left(
\begin{array}
[c]{cc}%
p/r & -q/r\\
q/r & p/r
\end{array}
\right)  $ is unitary. \medskip

\textbf{(b)} Let $\left(
\begin{array}
[c]{cc}%
a & b\\
c & d
\end{array}
\right)  $ be any unitary matrix with rational entries. Assume that $a$ and
$c$ are positive, and write $a$ and $c$ as $p/r$ and $q/r$ for some positive
integers $p,q,r$. Show that $\left(  p,q,r\right)  $ is a Pythagorean triple.
\medskip

\textbf{(c)} Find infinitely many Pythagorean triples that are pairwise
non-proportional (i.e., no two of them are obtained from one another just by
multiplying all three entries by the same number). \medskip

[\textbf{Hint:} Use the $S\mapsto U$ construction from Exercise
\ref{exe.unitary.skew-herm.1}.]
\end{exercise}

We shall soon see one more way to construct unitary matrices from smaller
ones, using the notion of block matrices, which we shall now introduce.

Incidentally, here is another simple but useful property of skew-Hermitian matrices:

\begin{exercise}
\fbox{2} Let $A,B\in\mathbb{C}^{n\times n}$ be two skew-Hermitian matrices.
Show that $AB-BA$ is again skew-Hermitian.
\end{exercise}

\subsection{Block matrices}

\subsubsection{Definition}

\begin{definition}
\label{def.blockmats.2x2}Let $\mathbb{F}$ be a field. Let $n,m,p,q\in
\mathbb{N}$. Let $A\in\mathbb{F}^{n\times p}$, $B\in\mathbb{F}^{n\times q}$,
$C\in\mathbb{F}^{m\times p}$ and $D\in\mathbb{F}^{m\times q}$ be four
matrices. Then, $\left(
\begin{array}
[c]{cc}%
A & B\\
C & D
\end{array}
\right)  $ shall denote the $\left(  n+m\right)  \times\left(  p+q\right)
$-matrix obtained by \textquotedblleft gluing\textquotedblright\ the four
matrices $A,B,C,D$ together in the manner suggested by the notation (i.e., we
glue $B$ to the right edge of $A$, we glue $C$ to the bottom edge of $A$, and
we glue $D$ to the right edge of $C$ and to the bottom edge of $B$). In other
words, we set%
\[
\left(
\begin{array}
[c]{cc}%
A & B\\
C & D
\end{array}
\right)  :=\left(
\begin{array}
[c]{cccccccc}%
A_{1,1} & A_{1,2} & \cdots & A_{1,p} & B_{1,1} & B_{1,2} & \cdots & B_{1,q}\\
A_{2,1} & A_{2,2} & \cdots & A_{2,p} & B_{2,1} & B_{2,2} & \cdots & B_{2,q}\\
\vdots & \vdots & \ddots & \vdots & \vdots & \vdots & \ddots & \vdots\\
A_{n,1} & A_{n,2} & \cdots & A_{n,p} & B_{n,1} & B_{n,2} & \cdots & B_{n,q}\\
C_{1,1} & C_{1,2} & \cdots & C_{1,p} & D_{1,1} & D_{1,2} & \cdots & D_{1,q}\\
C_{2,1} & C_{2,2} & \cdots & C_{2,p} & D_{2,1} & D_{2,2} & \cdots & D_{2,q}\\
\vdots & \vdots & \ddots & \vdots & \vdots & \vdots & \ddots & \vdots\\
C_{m,1} & C_{m,2} & \cdots & C_{m,p} & D_{m,1} & D_{m,2} & \cdots & D_{m,q}%
\end{array}
\right)
\]
(where, as we recall, the notation $M_{i,j}$ denotes the $\left(  i,j\right)
$-th entry of a matrix $M$).
\end{definition}

\begin{example}
If $A=\left(
\begin{array}
[c]{cc}%
a & a^{\prime}\\
a^{\prime\prime} & a^{\prime\prime\prime}%
\end{array}
\right)  $ and $B=\left(
\begin{array}
[c]{c}%
b\\
b^{\prime}%
\end{array}
\right)  $ and $C=\left(
\begin{array}
[c]{cc}%
c & c^{\prime}%
\end{array}
\right)  $ and $D=\left(
\begin{array}
[c]{c}%
d
\end{array}
\right)  $, then $\left(
\begin{array}
[c]{cc}%
A & B\\
C & D
\end{array}
\right)  =\left(
\begin{array}
[c]{ccc}%
a & a^{\prime} & b\\
a^{\prime\prime} & a^{\prime\prime\prime} & b^{\prime}\\
c & c^{\prime} & d
\end{array}
\right)  $.
\end{example}

The notation introduced in Definition \ref{def.blockmats.2x2} is called
\emph{block matrix notation}, and can be generalized to more than four matrices:

\begin{definition}
\label{def.blockmatrix.uxv}Let $\mathbb{F}$ be a field. Let $u,v\in\mathbb{N}%
$. Let $n_{1},n_{2},\ldots,n_{u}\in\mathbb{N}$ and $p_{1},p_{2},\ldots
,p_{v}\in\mathbb{N}$. For each $i\in\left[  u\right]  $ and $j\in\left[
v\right]  $, let $A\left(  i,j\right)  \in\mathbb{F}^{n_{i}\times p_{j}}$ be a
matrix. (We denote it by $A\left(  i,j\right)  $ instead of $A_{i,j}$ to avoid
mistaking it for a single entry.) Then,%
\begin{equation}
\left(
\begin{array}
[c]{cccc}%
A\left(  1,1\right)  & A\left(  1,2\right)  & \cdots & A\left(  1,v\right) \\
A\left(  2,1\right)  & A\left(  2,2\right)  & \cdots & A\left(  2,v\right) \\
\vdots & \vdots & \ddots & \vdots\\
A\left(  u,1\right)  & A\left(  u,2\right)  & \cdots & A\left(  u,v\right)
\end{array}
\right)  \label{eq.def.blockmatrix.uxv.blockmat}%
\end{equation}
shall denote the $\left(  n_{1}+n_{2}+\cdots+n_{u}\right)  \times\left(
p_{1}+p_{2}+\cdots+p_{v}\right)  $-matrix obtained by \textquotedblleft
gluing\textquotedblright\ the matrices $A\left(  i,j\right)  $ together in the
manner suggested by the notation. In other words,%
\[
\left(
\begin{array}
[c]{cccc}%
A\left(  1,1\right)  & A\left(  1,2\right)  & \cdots & A\left(  1,v\right) \\
A\left(  2,1\right)  & A\left(  2,2\right)  & \cdots & A\left(  2,v\right) \\
\vdots & \vdots & \ddots & \vdots\\
A\left(  u,1\right)  & A\left(  u,2\right)  & \cdots & A\left(  u,v\right)
\end{array}
\right)
\]
shall denote the $\left(  n_{1}+n_{2}+\cdots+n_{u}\right)  \times\left(
p_{1}+p_{2}+\cdots+p_{v}\right)  $-matrix whose $\left(  n_{1}+n_{2}%
+\cdots+n_{i-1}+k,\ \ p_{1}+p_{2}+\cdots+p_{j-1}+\ell\right)  $-th entry is
$\left(  A\left(  i,j\right)  \right)  _{k,\ell}$ for all $i\in\left[
u\right]  $ and $j\in\left[  v\right]  $ and $k\in\left[  n_{i}\right]  $ and
$\ell\in\left[  p_{j}\right]  $.
\end{definition}

Alternatively, this matrix can be defined abstractly using direct sums of
vector spaces; see \cite[Chapter II, \S 10, section 2]{Bourba74} for this definition.

\begin{example}
Let $0_{2\times2}$ denote the zero matrix of size $2\times2$. Then,%
\[
\left(
\begin{array}
[c]{ccc}%
0_{2\times2} & I_{2} & 0_{2\times2}\\
I_{2} & 0_{2\times2} & 0_{2\times2}\\
0_{2\times2} & -I_{2} & I_{2}%
\end{array}
\right)  =\left(
\begin{array}
[c]{cccccc}%
0 & 0 & 1 & 0 & 0 & 0\\
0 & 0 & 0 & 1 & 0 & 0\\
1 & 0 & 0 & 0 & 0 & 0\\
0 & 1 & 0 & 0 & 0 & 0\\
0 & 0 & -1 & 0 & 1 & 0\\
0 & 0 & 0 & -1 & 0 & 1
\end{array}
\right)  .
\]

\end{example}

In Definition \ref{def.blockmatrix.uxv}, the big matrix
(\ref{eq.def.blockmatrix.uxv.blockmat}) is called the \emph{block matrix}
formed out of the matrices $A\left(  i,j\right)  $; the single matrices
$A\left(  i,j\right)  $ are called its \emph{blocks}.

\subsubsection{Multiplying block matrices}

One of the most useful properties of block matrices is that they can be
multiplied \textquotedblleft as if the blocks were numbers\textquotedblright%
\ (i.e., by the same formula as for regular matrices), provided that the
products make sense. Let us state this more precisely -- first for the case of
four blocks:

\begin{proposition}
\label{prop.blockmatrix.mult-2x2}Let $\mathbb{F}$ be a field. Let $n$,
$n^{\prime}$, $m$, $m^{\prime}$, $\ell$ and $\ell^{\prime}$ be six nonnegative
integers. Let $A\in\mathbb{F}^{n\times m}$, $B\in\mathbb{F}^{n\times
m^{\prime}}$, $C\in\mathbb{F}^{n^{\prime}\times m}$, $D\in\mathbb{F}%
^{n^{\prime}\times m^{\prime}}$, $A^{\prime}\in\mathbb{F}^{m\times\ell}$,
$B^{\prime}\in\mathbb{F}^{m\times\ell^{\prime}}$, $C^{\prime}\in
\mathbb{F}^{m^{\prime}\times\ell}$ and $D^{\prime}\in\mathbb{F}^{m^{\prime
}\times\ell^{\prime}}$. Then,
\[
\left(
\begin{array}
[c]{cc}%
A & B\\
C & D
\end{array}
\right)  \left(
\begin{array}
[c]{cc}%
A^{\prime} & B^{\prime}\\
C^{\prime} & D^{\prime}%
\end{array}
\right)  =\left(
\begin{array}
[c]{cc}%
AA^{\prime}+BC^{\prime} & AB^{\prime}+BD^{\prime}\\
CA^{\prime}+DC^{\prime} & CB^{\prime}+DD^{\prime}%
\end{array}
\right)  .
\]

\end{proposition}

For comparison, here is the formula for the product of two $2\times2$-matrices
(consisting of numbers, not blocks):%
\[
\left(
\begin{array}
[c]{cc}%
a & b\\
c & d
\end{array}
\right)  \left(
\begin{array}
[c]{cc}%
a^{\prime} & b^{\prime}\\
c^{\prime} & d^{\prime}%
\end{array}
\right)  =\left(
\begin{array}
[c]{cc}%
aa^{\prime}+bc^{\prime} & ab^{\prime}+bd^{\prime}\\
ca^{\prime}+dc^{\prime} & cb^{\prime}+dd^{\prime}%
\end{array}
\right)
\]
(for any $a,b,c,d,a^{\prime},b^{\prime},c^{\prime},d^{\prime}\in\mathbb{F}$).
Thus, Proposition \ref{prop.blockmatrix.mult-2x2} is saying that the same
formula can be used to multiply block matrices made of appropriately sized
blocks. Thus, roughly speaking, we can multiply block matrices
\textquotedblleft as if the blocks were numbers\textquotedblright. To be fully
honest, two caveats apply here:

\begin{itemize}
\item In the formula for $\left(
\begin{array}
[c]{cc}%
a & b\\
c & d
\end{array}
\right)  \left(
\begin{array}
[c]{cc}%
a^{\prime} & b^{\prime}\\
c^{\prime} & d^{\prime}%
\end{array}
\right)  $, we can write the right hand side in many different ways: e.g., we
can replace $aa^{\prime}$ by $a^{\prime}a$, because multiplication of numbers
is commutative. In contrast, multiplication of matrices is not commutative, so
that we cannot replace $AA^{\prime}$ by $A^{\prime}A$ in Proposition
\ref{prop.blockmatrix.mult-2x2}. Thus, we can multiply block matrices
\textquotedblleft as if the blocks were numbers\textquotedblright, but we have
to keep the blocks in the correct order (viz., in the order in which they
appear on the left hand side).

\item We cannot use Proposition \ref{prop.blockmatrix.mult-2x2} to multiply
two arbitrary block matrices; indeed, Proposition
\ref{prop.blockmatrix.mult-2x2} requires the blocks to have \textquotedblleft
matching\textquotedblright\ dimensions. For example, $A$ must have as many
columns as $A^{\prime}$ has rows (this is enforced by the assumptions
$A\in\mathbb{F}^{n\times m}$ and $A^{\prime}\in\mathbb{F}^{m\times\ell}$). If
this wasn't the case, then the product $AA^{\prime}$ on the right hand side
wouldn't even make sense!
\end{itemize}

\begin{proof}
[Proof of Proposition \ref{prop.blockmatrix.mult-2x2}.]Just check that each
entry on the left hand side equals the corresponding entry on the right. This
is a straightforward computation that is made painful by the notational load
and the need to distinguish between four cases (depending on which block our
entry lies in). Do one of the four cases to convince yourself that there is
nothing difficult here. (See \cite{detnotes} for all the gory details.)
\end{proof}

Unsurprisingly, Proposition \ref{prop.blockmatrix.mult-2x2} generalizes to the
multi-block case:

\begin{proposition}
\label{prop.blockmatrix.mult-uxv}Let $\mathbb{F}$ be a field. Let
$u,v,w\in\mathbb{N}$. Let $n_{1},n_{2},\ldots,n_{u}\in\mathbb{N}$ and
$p_{1},p_{2},\ldots,p_{v}\in\mathbb{N}$ and $q_{1},q_{2},\ldots,q_{w}%
\in\mathbb{N}$. For each $i\in\left[  u\right]  $ and $j\in\left[  v\right]
$, let $A\left(  i,j\right)  \in\mathbb{F}^{n_{i}\times p_{j}}$ be a matrix.
For each $j\in\left[  v\right]  $ and $k\in\left[  w\right]  $, let $B\left(
j,k\right)  \in\mathbb{F}^{p_{j}\times q_{k}}$ be a matrix. Then,%
\begin{align*}
&  \left(
\begin{array}
[c]{cccc}%
A\left(  1,1\right)  & A\left(  1,2\right)  & \cdots & A\left(  1,v\right) \\
A\left(  2,1\right)  & A\left(  2,2\right)  & \cdots & A\left(  2,v\right) \\
\vdots & \vdots & \ddots & \vdots\\
A\left(  u,1\right)  & A\left(  u,2\right)  & \cdots & A\left(  u,v\right)
\end{array}
\right)  \left(
\begin{array}
[c]{cccc}%
B\left(  1,1\right)  & B\left(  1,2\right)  & \cdots & B\left(  1,w\right) \\
B\left(  2,1\right)  & B\left(  2,2\right)  & \cdots & B\left(  2,w\right) \\
\vdots & \vdots & \ddots & \vdots\\
B\left(  v,1\right)  & B\left(  v,2\right)  & \cdots & B\left(  v,w\right)
\end{array}
\right) \\
&  =\left(
\begin{array}
[c]{cccc}%
\sum_{j=1}^{v}A\left(  1,j\right)  B\left(  j,1\right)  & \sum_{j=1}%
^{v}A\left(  1,j\right)  B\left(  j,2\right)  & \cdots & \sum_{j=1}%
^{v}A\left(  1,j\right)  B\left(  j,w\right) \\
\sum_{j=1}^{v}A\left(  2,j\right)  B\left(  j,1\right)  & \sum_{j=1}%
^{v}A\left(  2,j\right)  B\left(  j,2\right)  & \cdots & \sum_{j=1}%
^{v}A\left(  2,j\right)  B\left(  j,w\right) \\
\vdots & \vdots & \ddots & \vdots\\
\sum_{j=1}^{v}A\left(  u,j\right)  B\left(  j,1\right)  & \sum_{j=1}%
^{v}A\left(  u,j\right)  B\left(  j,2\right)  & \cdots & \sum_{j=1}%
^{v}A\left(  u,j\right)  B\left(  j,w\right)
\end{array}
\right)  .
\end{align*}

\end{proposition}

\begin{proof}
Just like Proposition \ref{prop.blockmatrix.mult-2x2}, but with more indices.
In short, fun!
\end{proof}

\subsubsection{Block-diagonal matrices}

\begin{definition}
\emph{Block-diagonal matrices} are block matrices of the form
(\ref{eq.def.blockmatrix.uxv.blockmat}), where

\begin{itemize}
\item we have $u=v$,

\item all matrices $A\left(  i,i\right)  $ are square (i.e., we have
$n_{i}=p_{i}$ for all $i\in\left[  u\right]  $), and

\item all $A\left(  i,j\right)  $ with $i\neq j$ are zero matrices.
\end{itemize}

In other words, block-diagonal matrices are block matrices of the form%
\[
\left(
\begin{array}
[c]{cccc}%
A\left(  1,1\right)  & 0 & \cdots & 0\\
0 & A\left(  2,2\right)  & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & A\left(  u,u\right)
\end{array}
\right)  ,
\]
where $A\left(  1,1\right)  ,A\left(  2,2\right)  ,\ldots,A\left(  u,u\right)
$ are arbitrary square matrices, and where each \textquotedblleft%
$0$\textquotedblright\ means a zero matrix of appropriate dimensions.
\end{definition}

As an easy consequence of Proposition \ref{prop.blockmatrix.mult-uxv}, we
obtain a multiplication rule for block-diagonal matrices that looks exactly
like multiplication of usual diagonal matrices:

\begin{corollary}
\label{cor.blockmatrix.mult-diag}Let $u\in\mathbb{N}$. Let $n_{1},n_{2}%
,\ldots,n_{u}\in\mathbb{N}$. For each $i\in\left[  u\right]  $, let $A\left(
i,i\right)  $ and $B\left(  i,i\right)  $ be two $n_{i}\times n_{i}$-matrices.
Then,%
\begin{align*}
&  \left(
\begin{array}
[c]{cccc}%
A\left(  1,1\right)  & 0 & \cdots & 0\\
0 & A\left(  2,2\right)  & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & A\left(  u,u\right)
\end{array}
\right)  \left(
\begin{array}
[c]{cccc}%
B\left(  1,1\right)  & 0 & \cdots & 0\\
0 & B\left(  2,2\right)  & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & B\left(  u,u\right)
\end{array}
\right) \\
&  =\left(
\begin{array}
[c]{cccc}%
A\left(  1,1\right)  B\left(  1,1\right)  & 0 & \cdots & 0\\
0 & A\left(  2,2\right)  B\left(  2,2\right)  & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & A\left(  u,u\right)  B\left(  u,u\right)
\end{array}
\right)  .
\end{align*}
(Here, each \textquotedblleft$0$\textquotedblright\ means a zero matrix of
appropriate dimensions.)
\end{corollary}

\begin{example}
Let $u=2$ and $n_{1}=1$ and $n_{2}=2$. Let $A\left(  1,1\right)  =\left(
\begin{array}
[c]{c}%
a
\end{array}
\right)  $ and $A\left(  2,2\right)  =\left(
\begin{array}
[c]{cc}%
b & c\\
d & e
\end{array}
\right)  $ and $B\left(  1,1\right)  =\left(
\begin{array}
[c]{c}%
a^{\prime}%
\end{array}
\right)  $ and $B\left(  2,2\right)  =\left(
\begin{array}
[c]{cc}%
b^{\prime} & c^{\prime}\\
d^{\prime} & e^{\prime}%
\end{array}
\right)  $. Then, Corollary \ref{cor.blockmatrix.mult-diag} says that%
\[
\left(
\begin{array}
[c]{cc}%
A\left(  1,1\right)  & 0\\
0 & A\left(  2,2\right)
\end{array}
\right)  \left(
\begin{array}
[c]{cc}%
B\left(  1,1\right)  & 0\\
0 & B\left(  2,2\right)
\end{array}
\right)  =\left(
\begin{array}
[c]{cc}%
A\left(  1,1\right)  B\left(  1,1\right)  & 0\\
0 & A\left(  2,2\right)  B\left(  2,2\right)
\end{array}
\right)  ,
\]
i.e., that%
\[
\left(
\begin{array}
[c]{ccc}%
a & 0 & 0\\
0 & b & c\\
0 & d & e
\end{array}
\right)  \left(
\begin{array}
[c]{ccc}%
a^{\prime} & 0 & 0\\
0 & b^{\prime} & c^{\prime}\\
0 & d^{\prime} & e^{\prime}%
\end{array}
\right)  =\left(
\begin{array}
[c]{ccc}%
aa^{\prime} & 0 & 0\\
0 & bb^{\prime}+cd^{\prime} & bc^{\prime}+ce^{\prime}\\
0 & db^{\prime}+ed^{\prime} & dc^{\prime}+ee^{\prime}%
\end{array}
\right)  .
\]

\end{example}

Corollary \ref{cor.blockmatrix.mult-diag} can be stated (somewhat imprecisely)
as follows: To multiply two block-diagonal matrices, we just multiply
respective blocks with each other. The same applies to addition instead of
multiplication. Thus, one can think of the diagonal blocks in a block-diagonal
matrix as separate matrices, which are stuck together in a block-diagonal
shape but don't interfere with each other.

Taking powers of block-diagonal matrices follows the same paradigm:

\begin{corollary}
\label{cor.blockmatrix.powt-diag}Let $u\in\mathbb{N}$. Let $n_{1},n_{2}%
,\ldots,n_{u}\in\mathbb{N}$. For each $i\in\left[  u\right]  $, let $A\left(
i,i\right)  $ be an $n_{i}\times n_{i}$-matrix. Then,%
\begin{align*}
&  \left(
\begin{array}
[c]{cccc}%
A\left(  1,1\right)  & 0 & \cdots & 0\\
0 & A\left(  2,2\right)  & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & A\left(  u,u\right)
\end{array}
\right)  ^{k}\\
&  =\left(
\begin{array}
[c]{cccc}%
\left(  A\left(  1,1\right)  \right)  ^{k} & 0 & \cdots & 0\\
0 & \left(  A\left(  2,2\right)  \right)  ^{k} & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & \left(  A\left(  u,u\right)  \right)  ^{k}%
\end{array}
\right)
\end{align*}
for any $k\in\mathbb{N}$. (Here, each \textquotedblleft$0$\textquotedblright%
\ means a zero matrix of appropriate dimensions.)
\end{corollary}

\begin{proof}
Straightforward induction on $k$. The base case ($k=0$) says that%
\[
I_{n_{1}+n_{2}+\cdots+n_{u}}=\left(
\begin{array}
[c]{cccc}%
I_{n_{1}} & 0 & \cdots & 0\\
0 & I_{n_{2}} & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & I_{n_{u}}%
\end{array}
\right)  ,
\]
which should be fairly clear. The induction step is an easy application of
Corollary \ref{cor.blockmatrix.mult-diag}.
\end{proof}

Finally, the \textquotedblleft diagonal blocks stuck
together\textquotedblright\ philosophy for block-diagonal matrices holds for
nullities as well. To wit, the nullity of a block-diagonal matrix is the sum
of the nullities of its diagonal blocks. In other words:

\begin{proposition}
\label{prop.blockmatrix.nullity-diag}Let $u\in\mathbb{N}$. Let $n_{1}%
,n_{2},\ldots,n_{u}\in\mathbb{N}$. For each $i\in\left[  u\right]  $, let
$A_{i}$ be an $n_{i}\times n_{i}$-matrix. Then,%
\begin{align}
&  \dim\left(  \operatorname*{Ker}\left(
\begin{array}
[c]{cccc}%
A_{1} & 0 & \cdots & 0\\
0 & A_{2} & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & A_{u}%
\end{array}
\right)  \right) \nonumber\\
&  =\dim\left(  \operatorname*{Ker}\left(  A_{1}\right)  \right)  +\dim\left(
\operatorname*{Ker}\left(  A_{2}\right)  \right)  +\cdots+\dim\left(
\operatorname*{Ker}\left(  A_{u}\right)  \right)  .
\label{eq.prop.blockmatrix.nullity-diag.clm}%
\end{align}

\end{proposition}

\begin{proof}
Let $\mathbb{F}$ be the field that our matrices are defined over. If
$v_{\left\langle 1\right\rangle },v_{\left\langle 2\right\rangle }%
,\ldots,v_{\left\langle u\right\rangle }$ are $u$ column vectors (of whatever
sizes), then $\left(
\begin{array}
[c]{c}%
v_{\left\langle 1\right\rangle }\\
v_{\left\langle 2\right\rangle }\\
\vdots\\
v_{\left\langle u\right\rangle }%
\end{array}
\right)  $ shall mean the big column vector obtained by stacking these $u$
column vectors $v_{\left\langle 1\right\rangle },v_{\left\langle
2\right\rangle },\ldots,v_{\left\langle u\right\rangle }$ atop one another.
(This is the particular case of the block matrix notation from Definition
\ref{def.blockmatrix.uxv} for $v=1$ and $p_{1}=1$.) It is easy to see (e.g.,
using Proposition \ref{prop.blockmatrix.mult-uxv}) that if $v_{\left\langle
1\right\rangle },v_{\left\langle 2\right\rangle },\ldots,v_{\left\langle
u\right\rangle }$ are $u$ column vectors with $v_{\left\langle i\right\rangle
}\in\mathbb{F}^{n_{i}}$ for each $i\in\left[  u\right]  $, then
\begin{equation}
\left(
\begin{array}
[c]{cccc}%
A_{1} & 0 & \cdots & 0\\
0 & A_{2} & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & A_{u}%
\end{array}
\right)  \left(
\begin{array}
[c]{c}%
v_{\left\langle 1\right\rangle }\\
v_{\left\langle 2\right\rangle }\\
\vdots\\
v_{\left\langle u\right\rangle }%
\end{array}
\right)  =\left(
\begin{array}
[c]{c}%
A_{1}v_{\left\langle 1\right\rangle }\\
A_{2}v_{\left\langle 2\right\rangle }\\
\vdots\\
A_{u}v_{\left\langle u\right\rangle }%
\end{array}
\right)  . \label{pf.prop.blockmatrix.nullity-diag.1}%
\end{equation}


Let $N:=n_{1}+n_{2}+\cdots+n_{u}$. Any vector $v\in\mathbb{F}^{N}$ can be
uniquely written in block-matrix notation as $\left(
\begin{array}
[c]{c}%
v_{\left\langle 1\right\rangle }\\
v_{\left\langle 2\right\rangle }\\
\vdots\\
v_{\left\langle u\right\rangle }%
\end{array}
\right)  $, where each $v_{\left\langle i\right\rangle }$ is a vector in
$\mathbb{F}^{n_{i}}$. (To wit, we just subdivide $v$ into blocks of sizes
$n_{1},n_{2},\ldots,n_{u}$ from top to bottom; the topmost block will be
$v_{\left\langle 1\right\rangle }$, the second-topmost will be
$v_{\left\langle 2\right\rangle }$, and so on. Formally speaking, for each
$i\in\left[  u\right]  $, we set $N_{i}:=n_{1}+n_{2}+\cdots+n_{i-1}$, and we
let $v_{\left\langle i\right\rangle }$ be the column vector in $\mathbb{F}%
^{n_{i}}$ whose entries are the $\left(  N_{i}+1\right)  $-st, $\left(
N_{i}+2\right)  $-nd, $\ldots$, $\left(  N_{i}+n_{i}\right)  $-th entries of
$v$.)

Now, consider a vector $v\in\mathbb{F}^{N}$ that is written in block-matrix
notation $\left(
\begin{array}
[c]{c}%
v_{\left\langle 1\right\rangle }\\
v_{\left\langle 2\right\rangle }\\
\vdots\\
v_{\left\langle u\right\rangle }%
\end{array}
\right)  $, where each $v_{\left\langle i\right\rangle }$ is a vector in
$\mathbb{F}^{n_{i}}$. Then,
\begin{align*}
\left(
\begin{array}
[c]{cccc}%
A_{1} & 0 & \cdots & 0\\
0 & A_{2} & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & A_{u}%
\end{array}
\right)  v  &  =\left(
\begin{array}
[c]{cccc}%
A_{1} & 0 & \cdots & 0\\
0 & A_{2} & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & A_{u}%
\end{array}
\right)  \left(
\begin{array}
[c]{c}%
v_{\left\langle 1\right\rangle }\\
v_{\left\langle 2\right\rangle }\\
\vdots\\
v_{\left\langle u\right\rangle }%
\end{array}
\right) \\
&  =\left(
\begin{array}
[c]{c}%
A_{1}v_{\left\langle 1\right\rangle }\\
A_{2}v_{\left\langle 2\right\rangle }\\
\vdots\\
A_{u}v_{\left\langle u\right\rangle }%
\end{array}
\right)  \ \ \ \ \ \ \ \ \ \ \left(  \text{by
(\ref{pf.prop.blockmatrix.nullity-diag.1})}\right)  .
\end{align*}
Hence, $\left(
\begin{array}
[c]{cccc}%
A_{1} & 0 & \cdots & 0\\
0 & A_{2} & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & A_{u}%
\end{array}
\right)  v=0$ holds if and only if $A_{i}v_{\left\langle i\right\rangle }=0$
holds for each $i\in\left[  u\right]  $. In other words, $v\in
\operatorname*{Ker}\left(
\begin{array}
[c]{cccc}%
A_{1} & 0 & \cdots & 0\\
0 & A_{2} & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & A_{u}%
\end{array}
\right)  $ holds if and only if $v_{\left\langle i\right\rangle }%
\in\operatorname*{Ker}\left(  A_{i}\right)  $ holds for each $i\in\left[
u\right]  $. In other words, the vectors in $\operatorname*{Ker}\left(
\begin{array}
[c]{cccc}%
A_{1} & 0 & \cdots & 0\\
0 & A_{2} & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & A_{u}%
\end{array}
\right)  $ are precisely the vectors of the form $\left(
\begin{array}
[c]{c}%
v_{\left\langle 1\right\rangle }\\
v_{\left\langle 2\right\rangle }\\
\vdots\\
v_{\left\langle u\right\rangle }%
\end{array}
\right)  $, where $v_{\left\langle i\right\rangle }\in\operatorname*{Ker}%
\left(  A_{i}\right)  $ for each $i\in\left[  u\right]  $. Thus,%
\[
\operatorname*{Ker}\left(
\begin{array}
[c]{cccc}%
A_{1} & 0 & \cdots & 0\\
0 & A_{2} & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & A_{u}%
\end{array}
\right)  \cong\operatorname*{Ker}\left(  A_{1}\right)  \oplus
\operatorname*{Ker}\left(  A_{2}\right)  \oplus\cdots\oplus\operatorname*{Ker}%
\left(  A_{u}\right)
\]
as vector spaces. By taking dimensions on both sides, this yields
(\ref{eq.prop.blockmatrix.nullity-diag.clm}).
\end{proof}

\subsubsection{Unitarity}

Now, we claim that a block-diagonal matrix is unitary if and only if its
diagonal blocks are unitary:

\begin{proposition}
\label{prop.blockmatrix.unitary-diag}Let $u\in\mathbb{N}$. Let $n_{1}%
,n_{2},\ldots,n_{u}\in\mathbb{N}$. For each $i\in\left[  u\right]  $, let
$A_{i}\in\mathbb{C}^{n_{i}\times n_{i}}$ be a matrix. Then, the block-diagonal
matrix $\left(
\begin{array}
[c]{cccc}%
A_{1} & 0 & \cdots & 0\\
0 & A_{2} & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & A_{u}%
\end{array}
\right)  $ is unitary if and only if all $u$ matrices $A_{1},A_{2}%
,\ldots,A_{u}$ are unitary.
\end{proposition}

\begin{proof}
Let $N=n_{1}+n_{2}+\cdots+n_{u}$. Let
\begin{equation}
A=\left(
\begin{array}
[c]{cccc}%
A_{1} & 0 & \cdots & 0\\
0 & A_{2} & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & A_{u}%
\end{array}
\right)  . \label{pf.prop.blockmatrix.unitary-diag.A=}%
\end{equation}
Thus, we must prove that $A$ is unitary if and only if all $u$ matrices
$A_{1},A_{2},\ldots,A_{u}$ are unitary.

It is easy to see that
\[
A^{\ast}=\left(
\begin{array}
[c]{cccc}%
A_{1}^{\ast} & 0 & \cdots & 0\\
0 & A_{2}^{\ast} & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & A_{u}^{\ast}%
\end{array}
\right)  .
\]
Multiplying this equality by (\ref{pf.prop.blockmatrix.unitary-diag.A=}), we
obtain%
\begin{align*}
A^{\ast}A  &  =\left(
\begin{array}
[c]{cccc}%
A_{1}^{\ast} & 0 & \cdots & 0\\
0 & A_{2}^{\ast} & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & A_{u}^{\ast}%
\end{array}
\right)  \left(
\begin{array}
[c]{cccc}%
A_{1} & 0 & \cdots & 0\\
0 & A_{2} & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & A_{u}%
\end{array}
\right) \\
&  =\left(
\begin{array}
[c]{cccc}%
A_{1}^{\ast}A_{1} & 0 & \cdots & 0\\
0 & A_{2}^{\ast}A_{2} & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & A_{u}^{\ast}A_{u}%
\end{array}
\right)  \ \ \ \ \ \ \ \ \ \ \left(  \text{by Corollary
\ref{cor.blockmatrix.mult-diag}}\right)  .
\end{align*}
On the other hand, it is again easy to see that
\[
I_{N}=\left(
\begin{array}
[c]{cccc}%
I_{n_{1}} & 0 & \cdots & 0\\
0 & I_{n_{2}} & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & I_{n_{u}}%
\end{array}
\right)
\]
(since $N=n_{1}+n_{2}+\cdots+n_{u}$). In light of these two equalities, we see
that $A^{\ast}A=I_{N}$ holds if and only if
\[
\left(
\begin{array}
[c]{cccc}%
A_{1}^{\ast}A_{1} & 0 & \cdots & 0\\
0 & A_{2}^{\ast}A_{2} & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & A_{u}^{\ast}A_{u}%
\end{array}
\right)  =\left(
\begin{array}
[c]{cccc}%
I_{n_{1}} & 0 & \cdots & 0\\
0 & I_{n_{2}} & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & I_{n_{u}}%
\end{array}
\right)
\]
holds, i.e., if and only if we have $A_{i}^{\ast}A_{i}=I_{n_{i}}$ for each
$i\in\left[  u\right]  $. Likewise, we can see that $AA^{\ast}=I_{N}$ holds if
and only if we have $A_{i}A_{i}^{\ast}=I_{n_{i}}$ for each $i\in\left[
u\right]  $. Hence, we have the following chain of equivalences:%
\begin{align*}
&  \ \left(  A\text{ is unitary}\right) \\
&  \Longleftrightarrow\ \left(  AA^{\ast}=I_{N}\text{ and }A^{\ast}%
A=I_{N}\right) \\
&  \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \left(  \text{by the equivalence
}\mathcal{A}\Longleftrightarrow\mathcal{C}\text{ in Theorem
\ref{thm.unitary.unitary.eqs}}\right) \\
&  \Longleftrightarrow\ \left(  \text{we have }A_{i}A_{i}^{\ast}=I_{n_{i}%
}\text{ and }A_{i}^{\ast}A_{i}=I_{n_{i}}\text{ for each }i\in\left[  u\right]
\right) \\
&  \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \left(
\begin{array}
[c]{c}%
\text{since we have shown that }AA^{\ast}=I_{N}\text{ holds if and only if}\\
\text{we have }A_{i}A_{i}^{\ast}=I_{n_{i}}\text{ for each }i\in\left[
u\right]  \text{, and since we have}\\
\text{shown that }A^{\ast}A=I_{N}\text{ holds if and only if}\\
\text{we have }A_{i}^{\ast}A_{i}=I_{n_{i}}\text{ for each }i\in\left[
u\right]
\end{array}
\right) \\
&  \Longleftrightarrow\ \left(  \text{the matrix }A_{i}\text{ is unitary for
each }i\in\left[  u\right]  \right) \\
&  \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \left(  \text{by the equivalence
}\mathcal{C}\Longleftrightarrow\mathcal{A}\text{ in Theorem
\ref{thm.unitary.unitary.eqs}}\right) \\
&  \Longleftrightarrow\ \left(  \text{all }u\text{ matrices }A_{1}%
,A_{2},\ldots,A_{u}\text{ are unitary}\right)  .
\end{align*}
But this is precisely what we need to show. Thus, Proposition
\ref{prop.blockmatrix.unitary-diag} is proven.
\end{proof}

\subsection{The Gram--Schmidt process}%

\[%
\begin{tabular}
[c]{||l||}\hline\hline
\textbf{Lecture 3 starts here.}\\\hline\hline
\end{tabular}
\ \ \
\]


We now come to one of the most crucial algorithms in linear algebra.

\begin{theorem}
[Gram--Schmidt process]\label{thm.unitary.gs}Let $\left(  v_{1},v_{2}%
,\ldots,v_{m}\right)  $ be a linearly independent tuple of vectors in
$\mathbb{C}^{n}$.

Then, there is an orthogonal tuple $\left(  z_{1},z_{2},\ldots,z_{m}\right)  $
of vectors in $\mathbb{C}^{n}$ that satisfies%
\[
\operatorname*{span}\left\{  v_{1},v_{2},\ldots,v_{j}\right\}
=\operatorname*{span}\left\{  z_{1},z_{2},\ldots,z_{j}\right\}
\ \ \ \ \ \ \ \ \ \ \text{for all }j\in\left[  m\right]  .
\]


Furthermore, such a tuple $\left(  z_{1},z_{2},\ldots,z_{m}\right)  $ can be
constructed by the following recursive process:

\begin{itemize}
\item For each $p\in\left[  m\right]  $, if the first $p-1$ entries
$z_{1},z_{2},\ldots,z_{p-1}$ of this tuple have already been constructed, then
we define the $p$-th entry $z_{p}$ by the equality%
\begin{equation}
z_{p}=v_{p}-\sum_{k=1}^{p-1}\dfrac{\left\langle v_{p},z_{k}\right\rangle
}{\left\langle z_{k},z_{k}\right\rangle }z_{k}. \label{eq.thm.unitary.gs.zp=}%
\end{equation}
(Note that the sum on the right hand side of (\ref{eq.thm.unitary.gs.zp=}) is
an empty sum when $p=1$; thus, (\ref{eq.thm.unitary.gs.zp=}) simplifies to
$z_{1}=v_{1}$ in this case.)
\end{itemize}
\end{theorem}

Roughly speaking, the claim of Theorem \ref{thm.unitary.gs} is that if we
start with any linearly independent tuple $\left(  v_{1},v_{2},\ldots
,v_{m}\right)  $ of vectors in $\mathbb{C}^{n}$, then we can make this tuple
orthogonal by tweaking it as follows:

\begin{itemize}
\item leave $v_{1}$ unchanged;

\item modify $v_{2}$ by subtracting some scalar multiple of $v_{1}$;

\item modify $v_{3}$ by subtracting some linear combination of $v_{1}$ and
$v_{2}$;

\item modify $v_{4}$ by subtracting some linear combination of $v_{1}%
,v_{2},v_{3}$;

\item and so on.
\end{itemize}

\noindent Specifically, the equation (\ref{eq.thm.unitary.gs.zp=}) tells us
(recursively) the precise multiples (and linear combinations) that we need to
subtract. This recursive tweaking process is known as \emph{Gram--Schmidt
orthogonalization} or the \emph{Gram--Schmidt process}.

\begin{example}
Here is how the equalities (\ref{eq.thm.unitary.gs.zp=}) in Theorem
\ref{thm.unitary.gs} look like for $p\in\left\{  1,2,3,4\right\}  $:%
\begin{align*}
z_{1}  &  =v_{1};\\
z_{2}  &  =v_{2}-\dfrac{\left\langle v_{2},z_{1}\right\rangle }{\left\langle
z_{1},z_{1}\right\rangle }z_{1};\\
z_{3}  &  =v_{3}-\dfrac{\left\langle v_{3},z_{1}\right\rangle }{\left\langle
z_{1},z_{1}\right\rangle }z_{1}-\dfrac{\left\langle v_{3},z_{2}\right\rangle
}{\left\langle z_{2},z_{2}\right\rangle }z_{2};\\
z_{4}  &  =v_{4}-\dfrac{\left\langle v_{4},z_{1}\right\rangle }{\left\langle
z_{1},z_{1}\right\rangle }z_{1}-\dfrac{\left\langle v_{4},z_{2}\right\rangle
}{\left\langle z_{2},z_{2}\right\rangle }z_{2}-\dfrac{\left\langle v_{4}%
,z_{3}\right\rangle }{\left\langle z_{3},z_{3}\right\rangle }z_{3}.
\end{align*}

\end{example}

\begin{example}
Let us try out the recursive construction of $\left(  z_{1},z_{2},\ldots
,z_{m}\right)  $ from Theorem \ref{thm.unitary.gs} on an example. Let $n=4$
and $m=3$ and
\[
v_{1}=\left(
\begin{array}
[c]{c}%
1\\
1\\
1\\
1
\end{array}
\right)  ,\ \ \ \ \ \ \ \ \ \ v_{2}=\left(
\begin{array}
[c]{c}%
0\\
-2\\
0\\
-2
\end{array}
\right)  ,\ \ \ \ \ \ \ \ \ \ v_{3}=\left(
\begin{array}
[c]{c}%
2\\
-2\\
0\\
0
\end{array}
\right)  .
\]
Then, (\ref{eq.thm.unitary.gs.zp=}) becomes%
\begin{align*}
z_{1}  &  =v_{1}=\left(
\begin{array}
[c]{c}%
1\\
1\\
1\\
1
\end{array}
\right)  ;\\
z_{2}  &  =v_{2}-\dfrac{\left\langle v_{2},z_{1}\right\rangle }{\left\langle
z_{1},z_{1}\right\rangle }z_{1}=\left(
\begin{array}
[c]{c}%
0\\
-2\\
0\\
-2
\end{array}
\right)  -\dfrac{-4}{4}\left(
\begin{array}
[c]{c}%
1\\
1\\
1\\
1
\end{array}
\right)  =\left(
\begin{array}
[c]{c}%
1\\
-1\\
1\\
-1
\end{array}
\right)  ;\\
z_{3}  &  =v_{3}-\dfrac{\left\langle v_{3},z_{1}\right\rangle }{\left\langle
z_{1},z_{1}\right\rangle }z_{1}-\dfrac{\left\langle v_{3},z_{2}\right\rangle
}{\left\langle z_{2},z_{2}\right\rangle }z_{2}\\
&  =\left(
\begin{array}
[c]{c}%
2\\
-2\\
0\\
0
\end{array}
\right)  -\dfrac{0}{4}\left(
\begin{array}
[c]{c}%
1\\
1\\
1\\
1
\end{array}
\right)  -\dfrac{4}{4}\left(
\begin{array}
[c]{c}%
1\\
-1\\
1\\
-1
\end{array}
\right)  =\left(
\begin{array}
[c]{c}%
1\\
-1\\
-1\\
1
\end{array}
\right)  .
\end{align*}
So
\[
\left(  z_{1},z_{2},z_{3}\right)  =\left(  \left(
\begin{array}
[c]{c}%
1\\
1\\
1\\
1
\end{array}
\right)  ,\ \ \left(
\begin{array}
[c]{c}%
1\\
-1\\
1\\
-1
\end{array}
\right)  ,\ \ \left(
\begin{array}
[c]{c}%
1\\
-1\\
-1\\
1
\end{array}
\right)  \right)
\]
is an orthogonal tuple of vectors.

According to Proposition \ref{prop.unitary.innerprod.orth-norm}, we thus
obtain an orthonormal tuple%
\[
\left(  \dfrac{1}{\left\vert \left\vert z_{1}\right\vert \right\vert }%
z_{1},\ \ \dfrac{1}{\left\vert \left\vert z_{2}\right\vert \right\vert }%
z_{2},\ \ \dfrac{1}{\left\vert \left\vert z_{3}\right\vert \right\vert }%
z_{3}\right)  =\left(  \left(
\begin{array}
[c]{c}%
1/2\\
1/2\\
1/2\\
1/2
\end{array}
\right)  ,\ \ \left(
\begin{array}
[c]{c}%
1/2\\
-1/2\\
1/2\\
-1/2
\end{array}
\right)  ,\ \ \left(
\begin{array}
[c]{c}%
1/2\\
-1/2\\
-1/2\\
1/2
\end{array}
\right)  \right)  .
\]
(We are in luck with this example; normally we would get square roots at this step.)
\end{example}

For more examples of the Gram--Schmidt process, see \cite[Week 3,
\S 4]{Bartle14}. (These examples all use vectors in $\mathbb{R}^{n}$ rather
than $\mathbb{C}^{n}$, which allows for visualization and saves one the
trouble of complex conjugates.)

Our proof of Theorem \ref{thm.unitary.gs} will require a simple lemma from
elementary linear algebra:

\begin{lemma}
\label{lem.span-last-vec-change}Let $V$ be a vector space over some field. Let
$v_{1},v_{2},\ldots,v_{k}$ be some vectors in $V$. Let $x$ and $y$ be two
further vectors in $V$. Assume that $x-y\in\operatorname*{span}\left\{
v_{1},v_{2},\ldots,v_{k}\right\}  $. Then,%
\[
\operatorname*{span}\left\{  v_{1},v_{2},\ldots,v_{k},x\right\}
=\operatorname*{span}\left\{  v_{1},v_{2},\ldots,v_{k},y\right\}  .
\]

\end{lemma}

\begin{proof}
[Proof of Lemma \ref{lem.span-last-vec-change}.]Set%
\begin{align*}
S  &  :=\operatorname*{span}\left\{  v_{1},v_{2},\ldots,v_{k}\right\}  ;\\
X  &  :=\operatorname*{span}\left\{  v_{1},v_{2},\ldots,v_{k},x\right\}  ;\\
Y  &  :=\operatorname*{span}\left\{  v_{1},v_{2},\ldots,v_{k},y\right\}  .
\end{align*}
These three sets $S$, $X$ and $Y$ are vector subspaces of $V$ (since a span is
always a vector subspace). By assumption, we have $x-y\in\operatorname*{span}%
\left\{  v_{1},v_{2},\ldots,v_{k}\right\}  =S$. Therefore, $-\left(
x-y\right)  \in S$ as well (since $S$ is a vector subspace of $V$). In other
words, $y-x\in S$ (since $-\left(  x-y\right)  =y-x$). Hence, $x$ and $y$ play
symmetric roles in our situation.

However, $x-y\in S=\operatorname*{span}\left\{  v_{1},v_{2},\ldots
,v_{k}\right\}  $ shows that $x-y$ is a linear combination of $v_{1}%
,v_{2},\ldots,v_{k}$. In other words,%
\begin{equation}
x-y=\lambda_{1}v_{1}+\lambda_{2}v_{2}+\cdots+\lambda_{k}v_{k},
\label{pf.lem.span-last-vec-change.1}%
\end{equation}
where $\lambda_{1},\lambda_{2},\ldots,\lambda_{k}$ are some scalars (i.e.,
elements of the base field). Consider these scalars. Solving the equality
(\ref{pf.lem.span-last-vec-change.1}) for $x$, we obtain%
\[
x=\lambda_{1}v_{1}+\lambda_{2}v_{2}+\cdots+\lambda_{k}v_{k}+y.
\]
This shows that $x$ is a linear combination of $v_{1},v_{2},\ldots,v_{k},y$.
In other words, $x\in\operatorname*{span}\left\{  v_{1},v_{2},\ldots
,v_{k},y\right\}  $. In other words, $x\in Y$ (since $Y=\operatorname*{span}%
\left\{  v_{1},v_{2},\ldots,v_{k},y\right\}  $). On the other hand, each
$i\in\left[  k\right]  $ satisfies%
\[
v_{i}\in\left\{  v_{1},v_{2},\ldots,v_{k},y\right\}  \subseteq
\operatorname*{span}\left\{  v_{1},v_{2},\ldots,v_{k},y\right\}  =Y.
\]
In other words, the $k$ vectors $v_{1},v_{2},\ldots,v_{k}$ belong to $Y$.
Since we also know that $x\in Y$, we thus conclude that all $k+1$ vectors
$v_{1},v_{2},\ldots,v_{k},x$ belong to $Y$. Since $Y$ is a vector subspace of
$V$, this entails that any linear combination of $v_{1},v_{2},\ldots,v_{k},x$
must belong to $Y$. In other words,%
\[
\operatorname*{span}\left\{  v_{1},v_{2},\ldots,v_{k},x\right\}  \subseteq Y
\]
(since $\operatorname*{span}\left\{  v_{1},v_{2},\ldots,v_{k},x\right\}  $ is
the set of all linear combinations of $v_{1},v_{2},\ldots,v_{k},x$). In other
words, $X\subseteq Y$ (since $X=\operatorname*{span}\left\{  v_{1}%
,v_{2},\ldots,v_{k},x\right\}  $).

However, as we explained, $x$ and $y$ play symmetric roles in our situation.
Swapping $x$ with $y$ results in the exchange of $X$ with $Y$. Thus, just as
we have proved $X\subseteq Y$, we can show that $Y\subseteq X$. Combining
these two inclusions, we obtain $X=Y$. In view of $X=\operatorname*{span}%
\left\{  v_{1},v_{2},\ldots,v_{k},x\right\}  $ and $Y=\operatorname*{span}%
\left\{  v_{1},v_{2},\ldots,v_{k},y\right\}  $, this rewrites as%
\[
\operatorname*{span}\left\{  v_{1},v_{2},\ldots,v_{k},x\right\}
=\operatorname*{span}\left\{  v_{1},v_{2},\ldots,v_{k},y\right\}  .
\]
This proves Lemma \ref{lem.span-last-vec-change}.
\end{proof}

\begin{proof}
[Proof of Theorem \ref{thm.unitary.gs}.]We define a tuple $\left(  z_{1}%
,z_{2},\ldots,z_{m}\right)  $ recursively by (\ref{eq.thm.unitary.gs.zp=}).
First, we need to show that this tuple is actually well-defined -- i.e., that
the denominators $\left\langle z_{k},z_{k}\right\rangle $ in the equality
(\ref{eq.thm.unitary.gs.zp=}) never become $0$ in the process (which would
render (\ref{eq.thm.unitary.gs.zp=}) meaningless and therefore prevent $z_{p}$
from being well-defined). Second, we need to show that the resulting tuple
does indeed satisfy
\[
\operatorname*{span}\left\{  v_{1},v_{2},\ldots,v_{j}\right\}
=\operatorname*{span}\left\{  z_{1},z_{2},\ldots,z_{j}\right\}
\ \ \ \ \ \ \ \ \ \ \text{for all }j\in\left[  m\right]  .
\]
Finally, we need to show that the resulting tuple is orthogonal.

Let us prove the first two of these three claims in lockstep, by showing the
following claim:

\begin{statement}
\textit{Claim 1:} For each $p\in\left\{  0,1,\ldots,m\right\}  $, the vectors
$z_{1},z_{2},\ldots,z_{p}$ are well-defined and satisfy%
\[
\operatorname*{span}\left\{  v_{1},v_{2},\ldots,v_{p}\right\}
=\operatorname*{span}\left\{  z_{1},z_{2},\ldots,z_{p}\right\}  .
\]

\end{statement}

[\textit{Proof of Claim 1:} We induct on $p$.

\textit{Induction base:} Claim 1 is obviously true for $p=0$ (since
$\operatorname*{span}\left\{  {}\right\}  =\operatorname*{span}\left\{
{}\right\}  $).

\textit{Induction step:} Fix some $p\in\left[  m\right]  $, and assume that
the vectors $z_{1},z_{2},\ldots,z_{p-1}$ are well-defined and satisfy
\begin{equation}
\operatorname*{span}\left\{  v_{1},v_{2},\ldots,v_{p-1}\right\}
=\operatorname*{span}\left\{  z_{1},z_{2},\ldots,z_{p-1}\right\}  .
\label{pf.thm.unitary.gs.6}%
\end{equation}
We now need to show that the vectors $z_{1},z_{2},\ldots,z_{p}$ are
well-defined and satisfy%
\begin{equation}
\operatorname*{span}\left\{  v_{1},v_{2},\ldots,v_{p}\right\}
=\operatorname*{span}\left\{  z_{1},z_{2},\ldots,z_{p}\right\}  .
\label{pf.thm.unitary.gs.5}%
\end{equation}


The tuple $\left(  v_{1},v_{2},\ldots,v_{p}\right)  $ is linearly independent
(since the tuple $\left(  v_{1},v_{2},\ldots,v_{m}\right)  $ is linearly
independent). Thus, the span $\operatorname*{span}\left\{  v_{1},v_{2}%
,\ldots,v_{p-1}\right\}  $ is $\left(  p-1\right)  $-dimensional and we have
$v_{p}\notin\operatorname*{span}\left\{  v_{1},v_{2},\ldots,v_{p-1}\right\}
$. Hence,%
\[
v_{p}\notin\operatorname*{span}\left\{  v_{1},v_{2},\ldots,v_{p-1}\right\}
=\operatorname*{span}\left\{  z_{1},z_{2},\ldots,z_{p-1}\right\}
\ \ \ \ \ \ \ \ \ \ \left(  \text{by (\ref{pf.thm.unitary.gs.6})}\right)  .
\]


Now, recall that the span $\operatorname*{span}\left\{  v_{1},v_{2}%
,\ldots,v_{p-1}\right\}  $ is $\left(  p-1\right)  $-dimensional. In view of
(\ref{pf.thm.unitary.gs.6}), we can rewrite this as follows: The span
$\operatorname*{span}\left\{  z_{1},z_{2},\ldots,z_{p-1}\right\}  $ is
$\left(  p-1\right)  $-dimensional. In other words, the tuple $\left(
z_{1},z_{2},\ldots,z_{p-1}\right)  $ is linearly independent. Hence, for each
$k\in\left[  p-1\right]  $, we have $z_{k}\neq0$ and therefore $\left\langle
z_{k},z_{k}\right\rangle >0$ (by Proposition \ref{prop.unitary.innerprod.pos}
\textbf{(b)}), so that $\left\langle z_{k},z_{k}\right\rangle \neq0$. Thus,
the denominators on the right hand side of (\ref{eq.thm.unitary.gs.zp=}) are
nonzero, so that $z_{p}$ is well-defined. Hence, the vectors $z_{1}%
,z_{2},\ldots,z_{p}$ are well-defined (since we already know that the vectors
$z_{1},z_{2},\ldots,z_{p-1}$ are well-defined).

It remains to prove that
\[
\operatorname*{span}\left\{  v_{1},v_{2},\ldots,v_{p}\right\}
=\operatorname*{span}\left\{  z_{1},z_{2},\ldots,z_{p}\right\}  .
\]
But this is easy: From (\ref{eq.thm.unitary.gs.zp=}), we obtain%
\[
v_{p}-z_{p}=\sum_{k=1}^{p-1}\dfrac{\left\langle v_{p},z_{k}\right\rangle
}{\left\langle z_{k},z_{k}\right\rangle }z_{k}\in\operatorname*{span}\left\{
z_{1},z_{2},\ldots,z_{p-1}\right\}
\]
(since $\sum_{k=1}^{p-1}\dfrac{\left\langle v_{p},z_{k}\right\rangle
}{\left\langle z_{k},z_{k}\right\rangle }z_{k}$ is clearly a linear
combination of $z_{1},z_{2},\ldots,z_{p-1}$). Hence, Lemma
\ref{lem.span-last-vec-change} (applied to $k=p-1$ and $x=v_{p}$ and $y=z_{p}%
$) yields\footnote{We are here using the following notion: If $P$ and $Q$ are
two vector subspaces of a vector space $V$, then
\[
P+Q:=\left\{  p+q\ \mid\ p\in P\text{ and }q\in Q\right\}  .
\]
This is again a vector subspace of $V$. (It is, in fact, the smallest subspace
that contains both $P$ and $Q$ as subsets.)}%
\begin{align*}
\operatorname*{span}\left\{  v_{1},v_{2},\ldots,v_{p-1},v_{p}\right\}   &
=\operatorname*{span}\left\{  v_{1},v_{2},\ldots,v_{p-1},z_{p}\right\} \\
&  =\underbrace{\operatorname*{span}\left\{  v_{1},v_{2},\ldots,v_{p-1}%
\right\}  }_{=\operatorname*{span}\left\{  z_{1},z_{2},\ldots,z_{p-1}\right\}
}+\operatorname*{span}\left\{  z_{p}\right\} \\
&  \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \left(
\begin{array}
[c]{c}%
\text{since }\operatorname*{span}\left(  A\cup B\right)  =\operatorname*{span}%
A+\operatorname*{span}B\\
\text{for any two sets }A\text{ and }B\text{ of vectors}%
\end{array}
\right) \\
&  =\operatorname*{span}\left\{  z_{1},z_{2},\ldots,z_{p-1}\right\}
+\operatorname*{span}\left\{  z_{p}\right\} \\
&  =\operatorname*{span}\left\{  z_{1},z_{2},\ldots,z_{p-1},z_{p}\right\} \\
&  \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \left(
\begin{array}
[c]{c}%
\text{since }\operatorname*{span}A+\operatorname*{span}B=\operatorname*{span}%
\left(  A\cup B\right) \\
\text{for any two sets }A\text{ and }B\text{ of vectors}%
\end{array}
\right)  .
\end{align*}
In other words, $\operatorname*{span}\left\{  v_{1},v_{2},\ldots
,v_{p}\right\}  =\operatorname*{span}\left\{  z_{1},z_{2},\ldots
,z_{p}\right\}  $. Thus, the induction step is complete, so that Claim 1 is
proved by induction.] \medskip

Claim 1 (applied to $p=m$) shows that the vectors $z_{1},z_{2},\ldots,z_{m}$
are well-defined. In other words, the tuple $\left(  z_{1},z_{2},\ldots
,z_{m}\right)  $ is well-defined. Furthermore, this tuple satisfies%
\[
\operatorname*{span}\left\{  v_{1},v_{2},\ldots,v_{j}\right\}
=\operatorname*{span}\left\{  z_{1},z_{2},\ldots,z_{j}\right\}
\ \ \ \ \ \ \ \ \ \ \text{for all }j\in\left[  m\right]
\]
(by Claim 1, applied to $p=j$). It now remains to show that this tuple is
orthogonal. We shall achieve this by showing the following claim:

\begin{statement}
\textit{Claim 2:} For any $j\in\left\{  0,1,\ldots,m\right\}  $, the tuple
$\left(  z_{1},z_{2},\ldots,z_{j}\right)  $ is orthogonal.
\end{statement}

[\textit{Proof of Claim 2:} We proceed by induction on $j$:

\textit{Induction base:} Claim 2 clearly holds for $j=0$, since the (empty)
$0$-tuple is vacuously orthogonal.

\textit{Induction step:} Let $p\in\left[  m\right]  $. Assume (as the
induction hypothesis) that Claim 2 holds for $j=p-1$. We must show that Claim
2 holds for $j=p$.

Our induction hypothesis says that Claim 2 holds for $j=p-1$. In other words,
the tuple $\left(  z_{1},z_{2},\ldots,z_{p-1}\right)  $ is orthogonal. In
other words, we have%
\begin{equation}
z_{a}\perp z_{b}\ \ \ \ \ \ \ \ \ \ \text{whenever }a,b\in\left[  p-1\right]
\text{ satisfy }a\neq b. \label{pf.thm.unitary.gs.12}%
\end{equation}
In other words, we have%
\begin{equation}
\left\langle z_{a},z_{b}\right\rangle =0\ \ \ \ \ \ \ \ \ \ \text{whenever
}a,b\in\left[  p-1\right]  \text{ satisfy }a\neq b.
\label{pf.thm.unitary.gs.12b}%
\end{equation}


We must show that Claim 2 holds for $j=p$. In other words, we must show that
the tuple $\left(  z_{1},z_{2},\ldots,z_{p}\right)  $ is orthogonal. In other
words, we must show that%
\begin{equation}
z_{a}\perp z_{b}\ \ \ \ \ \ \ \ \ \ \text{whenever }a,b\in\left[  p\right]
\text{ satisfy }a\neq b. \label{pf.thm.unitary.gs.13}%
\end{equation}
It will clearly suffice to prove (\ref{pf.thm.unitary.gs.13}) in the case when
one of $a$ and $b$ equals $p$ (because in all other cases, we have
$a,b\in\left[  p-1\right]  $, and thus $z_{a}\perp z_{b}$ follows from
(\ref{pf.thm.unitary.gs.12})).

Thus, let $a,b\in\left[  p\right]  $ satisfy $a\neq b$, and assume that one of
$a$ and $b$ equals $p$. We must prove that $z_{a}\perp z_{b}$. Proposition
\ref{prop.unitary.innerprod.orth-symm} shows that $z_{a}\perp z_{b}$ is
equivalent to $z_{b}\perp z_{a}$. Thus, $a$ and $b$ play symmetric roles in
our claim. Hence, in our proof of $z_{a}\perp z_{b}$, we can WLOG assume that
$a\leq b$ (since otherwise, we can swap $a$ with $b$). Assume this. Hence,
$a<b$ (since $a\neq b$). Thus, $a<b\leq p$, so that $a\neq p$. However, we
assumed that one of $a$ and $b$ equals $p$; hence, $b=p$ (since $a\neq p$).
Also, we have $a\in\left[  p-1\right]  $ (since $a<p$).

Now, (\ref{eq.thm.unitary.gs.zp=}) yields%
\[
\left\langle z_{p},z_{a}\right\rangle =\left\langle v_{p}-\sum_{k=1}%
^{p-1}\dfrac{\left\langle v_{p},z_{k}\right\rangle }{\left\langle z_{k}%
,z_{k}\right\rangle }z_{k},z_{a}\right\rangle =\left\langle v_{p}%
,z_{a}\right\rangle -\left\langle \sum_{k=1}^{p-1}\dfrac{\left\langle
v_{p},z_{k}\right\rangle }{\left\langle z_{k},z_{k}\right\rangle }z_{k}%
,z_{a}\right\rangle
\]
(by Proposition \ref{prop.unitary.innerprod.props} \textbf{(h)}). In view of%
\begin{align*}
&  \left\langle \sum_{k=1}^{p-1}\dfrac{\left\langle v_{p},z_{k}\right\rangle
}{\left\langle z_{k},z_{k}\right\rangle }z_{k},z_{a}\right\rangle \\
&  =\sum_{k=1}^{p-1}\dfrac{\left\langle v_{p},z_{k}\right\rangle
}{\left\langle z_{k},z_{k}\right\rangle }\left\langle z_{k},z_{a}\right\rangle
\ \ \ \ \ \ \ \ \ \ \left(  \text{by Proposition
\ref{prop.unitary.innerprod.props} \textbf{(i)}}\right) \\
&  =\sum_{\substack{k\in\left[  p-1\right]  ;\\k\neq a}}\dfrac{\left\langle
v_{p},z_{k}\right\rangle }{\left\langle z_{k},z_{k}\right\rangle
}\underbrace{\left\langle z_{k},z_{a}\right\rangle }_{\substack{=0\\\text{(by
(\ref{pf.thm.unitary.gs.13}), applied to }k\text{ and }a\\\text{instead of
}a\text{ and }b\text{)}}}+\underbrace{\dfrac{\left\langle v_{p},z_{a}%
\right\rangle }{\left\langle z_{a},z_{a}\right\rangle }\left\langle
z_{a},z_{a}\right\rangle }_{=\left\langle v_{p},z_{a}\right\rangle }\\
&  \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \left(
\begin{array}
[c]{c}%
\text{here, we have split off the addend for }k=a\\
\text{from the sum, since }a\in\left[  p-1\right]
\end{array}
\right) \\
&  =\underbrace{\sum_{\substack{k\in\left[  p-1\right]  ;\\k\neq a}%
}\dfrac{\left\langle v_{p},z_{k}\right\rangle }{\left\langle z_{k}%
,z_{k}\right\rangle }0}_{=0}+\left\langle v_{p},z_{a}\right\rangle
=\left\langle v_{p},z_{a}\right\rangle ,
\end{align*}
we can rewrite this as%
\[
\left\langle z_{p},z_{a}\right\rangle =\left\langle v_{p},z_{a}\right\rangle
-\left\langle v_{p},z_{a}\right\rangle =0.
\]
In view of $b=p$, this rewrites as $\left\langle z_{b},z_{a}\right\rangle =0$.
Thus, $z_{b}\perp z_{a}$, so that $z_{a}\perp z_{b}$ (by Proposition
\ref{prop.unitary.innerprod.orth-symm}).

As explained above, this completes our proof of the fact that Claim 2 holds
for $j=p$. Thus, the induction step is complete, and Claim 2 is proven.]
\medskip

Now, applying Claim 2 to $j=m$, we obtain that the tuple $\left(  z_{1}%
,z_{2},\ldots,z_{m}\right)  $ is orthogonal. Thus, the proof of Theorem
\ref{thm.unitary.gs} is complete.
\end{proof}

One might wonder how the Gram--Schmidt process could be adapted to a tuple
$\left(  v_{1},v_{2},\ldots,v_{m}\right)  $ of vectors that is \textbf{not}
linearly independent. The equality (\ref{eq.thm.unitary.gs.zp=}) requires the
vectors $z_{k}$ to be nonzero, since the denominators in which they appear
would be $0$ otherwise. In Theorem \ref{thm.unitary.gs}, this requirement is
indeed satisfied (as we have shown in the proof above). However, if we do not
assume $\left(  v_{1},v_{2},\ldots,v_{m}\right)  $ to be linearly independent,
then some of the $z_{k}$ can be zero, and so the construction of the following
$z_{p}$ will fail. There are several ways to adapt the process to this
complication. We will take the most stupid-sounding one: In the cases where
the equality (\ref{eq.thm.unitary.gs.zp=}) would produce a zero vector $z_{p}%
$, we opt to instead pick some nonzero vector orthogonal to $z_{1}%
,z_{2},\ldots,z_{p-1}$ (using Lemma \ref{lem.unitary.orthog.one-more}) and
declare it to be $z_{p}$. This works well as long as $m\leq n$; here is the result:

\begin{theorem}
[Gram--Schmidt process, take 2]\label{thm.unitary.gs-2}Let $\left(
v_{1},v_{2},\ldots,v_{m}\right)  $ be any tuple of vectors in $\mathbb{C}^{n}$
with $m\leq n$.

Then, there is an orthogonal tuple $\left(  z_{1},z_{2},\ldots,z_{m}\right)  $
of nonzero vectors in $\mathbb{C}^{n}$ that satisfies%
\[
\operatorname*{span}\left\{  v_{1},v_{2},\ldots,v_{j}\right\}  \subseteq
\operatorname*{span}\left\{  z_{1},z_{2},\ldots,z_{j}\right\}
\ \ \ \ \ \ \ \ \ \ \text{for all }j\in\left[  m\right]  .
\]


Furthermore, such a tuple $\left(  z_{1},z_{2},\ldots,z_{m}\right)  $ can be
constructed by the following recursive process:

\begin{itemize}
\item For each $p\in\left[  m\right]  $, if the first $p-1$ entries
$z_{1},z_{2},\ldots,z_{p-1}$ of this tuple have already been constructed, then
we define the $p$-th entry $z_{p}$ as follows:

\begin{itemize}
\item \textit{If }$v_{p}-\sum_{k=1}^{p-1}\dfrac{\left\langle v_{p}%
,z_{k}\right\rangle }{\left\langle z_{k},z_{k}\right\rangle }z_{k}\neq0$, then
we define $z_{p}$ by the equality%
\begin{equation}
z_{p}=v_{p}-\sum_{k=1}^{p-1}\dfrac{\left\langle v_{p},z_{k}\right\rangle
}{\left\langle z_{k},z_{k}\right\rangle }z_{k}.
\label{eq.thm.unitary.gs-2.zp=sum}%
\end{equation}


\item \textit{If }$v_{p}-\sum_{k=1}^{p-1}\dfrac{\left\langle v_{p}%
,z_{k}\right\rangle }{\left\langle z_{k},z_{k}\right\rangle }z_{k}=0$, then we
pick an arbitrary nonzero vector $b\in\mathbb{C}^{n}$ that is orthogonal to
each of $z_{1},z_{2},\ldots,z_{p-1}$ (indeed, such a vector $b$ exists by
Lemma \ref{lem.unitary.orthog.one-more}, because $p-1<p\leq m\leq n$), and we
set%
\begin{equation}
z_{p}=b. \label{eq.thm.unitary.gs-2.zp=b}%
\end{equation}

\end{itemize}
\end{itemize}
\end{theorem}

\begin{proof}
[Proof of Theorem \ref{thm.unitary.gs-2}.]We define a tuple $\left(
z_{1},z_{2},\ldots,z_{m}\right)  $ by the recursive process described in
Theorem \ref{thm.unitary.gs-2}. It is clear that this tuple is actually
well-defined (indeed, the vectors $z_{p}$ are nonzero by their construction,
and thus the denominators $\left\langle z_{k},z_{k}\right\rangle $ in
(\ref{eq.thm.unitary.gs-2.zp=sum}) never become $0$, because Proposition
\ref{prop.unitary.innerprod.pos} \textbf{(b)} shows that any nonzero vector
$z$ satisfies $\left\langle z,z\right\rangle \neq0$). We do, however, need to
show that the resulting tuple does indeed satisfy
\[
\operatorname*{span}\left\{  v_{1},v_{2},\ldots,v_{j}\right\}  \subseteq
\operatorname*{span}\left\{  z_{1},z_{2},\ldots,z_{j}\right\}
\ \ \ \ \ \ \ \ \ \ \text{for all }j\in\left[  m\right]  ,
\]
and that this tuple is orthogonal.

Let us prove the first of these two claims:

\begin{statement}
\textit{Claim 1:} For each $p\in\left\{  0,1,\ldots,m\right\}  $, we have
$\operatorname*{span}\left\{  v_{1},v_{2},\ldots,v_{p}\right\}  \subseteq
\operatorname*{span}\left\{  z_{1},z_{2},\ldots,z_{p}\right\}  $.
\end{statement}

[\textit{Proof of Claim 1:} We induct on $p$:

\textit{Induction base:} Claim 1 obviously holds for $p=0$.

\textit{Induction step:} Fix some $p\in\left[  m\right]  $, and assume that%
\begin{equation}
\operatorname*{span}\left\{  v_{1},v_{2},\ldots,v_{p-1}\right\}
\subseteq\operatorname*{span}\left\{  z_{1},z_{2},\ldots,z_{p-1}\right\}  .
\label{pf.thm.unitary.gs-2.4}%
\end{equation}
We now need to show that
\begin{equation}
\operatorname*{span}\left\{  v_{1},v_{2},\ldots,v_{p}\right\}  \subseteq
\operatorname*{span}\left\{  z_{1},z_{2},\ldots,z_{p}\right\}  .
\label{pf.thm.unitary.gs-2.5}%
\end{equation}


We shall first show that
\begin{equation}
v_{p}\in\operatorname*{span}\left\{  z_{1},z_{2},\ldots,z_{p}\right\}  .
\label{pf.thm.unitary.gs-2.5a}%
\end{equation}


Indeed, we recall our definition of $z_{p}$. This definition distinguishes
between two cases, depending on whether the difference $v_{p}-\sum_{k=1}%
^{p-1}\dfrac{\left\langle v_{p},z_{k}\right\rangle }{\left\langle z_{k}%
,z_{k}\right\rangle }z_{k}$ is $\neq0$ or $=0$. Let us analyze these two cases separately:

\begin{itemize}
\item \textit{Case 1:} We have $v_{p}-\sum_{k=1}^{p-1}\dfrac{\left\langle
v_{p},z_{k}\right\rangle }{\left\langle z_{k},z_{k}\right\rangle }z_{k}\neq0$.
In this case, $z_{p}$ is defined by the equality
(\ref{eq.thm.unitary.gs-2.zp=sum}). Solving this equality for $v_{p}$, we
obtain%
\[
v_{p}=z_{p}+\sum_{k=1}^{p-1}\dfrac{\left\langle v_{p},z_{k}\right\rangle
}{\left\langle z_{k},z_{k}\right\rangle }z_{k}\in\operatorname*{span}\left\{
z_{1},z_{2},\ldots,z_{p}\right\}  .
\]
Thus, (\ref{pf.thm.unitary.gs-2.5a}) is proved in Case 1.

\item \textit{Case 2:} We have $v_{p}-\sum_{k=1}^{p-1}\dfrac{\left\langle
v_{p},z_{k}\right\rangle }{\left\langle z_{k},z_{k}\right\rangle }z_{k}=0$. In
this case, we have%
\[
v_{p}=\sum_{k=1}^{p-1}\dfrac{\left\langle v_{p},z_{k}\right\rangle
}{\left\langle z_{k},z_{k}\right\rangle }z_{k}\in\operatorname*{span}\left\{
z_{1},z_{2},\ldots,z_{p-1}\right\}  \subseteq\operatorname*{span}\left\{
z_{1},z_{2},\ldots,z_{p}\right\}  .
\]
Hence, (\ref{pf.thm.unitary.gs-2.5a}) is proved in Case 2.
\end{itemize}

We have now proved (\ref{pf.thm.unitary.gs-2.5a}) in both cases. However, for
each $i\in\left[  p-1\right]  $, we have%
\begin{align*}
v_{i}  &  \in\left\{  v_{1},v_{2},\ldots,v_{p-1}\right\}  \subseteq
\operatorname*{span}\left\{  v_{1},v_{2},\ldots,v_{p-1}\right\} \\
&  \subseteq\operatorname*{span}\left\{  z_{1},z_{2},\ldots,z_{p-1}\right\}
\ \ \ \ \ \ \ \ \ \ \left(  \text{by (\ref{pf.thm.unitary.gs-2.4})}\right) \\
&  \subseteq\operatorname*{span}\left\{  z_{1},z_{2},\ldots,z_{p}\right\}  .
\end{align*}
In other words, all $p-1$ vectors $v_{1},v_{2},\ldots,v_{p-1}$ belong to
$\operatorname*{span}\left\{  z_{1},z_{2},\ldots,z_{p}\right\}  $. Since the
vector $v_{p}$ also belongs to $\operatorname*{span}\left\{  z_{1}%
,z_{2},\ldots,z_{p}\right\}  $ (by (\ref{pf.thm.unitary.gs-2.5a})), we thus
conclude that all $p$ vectors $v_{1},v_{2},\ldots,v_{p}$ belong to
$\operatorname*{span}\left\{  z_{1},z_{2},\ldots,z_{p}\right\}  $. Therefore,
each linear combination of these $p$ vectors $v_{1},v_{2},\ldots,v_{p}$ must
also belong to $\operatorname*{span}\left\{  z_{1},z_{2},\ldots,z_{p}\right\}
$ (because $\operatorname*{span}\left\{  z_{1},z_{2},\ldots,z_{p}\right\}  $
is a vector subspace of $\mathbb{C}^{n}$). In other words,
$\operatorname*{span}\left\{  v_{1},v_{2},\ldots,v_{p}\right\}  \subseteq
\operatorname*{span}\left\{  z_{1},z_{2},\ldots,z_{p}\right\}  $. Thus, the
induction step is complete, so that Claim 1 is proved by induction.] \medskip

It now remains to show that the tuple $\left(  z_{1},z_{2},\ldots
,z_{m}\right)  $ is orthogonal. We shall achieve this by showing the following claim:

\begin{statement}
\textit{Claim 2:} For any $j\in\left\{  0,1,\ldots,m\right\}  $, the tuple
$\left(  z_{1},z_{2},\ldots,z_{j}\right)  $ is orthogonal.
\end{statement}

[\textit{Proof of Claim 2:} We proceed by induction on $j$, similarly to the
proof of Claim 2 in the proof of Theorem \ref{thm.unitary.gs}. Only one minor
complication emerges in the induction step:

\textit{Induction step:} Let $p\in\left[  m\right]  $. Assume (as the
induction hypothesis) that Claim 2 holds for $j=p-1$. We must show that Claim
2 holds for $j=p$.

As in the proof of Theorem \ref{thm.unitary.gs}, we can convince ourselves
that it suffices to show that
\begin{equation}
z_{a}\perp z_{b}\ \ \ \ \ \ \ \ \ \ \text{whenever }a,b\in\left[  p\right]
\text{ satisfy }a\neq b. \label{pf.thm.unitary.gs-2.13}%
\end{equation}
Moreover, we only need to show this in the case when one of $a$ and $b$ equals
$p$ (because in all other cases, it follows from the induction hypothesis). In
other words, we only need to show that the vector $z_{p}$ is orthogonal to
each of $z_{1},z_{2},\ldots,z_{p-1}$.

Recall our definition of $z_{p}$. This definition distinguishes between two
cases, depending on whether the difference $v_{p}-\sum_{k=1}^{p-1}%
\dfrac{\left\langle v_{p},z_{k}\right\rangle }{\left\langle z_{k}%
,z_{k}\right\rangle }z_{k}$ is $\neq0$ or $=0$. In the first of these two
cases, the proof proceeds exactly as in the proof of Theorem
\ref{thm.unitary.gs}. Let us thus WLOG assume that we are in the second case.
That is, we assume that $v_{p}-\sum_{k=1}^{p-1}\dfrac{\left\langle v_{p}%
,z_{k}\right\rangle }{\left\langle z_{k},z_{k}\right\rangle }z_{k}=0$. Hence,
$z_{p}$ is defined by (\ref{eq.thm.unitary.gs-2.zp=b}), where $b$ is a nonzero
vector in $\mathbb{C}^{n}$ that is orthogonal to each of $z_{1},z_{2}%
,\ldots,z_{p-1}$. This shows that $z_{p}$ is orthogonal to each of
$z_{1},z_{2},\ldots,z_{p-1}$. But as we explained above, this is exactly what
we need to show. Thus, Claim 2 holds for $j=p$. The induction step is
complete, and Claim 2 is proved.] \medskip

Now, applying Claim 2 to $j=m$, we obtain that the tuple $\left(  z_{1}%
,z_{2},\ldots,z_{m}\right)  $ is orthogonal. Thus, the proof of Theorem
\ref{thm.unitary.gs-2} is complete.
\end{proof}

\begin{corollary}
\label{cor.unitary.gs-2nor}Let $\left(  v_{1},v_{2},\ldots,v_{m}\right)  $ be
any tuple of vectors in $\mathbb{C}^{n}$ with $m\leq n$.

Then, there is an orthonormal tuple $\left(  q_{1},q_{2},\ldots,q_{m}\right)
$ of vectors in $\mathbb{C}^{n}$ that satisfies%
\[
\operatorname*{span}\left\{  v_{1},v_{2},\ldots,v_{j}\right\}  \subseteq
\operatorname*{span}\left\{  q_{1},q_{2},\ldots,q_{j}\right\}
\ \ \ \ \ \ \ \ \ \ \text{for all }j\in\left[  m\right]  .
\]

\end{corollary}

\begin{proof}
[Proof of Corollary \ref{cor.unitary.gs-2nor}.]We have $m\leq n$. Hence,
Theorem \ref{thm.unitary.gs-2} shows that there is an orthogonal tuple
$\left(  z_{1},z_{2},\ldots,z_{m}\right)  $ of nonzero vectors in
$\mathbb{C}^{n}$ that satisfies%
\[
\operatorname*{span}\left\{  v_{1},v_{2},\ldots,v_{j}\right\}  \subseteq
\operatorname*{span}\left\{  z_{1},z_{2},\ldots,z_{j}\right\}
\ \ \ \ \ \ \ \ \ \ \text{for all }j\in\left[  m\right]  .
\]
Consider this tuple $\left(  z_{1},z_{2},\ldots,z_{m}\right)  $. Proposition
\ref{prop.unitary.innerprod.orth-norm} (applied to $\left(  z_{1},z_{2}%
,\ldots,z_{m}\right)  $ instead of $\left(  u_{1},u_{2},\ldots,u_{k}\right)
$) then shows that the tuple%
\[
\left(  \dfrac{1}{\left\vert \left\vert z_{1}\right\vert \right\vert }%
z_{1},\ \ \dfrac{1}{\left\vert \left\vert z_{2}\right\vert \right\vert }%
z_{2},\ \ \ldots,\ \ \dfrac{1}{\left\vert \left\vert z_{m}\right\vert
\right\vert }z_{m}\right)
\]
is orthonormal. Moreover, we have
\[
\operatorname*{span}\left\{  v_{1},v_{2},\ldots,v_{j}\right\}  \subseteq
\operatorname*{span}\left\{  z_{1},z_{2},\ldots,z_{j}\right\}
=\operatorname*{span}\left\{  \dfrac{1}{\left\vert \left\vert z_{1}\right\vert
\right\vert }z_{1},\ \ \dfrac{1}{\left\vert \left\vert z_{2}\right\vert
\right\vert }z_{2},\ \ \ldots,\ \ \dfrac{1}{\left\vert \left\vert
z_{j}\right\vert \right\vert }z_{j}\right\}
\]
for all $j\in\left[  m\right]  $. Hence, Corollary \ref{cor.unitary.gs-2nor}
is proven (just take $q_{i}=\dfrac{1}{\left\vert \left\vert z_{i}\right\vert
\right\vert }z_{i}$).
\end{proof}

\subsection{QR factorization}

Recall that an isometry is a matrix whose columns form an orthonormal tuple.
(We saw this in Proposition \ref{prop.unitary.innerprod.isometry.2}.)

\begin{theorem}
[QR factorization, isometry version]\label{thm.unitary.QR1}Let $A\in
\mathbb{C}^{n\times m}$ satisfy $n\geq m$. Then, there exist an isometry
$Q\in\mathbb{C}^{n\times m}$ and an upper-triangular matrix $R\in
\mathbb{C}^{m\times m}$ such that $A=QR$.
\end{theorem}

The pair $\left(  Q,R\right)  $ in Theorem \ref{thm.unitary.QR1} is called a
\emph{QR factorization} of $A$. (We are using the indefinite article, since it
is usually not unique.)

\begin{example}
Let%
\[
A=\left(
\begin{array}
[c]{cccc}%
1 & 0 & 1 & 2\\
1 & -2 & 0 & 2\\
1 & 0 & 1 & 0\\
1 & -2 & 0 & 0
\end{array}
\right)  \in\mathbb{C}^{4\times4}.
\]
Then, one QR factorization of $A$ is given by%
\[
A=\underbrace{\left(
\begin{array}
[c]{cccc}%
1/2 & 1/2 & 1/2 & 1/2\\
1/2 & -1/2 & 1/2 & -1/2\\
1/2 & 1/2 & -1/2 & -1/2\\
1/2 & -1/2 & -1/2 & 1/2
\end{array}
\right)  }_{=Q}\underbrace{\left(
\begin{array}
[c]{cccc}%
2 & -2 & 1 & 0\\
0 & 2 & 1 & 2\\
0 & 0 & 0 & 0\\
0 & 0 & 0 & 2
\end{array}
\right)  }_{=R}.
\]
Another is given by%
\[
A=\underbrace{\left(
\begin{array}
[c]{cccc}%
1/2 & 1/2 & \sqrt{2}/2 & 0\\
1/2 & -1/2 & 0 & \sqrt{2}/2\\
1/2 & 1/2 & -\sqrt{2}/2 & 0\\
1/2 & -1/2 & 0 & -\sqrt{2}/2
\end{array}
\right)  }_{=Q}\underbrace{\left(
\begin{array}
[c]{cccc}%
2 & -2 & 1 & 2\\
0 & 2 & 1 & 0\\
0 & 0 & 0 & \sqrt{2}\\
0 & 0 & 0 & \sqrt{2}%
\end{array}
\right)  }_{=R}.
\]

\end{example}

\begin{proof}
[Proof of Theorem \ref{thm.unitary.QR1}.]Recall that $A_{\bullet,1}%
,A_{\bullet,2},\ldots,A_{\bullet,m}$ denote the $m$ columns of the matrix $A$.
We have $m\leq n$ (since $n\geq m$). Hence, applying Corollary
\ref{cor.unitary.gs-2nor} to $\left(  v_{1},v_{2},\ldots,v_{m}\right)
=\left(  A_{\bullet,1},A_{\bullet,2},\ldots,A_{\bullet,m}\right)  $, we
conclude that there is an orthonormal tuple $\left(  q_{1},q_{2},\ldots
,q_{m}\right)  $ of vectors in $\mathbb{C}^{n}$ that satisfies%
\begin{align}
\operatorname*{span}\left\{  A_{\bullet,1},A_{\bullet,2},\ldots,A_{\bullet
,j}\right\}   &  \subseteq\operatorname*{span}\left\{  q_{1},q_{2}%
,\ldots,q_{j}\right\} \label{pf.thm.unitary.QR1.spansequal}\\
\ \ \ \ \ \ \ \ \ \ \text{for all }j  &  \in\left[  m\right]  .\nonumber
\end{align}
Consider this tuple $\left(  q_{1},q_{2},\ldots,q_{m}\right)  $. Let
$Q\in\mathbb{C}^{n\times m}$ be the matrix whose columns are $q_{1}%
,q_{2},\ldots,q_{m}$. Then, $Q$ is an isometry (by Proposition
\ref{prop.unitary.innerprod.isometry.2}, since its columns form an orthonormal
tuple). The definition of $Q$ shows that%
\begin{equation}
Q_{\bullet,i}=q_{i}\ \ \ \ \ \ \ \ \ \ \text{for each }i\in\left[  m\right]  .
\label{pf.thm.unitary.QR1.Qbull}%
\end{equation}


Now, let $j\in\left[  m\right]  $. Then,%
\[
A_{\bullet,j}\in\operatorname*{span}\left\{  A_{\bullet,1},A_{\bullet
,2},\ldots,A_{\bullet,j}\right\}  \subseteq\operatorname*{span}\left\{
q_{1},q_{2},\ldots,q_{j}\right\}  \ \ \ \ \ \ \ \ \ \ \left(  \text{by
(\ref{pf.thm.unitary.QR1.spansequal})}\right)  .
\]
In other words, there exist scalars $r_{1,j},r_{2,j},\ldots,r_{j,j}%
\in\mathbb{C}$ such that $A_{\bullet,j}=\sum_{i=1}^{j}r_{i,j}q_{i}$. Consider
these scalars $r_{1,j},r_{2,j},\ldots,r_{j,j}$. Also, set
\begin{equation}
r_{i,j}=0\ \ \ \ \ \ \ \ \ \ \text{for each integer }i>j.
\label{pf.thm.unitary.QR1.triang}%
\end{equation}
Thus,%
\begin{equation}
A_{\bullet,j}=\sum_{i=1}^{j}r_{i,j}q_{i}=\sum_{i=1}^{m}r_{i,j}q_{i}
\label{pf.thm.unitary.QR1.col=}%
\end{equation}
(since $\sum_{i=1}^{m}r_{i,j}q_{i}=\sum_{i=1}^{j}r_{i,j}q_{i}+\sum_{i=j+1}%
^{m}\underbrace{r_{i,j}}_{\substack{=0\\\text{(by
(\ref{pf.thm.unitary.QR1.triang}))}}}q_{i}=\sum_{i=1}^{j}r_{i,j}q_{i}$).

Forget that we fixed $j$. Thus, for each $j\in\left[  m\right]  $, we have
defined scalars $r_{1,j},r_{2,j},r_{3,j},\ldots\in\mathbb{C}$ that satisfy
(\ref{pf.thm.unitary.QR1.triang}) and (\ref{pf.thm.unitary.QR1.col=}).

Now, let $R\in\mathbb{C}^{m\times m}$ be the $m\times m$-matrix whose $\left(
i,j\right)  $-th entry is $r_{i,j}$ for each $i,j\in\left[  m\right]  $. This
matrix $R$ is upper-triangular, because of (\ref{pf.thm.unitary.QR1.triang}).
The definition of $R$ yields\footnote{Recall that $M_{i,j}$ is our general
notation for the $\left(  i,j\right)  $-th entry of a matrix $M$.}%
\begin{equation}
R_{i,j}=r_{i,j}\ \ \ \ \ \ \ \ \ \ \text{for all }i,j\in\left[  m\right]  .
\label{pf.thm.unitary.QR1.Rij=}%
\end{equation}
Furthermore, for each $j\in\left[  m\right]  $, we have%
\begin{align*}
A_{\bullet,j}  &  =\sum_{i=1}^{m}\underbrace{r_{i,j}}_{\substack{=R_{i,j}%
\\\text{(by (\ref{pf.thm.unitary.QR1.Rij=}))}}}\ \ \underbrace{q_{i}%
}_{\substack{=Q_{\bullet,i}\\\text{(by (\ref{pf.thm.unitary.QR1.Qbull}))}%
}}\ \ \ \ \ \ \ \ \ \ \left(  \text{by (\ref{pf.thm.unitary.QR1.col=})}\right)
\\
&  =\sum_{i=1}^{m}R_{i,j}Q_{\bullet,i}=\left(  QR\right)  _{\bullet,j}%
\end{align*}
(by the definition of the product of two matrices\footnote{Actually, let's be
a bit more explicit here: The standard definition of the product of two
matrices yields%
\[
\left(  QR\right)  _{k,j}=\sum_{i=1}^{m}\underbrace{Q_{k,i}R_{i,j}}%
_{=R_{i,j}Q_{k,i}}=\sum_{i=1}^{m}R_{i,j}Q_{k,i}\ \ \ \ \ \ \ \ \ \ \text{for
each }k\in\left[  n\right]  .
\]
In other words, $\left(  QR\right)  _{\bullet,j}=\sum_{i=1}^{m}R_{i,j}%
Q_{\bullet,i}$, which is precisely what we are claiming.}). In other words,
$A=QR$.

Thus, we have found an isometry $Q\in\mathbb{C}^{n\times m}$ and an
upper-triangular matrix $R\in\mathbb{C}^{m\times m}$ such that $A=QR$. This
proves Theorem \ref{thm.unitary.QR1}.
\end{proof}

\begin{exercise}
\label{exe.unitary.QR2-uni}\fbox{4} Let $A\in\mathbb{C}^{n\times m}$ satisfy
$n\geq m$ and $\operatorname*{rank}A=m$. Prove that there exists exactly one
QR factorization $\left(  Q,R\right)  $ of $A$ such that the diagonal entries
of $R$ are positive reals.
\end{exercise}

Note that there are other variants of QR factorization, such as the following one:

\begin{theorem}
[QR factorization, unitary version]\label{thm.unitary.QR2}Let $A\in
\mathbb{C}^{n\times m}$. Then, there exist a unitary matrix $Q\in
\mathbb{C}^{n\times n}$ and an upper-triangular matrix $R\in\mathbb{C}%
^{n\times m}$ such that $A=QR$. Here, a rectangular matrix $R\in
\mathbb{C}^{n\times m}$ is said to be \emph{upper-triangular} if and only if
it satisfies%
\[
R_{i,j}=0\ \ \ \ \ \ \ \ \ \ \text{for all }i>j.
\]

\end{theorem}

\begin{exercise}
\label{exe.unitary.QR2}\fbox{5} Prove Theorem \ref{thm.unitary.QR2}.

[\textbf{Hint:} Reduce both cases $n>m$ and $n<m$ to the case $n=m$.]
\end{exercise}

\section{Schur triangularization (\cite[Chapter 2]{HorJoh13})}

In this chapter, we will meet \emph{Schur triangularization}: a way to
transform an arbitrary $n\times n$-matrix with complex entries into an
upper-triangular matrix by conjugating it (i.e., multiplying it by an
invertible matrix $W$ on the left and simultaneously by its inverse $W^{-1}$
on the right). This is both of theoretical and of practical significance, but
we will focus on the theoretical applications.

Before we get to Schur triangularization, we will have to set some groundwork.\setcounter{subsection}{-1}

\subsection{Reminders on the characteristic polynomial and eigenvalues}

First, let us recall some properties of the characteristic polynomial of an
$n\times n$-matrix $A$, starting with its definition:

\begin{definition}
\label{def.schurtri.ch.pA}Let $\mathbb{F}$ be a field. Let $A\in
\mathbb{F}^{n\times n}$ be an $n\times n$-matrix over $\mathbb{F}$.

The \emph{characteristic polynomial} of $A$ is defined to be the polynomial%
\[
\det\left(  tI_{n}-A\right)  \ \ \ \ \ \ \ \ \ \ \text{in the indeterminate
}t\text{ with coefficients in }\mathbb{F}.
\]
(Note that $tI_{n}-A$ is an $n\times n$-matrix whose entries are polynomials
in $t$. Thus, its determinant $\det\left(  tI_{n}-A\right)  $ is itself a
polynomial in $t$.)

The characteristic polynomial of $A$ is denoted by $p_{A}$.
\end{definition}

\begin{example}
\label{exa.schurtri.ch.pA.2x2}Let $n=2$ and $A=\left(
\begin{array}
[c]{cc}%
a & b\\
c & d
\end{array}
\right)  $. Then,%
\begin{align*}
tI_{n}-A  &  =tI_{2}-A=t\left(
\begin{array}
[c]{cc}%
1 & 0\\
0 & 1
\end{array}
\right)  -\left(
\begin{array}
[c]{cc}%
a & b\\
c & d
\end{array}
\right) \\
&  =\left(
\begin{array}
[c]{cc}%
t & 0\\
0 & t
\end{array}
\right)  -\left(
\begin{array}
[c]{cc}%
a & b\\
c & d
\end{array}
\right)  =\left(
\begin{array}
[c]{cc}%
t-a & -b\\
-c & t-d
\end{array}
\right)  ,
\end{align*}
so that%
\begin{align*}
p_{A}  &  =\det\left(  tI_{n}-A\right)  =\det\left(
\begin{array}
[c]{cc}%
t-a & -b\\
-c & t-d
\end{array}
\right)  =\left(  t-a\right)  \left(  t-d\right)  -\left(  -b\right)  \left(
-c\right) \\
&  =t^{2}-\left(  a+d\right)  t+\left(  ad-bc\right)  .
\end{align*}

\end{example}

\begin{example}
\label{exa.schurtri.ch.pA.3x3}Let $n=3$ and $A=\left(
\begin{array}
[c]{ccc}%
a & b & c\\
a^{\prime} & b^{\prime} & c^{\prime}\\
a^{\prime\prime} & b^{\prime\prime} & c^{\prime\prime}%
\end{array}
\right)  $. Then,%
\begin{align*}
tI_{n}-A  &  =tI_{3}-A=t\left(
\begin{array}
[c]{ccc}%
1 & 0 & 0\\
0 & 1 & 0\\
0 & 0 & 1
\end{array}
\right)  -\left(
\begin{array}
[c]{ccc}%
a & b & c\\
a^{\prime} & b^{\prime} & c^{\prime}\\
a^{\prime\prime} & b^{\prime\prime} & c^{\prime\prime}%
\end{array}
\right) \\
&  =\left(
\begin{array}
[c]{ccc}%
t-a & -b & -c\\
-a^{\prime} & t-b^{\prime} & -c^{\prime}\\
-a^{\prime\prime} & -b^{\prime\prime} & t-c^{\prime\prime}%
\end{array}
\right)  ,
\end{align*}
so that%
\begin{align*}
p_{A}  &  =\det\left(  tI_{n}-A\right)  =\det\left(
\begin{array}
[c]{ccc}%
t-a & -b & -c\\
-a^{\prime} & t-b^{\prime} & -c^{\prime}\\
-a^{\prime\prime} & -b^{\prime\prime} & t-c^{\prime\prime}%
\end{array}
\right) \\
&  =t^{3}-\left(  a+b^{\prime}+c^{\prime\prime}\right)  t^{2}+\left(
ab^{\prime}-ba^{\prime}+ac^{\prime\prime}-ca^{\prime\prime}+b^{\prime
}c^{\prime\prime}-b^{\prime\prime}c^{\prime}\right)  t\\
&  \ \ \ \ \ \ \ \ \ \ -\left(  ab^{\prime}c^{\prime\prime}-ab^{\prime\prime
}c^{\prime}-ba^{\prime}c^{\prime\prime}+ba^{\prime\prime}c^{\prime}%
+ca^{\prime}b^{\prime\prime}-ca^{\prime\prime}b^{\prime}\right)  .
\end{align*}

\end{example}

\begin{example}
If $n=1$ and $A=\left(
\begin{array}
[c]{c}%
a
\end{array}
\right)  $, then $tI_{n}-A=\left(
\begin{array}
[c]{c}%
t-a
\end{array}
\right)  $ and thus $p_{A}=t-a$.
\end{example}

\begin{example}
If $n=0$ and $A=\left(  {}\right)  $, then $tI_{n}-A=\left(  {}\right)  $ and
thus $p_{A}=1$ (since the determinant of the $0\times0$-matrix $\left(
{}\right)  $ is defined to be $1$).
\end{example}

\begin{remark}
\label{rmk.schurtri.ch.conventions}\textbf{(a)} Some authors define the
characteristic polynomial $p_{A}$ of an $n\times n$-matrix $A$ to be
$\det\left(  A-tI_{n}\right)  $ instead of $\det\left(  tI_{n}-A\right)  $.
This differs from our definition only by a factor of $\left(  -1\right)  ^{n}%
$, which is immaterial for most properties of the characteristic polynomial
but still can cause the occasional confusion. \medskip

\textbf{(b)} Some other common notations for $p_{A}$ are $\chi_{A}$ and
$c_{A}$.
\end{remark}

The patterns you might have spotted in Example \ref{exa.schurtri.ch.pA.2x2}
and in Example \ref{exa.schurtri.ch.pA.3x3} are not accidental. Indeed, the
coefficients of the characteristic polynomial of any square matrix can be
expressed explicitly, if you consider sums of determinants to be explicit:

\begin{proposition}
\label{prop.schurtri.ch.props}Let $\mathbb{F}$ be a field. Let $A\in
\mathbb{F}^{n\times n}$ be an $n\times n$-matrix over $\mathbb{F}$. \medskip

\textbf{(a)} The characteristic polynomial $p_{A}$ is a monic polynomial in
$t$ of degree $n$. (That is, its leading term is $t^{n}$.) \medskip

\textbf{(b)} The constant term of the polynomial $p_{A}$ is $\left(
-1\right)  ^{n}\det A$. \medskip

\textbf{(c)} The $t^{n-1}$-coefficient of the polynomial $p_{A}$ is
$-\operatorname*{Tr}A$. (Recall that $\operatorname*{Tr}A$ is defined to be
the sum of all diagonal entries of $A$; this sum is known as the \emph{trace}
of $A$.) \medskip

\textbf{(d)} More generally: Let $k\in\left\{  0,1,\ldots,n\right\}  $. Then,
the $t^{n-k}$-coefficient of the polynomial $p_{A}$ is $\left(  -1\right)
^{k}$ times the sum of all principal $k\times k$-minors of $A$. (Recall that a
$k\times k$\emph{-minor} of $A$ means the determinant of a $k\times
k$-submatrix of $A$. This $k\times k$-minor is said to be \emph{principal} if
the $k\times k$-submatrix is obtained by removing some $n-k$ rows and the
corresponding $n-k$ columns from $A$. For example, the principal $2\times
2$-minors of a $3\times3$-matrix $A$ are $\det\left(
\begin{array}
[c]{cc}%
A_{1,1} & A_{1,2}\\
A_{2,1} & A_{2,2}%
\end{array}
\right)  $, $\det\left(
\begin{array}
[c]{cc}%
A_{1,1} & A_{1,3}\\
A_{3,1} & A_{3,3}%
\end{array}
\right)  $ and $\det\left(
\begin{array}
[c]{cc}%
A_{2,2} & A_{2,3}\\
A_{3,2} & A_{3,3}%
\end{array}
\right)  $.) In other words, the $t^{n-k}$-coefficient of $p_{A}$ is
\[
\left(  -1\right)  ^{k}\sum_{1\leq i_{1}<i_{2}<\cdots<i_{k}\leq n}\det\left(
\operatorname*{sub}\nolimits_{i_{1},i_{2},\ldots,i_{k}}^{i_{1},i_{2}%
,\ldots,i_{k}}A\right)  ,
\]
where $\operatorname*{sub}\nolimits_{i_{1},i_{2},\ldots,i_{k}}^{i_{1}%
,i_{2},\ldots,i_{k}}A$ denotes the $k\times k$-matrix whose $\left(
u,v\right)  $-th entry is $A_{i_{u},i_{v}}$ for all $u,v\in\left[  k\right]  $.
\end{proposition}

\begin{proof}
[Proof sketch.]We have $A=\left(
\begin{array}
[c]{cccc}%
A_{1,1} & A_{1,2} & \cdots & A_{1,n}\\
A_{2,1} & A_{2,2} & \cdots & A_{2,n}\\
\vdots & \vdots & \ddots & \vdots\\
A_{n,1} & A_{n,2} & \cdots & A_{n,n}%
\end{array}
\right)  $, so that%
\begin{align*}
tI_{n}-A  &  =t\left(
\begin{array}
[c]{cccc}%
1 & 0 & \cdots & 0\\
0 & 1 & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & 1
\end{array}
\right)  -\left(
\begin{array}
[c]{cccc}%
A_{1,1} & A_{1,2} & \cdots & A_{1,n}\\
A_{2,1} & A_{2,2} & \cdots & A_{2,n}\\
\vdots & \vdots & \ddots & \vdots\\
A_{n,1} & A_{n,2} & \cdots & A_{n,n}%
\end{array}
\right) \\
&  =\left(
\begin{array}
[c]{cccc}%
t-A_{1,1} & -A_{1,2} & \cdots & -A_{1,n}\\
-A_{2,1} & t-A_{2,2} & \cdots & -A_{2,n}\\
\vdots & \vdots & \ddots & \vdots\\
-A_{n,1} & -A_{n,2} & \cdots & t-A_{n,n}%
\end{array}
\right)  .
\end{align*}
The determinant $\det\left(  tI_{n}-A\right)  $ of this matrix is a sum of
certain products of its entries\footnote{Here, we are using the \emph{Leibniz
formula} for the determinant of a matrix, which says that $\det B=\sum
_{\sigma\in S_{n}}\left(  -1\right)  ^{\sigma}B_{1,\sigma\left(  1\right)
}B_{2,\sigma\left(  2\right)  }\cdots B_{n,\sigma\left(  n\right)  }$ for each
$n\times n$-matrix $B$. (We are applying this to $B=tI_{n}-A$.)}. One of these
products is%
\begin{align*}
&  \left(  t-A_{1,1}\right)  \left(  t-A_{2,2}\right)  \cdots\left(
t-A_{n,n}\right) \\
&  =t^{n}-\left(  A_{1,1}+A_{2,2}+\cdots+A_{n,n}\right)  t^{n-1}+\left(
\text{terms with lower powers of }t\right)  .
\end{align*}
None of the other products appearing in this sum includes any power of $t$
higher than $t^{n-2}$ (because the product picks out at least two entries of
$A$ that lie outside of the main diagonal, and thus contain no $t$ whatsoever;
the remaining factors of the product contribute at most $t^{n-2}$). Hence, the
entire determinant $\det\left(  tI_{n}-A\right)  $ can be written as
\[
\det\left(  tI_{n}-A\right)  =t^{n}-\left(  A_{1,1}+A_{2,2}+\cdots
+A_{n,n}\right)  t^{n-1}+\left(  \text{terms with lower powers of }t\right)
.
\]
In other words,%
\[
p_{A}=t^{n}-\left(  A_{1,1}+A_{2,2}+\cdots+A_{n,n}\right)  t^{n-1}+\left(
\text{terms with lower powers of }t\right)
\]
(since $p_{A}=\det\left(  tI_{n}-A\right)  $). This yields parts \textbf{(a)}
and \textbf{(c)} of Proposition \ref{prop.schurtri.ch.props}.

To prove Proposition \ref{prop.schurtri.ch.props} \textbf{(b)}, we substitute
$0$ for $t$ in the polynomial identity $p_{A}=\det\left(  tI_{n}-A\right)  $.
We obtain%
\[
p_{A}\left(  0\right)  =\det\left(  0I_{n}-A\right)  =\det\left(  -A\right)
=\left(  -1\right)  ^{n}\det A.
\]
Since $p_{A}\left(  0\right)  $ is the constant term of $p_{A}$ (in fact, if
$f$ is any polynomial, then $f\left(  0\right)  $ is the constant term of
$f$), we thus conclude that the constant term of $p_{A}$ is $\left(
-1\right)  ^{n}\det A$. This proves Proposition \ref{prop.schurtri.ch.props}
\textbf{(b).}

Finally, Proposition \ref{prop.schurtri.ch.props} \textbf{(d)} can be
established through a more accurate combinatorial analysis of the products
that sum up to $\det\left(  tI_{n}-A\right)  $. See \cite[Proposition
6.4.29]{21s} for the details. (A combinatorially prepared reader might glean
the idea from Example \ref{exa.schurtri.ch.pA.3x3}.)

We note that parts \textbf{(a)}, \textbf{(b)} and \textbf{(c)} of Proposition
\ref{prop.schurtri.ch.props} can all be derived from part \textbf{(d)} as well.
\end{proof}

Next, we recall some basic notions around the eigenvalues of a matrix:

\begin{definition}
\label{def.schurtri.ch.evals}Let $\mathbb{F}$ be a field. Let $A\in
\mathbb{F}^{n\times n}$ be an $n\times n$-matrix, and let $\lambda
\in\mathbb{F}$. \medskip

\textbf{(a)} We say that $\lambda$ is an \emph{eigenvalue} of $A$ if and only
if $\det\left(  \lambda I_{n}-A\right)  =0$. In other words, $\lambda$ is an
\emph{eigenvalue} of $A$ if and only if $\lambda$ is a root of the
characteristic polynomial $p_{A}=\det\left(  tI_{n}-A\right)  $. \medskip

\textbf{(b)} The $\lambda$\emph{-eigenspace} of $A$ is defined to be the set
of all vectors $v\in\mathbb{F}^{n}$ satisfying $Av=\lambda v$. In other words,
it is the kernel $\operatorname*{Ker}\left(  \lambda I_{n}-A\right)
=\operatorname*{Ker}\left(  A-\lambda I_{n}\right)  $. Thus, it is a vector
subspace of $\mathbb{F}^{n}$. The elements of this $\lambda$-eigenspace are
called the $\lambda$\emph{-eigenvectors} of $A$ (or the \emph{eigenvectors} of
$A$ for eigenvalue $\lambda$). (Some authors exclude the zero vector $0$ from
being an eigenvector; we allow it. Thus, $0$ is a $\lambda$-eigenvector for
any $\lambda$, even if $\lambda$ is not an eigenvalue.) \medskip

\textbf{(c)} The \emph{algebraic multiplicity} of $\lambda$ as an eigenvalue
of $A$ is defined to be the multiplicity of $\lambda$ as a root of $p_{A}$.
(If $\lambda$ is not an eigenvalue of $A$, then this is $0$.) \medskip

\textbf{(d)} The \emph{geometric multiplicity} of $\lambda$ as an eigenvalue
of $A$ is defined to be $\dim\left(  \operatorname*{Ker}\left(  A-\lambda
I_{n}\right)  \right)  $. In other words, it is the dimension of the $\lambda
$-eigenspace of $A$. In other words, it is the maximum number of linearly
independent $\lambda$-eigenvectors. (If $\lambda$ is not an eigenvalue of $A$,
then this is $0$.)
\end{definition}

It can be shown that if $A\in\mathbb{F}^{n\times n}$ is a matrix and
$\lambda\in\mathbb{F}$ is arbitrary, then the geometric multiplicity of
$\lambda$ as an eigenvalue of $A$ is always $\leq$ to the algebraic
multiplicity of $\lambda$ as an eigenvalue of $A$. The two multiplicities can
be equal, but don't have to be.

\begin{example}
Let $A=\left(
\begin{array}
[c]{ccc}%
1 & 0 & 0\\
0 & 1 & 2\\
0 & 0 & 1
\end{array}
\right)  \in\mathbb{C}^{3\times3}$. Then, the only eigenvalue of $A$ is $1$.
Its algebraic multiplicity is $3$, while its geometric multiplicity is $2$.
\end{example}

\begin{theorem}
\label{thm.schurtri.ch.fta-cons}Let $A\in\mathbb{C}^{n\times n}$ be an
$n\times n$-matrix with complex entries. Then: \medskip

\textbf{(a)} Its characteristic polynomial $p_{A}$ factors into $n$ linear
terms:%
\begin{equation}
p_{A}=\left(  t-\lambda_{1}\right)  \left(  t-\lambda_{2}\right)
\cdots\left(  t-\lambda_{n}\right)  , \label{eq.schurtri.ch.pA-factors}%
\end{equation}
where $\lambda_{1},\lambda_{2},\ldots,\lambda_{n}\in\mathbb{C}$ are its roots
(with their algebraic multiplicities). \medskip

\textbf{(b)} These roots $\lambda_{1},\lambda_{2},\ldots,\lambda_{n}$ are the
eigenvalues of $A$, appearing with their algebraic multiplicities. \medskip

\textbf{(c)} The sum of the algebraic multiplicities of all eigenvalues of $A$
is $n$. \medskip

\textbf{(d)} The sum of all eigenvalues of $A$ (with their algebraic
multiplicities) is $\operatorname*{Tr}A$ (that is, the trace of $A$). \medskip

\textbf{(e)} The product of all eigenvalues of $A$ (with their algebraic
multiplicities) is $\det A$. \medskip

\textbf{(f)} If $n>0$, then the matrix $A$ has at least one eigenvalue and at
least one nonzero eigenvector.
\end{theorem}

\begin{proof}
The polynomial $p_{A}$ is a monic polynomial of degree $n$ (by Proposition
\ref{prop.schurtri.ch.props} \textbf{(a)}), and therefore factors into linear
terms (by
\href{https://en.wikipedia.org/wiki/Fundamental_theorem_of_algebra}{the
Fundamental Theorem of Algebra}). This proves Theorem
\ref{thm.schurtri.ch.fta-cons} \textbf{(a)}. \medskip

\textbf{(b)} This follows from the definition of eigenvalues and algebraic
multiplicities. \medskip

\textbf{(c)} This follows from part \textbf{(b)}. \medskip

\textbf{(d)} Let $\lambda_{1},\lambda_{2},\ldots,\lambda_{n}$ be the roots of
$p_{A}$ (with their multiplicities). Then, these roots are the eigenvalues of
$A$, appearing with their algebraic multiplicities (by Theorem
\ref{thm.schurtri.ch.fta-cons} \textbf{(b)}). Hence, their sum $\lambda
_{1}+\lambda_{2}+\cdots+\lambda_{n}$ is the sum of the eigenvalues of $A$
(with their algebraic multiplicities). On the other hand, we know from Theorem
\ref{thm.schurtri.ch.fta-cons} \textbf{(a)} that the equality
(\ref{eq.schurtri.ch.pA-factors}) holds. Comparing the coefficients of
$t^{n-1}$ on both sides of this equality, we obtain%
\begin{align*}
\left(  \text{the coefficient of }t^{n-1}\text{ in }p_{A}\right)   &  =\left(
\text{the coefficient of }t^{n-1}\text{ in }\left(  t-\lambda_{1}\right)
\left(  t-\lambda_{2}\right)  \cdots\left(  t-\lambda_{n}\right)  \right) \\
&  =-\left(  \lambda_{1}+\lambda_{2}+\cdots+\lambda_{n}\right)  .
\end{align*}
However, Proposition \ref{prop.schurtri.ch.props} \textbf{(c)} yields
\[
\left(  \text{the coefficient of }t^{n-1}\text{ in }p_{A}\right)
=-\operatorname*{Tr}A.
\]
Comparing these two equalities, we obtain $-\left(  \lambda_{1}+\lambda
_{2}+\cdots+\lambda_{n}\right)  =-\operatorname*{Tr}A$. In other words,
$\lambda_{1}+\lambda_{2}+\cdots+\lambda_{n}=\operatorname*{Tr}A$. This proves
Theorem \ref{thm.schurtri.ch.fta-cons} \textbf{(d)} (since $\lambda
_{1}+\lambda_{2}+\cdots+\lambda_{n}$ is the sum of the eigenvalues of $A$).
\medskip

\textbf{(e)} This is similar to part \textbf{(d)}, except that we have to
compare the coefficients of $t^{0}$ (instead of $t^{n-1}$) on both sides of
(\ref{eq.schurtri.ch.pA-factors}), and we have to use Proposition
\ref{prop.schurtri.ch.props} \textbf{(b)} (instead of Proposition
\ref{prop.schurtri.ch.props} \textbf{(c)}).\medskip

\textbf{(f)} Assume that $n>0$. Thus, $n\geq1$. However, Theorem
\ref{thm.schurtri.ch.fta-cons} \textbf{(b)} shows that $A$ has exactly $n$
eigenvalues, counted with algebraic multiplicities. Hence, $A$ has at least
one eigenvalue $\lambda$ (since $n\geq1$). Consider this $\lambda$. Since
$\lambda$ is an eigenvalue of $A$, we have $\det\left(  \lambda I_{n}%
-A\right)  =0$. Hence, the $n\times n$-matrix $\lambda I_{n}-A$ is singular,
so that its kernel $\operatorname*{Ker}\left(  \lambda I_{n}-A\right)  $ is
nonzero. In other words, there exists a nonzero vector $v\in
\operatorname*{Ker}\left(  \lambda I_{n}-A\right)  $. This vector $v$ must be
a $\lambda$-eigenvector of $A$ (since $v\in\operatorname*{Ker}\left(  \lambda
I_{n}-A\right)  $). Hence, the matrix $A$ has a nonzero eigenvector (namely,
$v$). This completes the proof of Theorem \ref{thm.schurtri.ch.fta-cons}
\textbf{(f)}.
\end{proof}

\begin{exercise}
\fbox{1} Let $\mathbb{F}$ be a field. Let $A\in\mathbb{F}^{n\times n}$ be any
$n\times n$-matrix. \medskip

\textbf{(a)} Prove that $p_{A^{T}}=p_{A}$, where $A^{T}$ denotes the transpose
of the matrix $A$. \medskip

\textbf{(b)} Assume that $\mathbb{F}=\mathbb{C}$. Prove that $p_{A^{\ast}%
}=\overline{p_{A}}$, where $\overline{p_{A}}$ denotes the result of replacing
all coefficients of the polynomial $p_{A}$ by their complex conjugates.
\end{exercise}

For occasional future use, let us state some properties of traces as exercises:

\begin{exercise}
\label{exe.trace.TrAB}\fbox{1} Let $\mathbb{F}$ be a field. Let $n,m\in
\mathbb{N}$. Let $A\in\mathbb{F}^{n\times m}$ and $B\in\mathbb{F}^{m\times n}$
be two matrices. Show that%
\[
\operatorname*{Tr}\left(  AB\right)  =\operatorname*{Tr}\left(  BA\right)  .
\]

\end{exercise}

\begin{exercise}
\label{exe.trace.A*A=0}\fbox{1} Let $n,m\in\mathbb{N}$. Let $A\in
\mathbb{C}^{n\times m}$ be any matrix. \medskip

\textbf{(a)} Show that
\[
\operatorname*{Tr}\left(  A^{\ast}A\right)  =\sum_{i=1}^{n}\ \ \sum_{j=1}%
^{m}\left\vert A_{i,j}\right\vert ^{2}.
\]


\textbf{(b)} Show that $\operatorname*{Tr}\left(  A^{\ast}A\right)  =0$ if and
only if $A=0$.
\end{exercise}

\begin{noncompile}%
\[%
\begin{tabular}
[c]{||l||}\hline\hline
\textbf{Old lecture 2 starts here.}\\\hline\hline
\end{tabular}
\ \ \
\]

\end{noncompile}

\subsection{\label{sec.schur.similar}Similarity of matrices}

Next, let us recall the notion of similar matrices:

\begin{definition}
\label{def.schurtri.similar.def}Let $\mathbb{F}$ be a field. Let $A$ and $B$
be two matrices in $\mathbb{F}^{n\times n}$. We say that $A$ is \emph{similar}
to $B$ if there exists an invertible matrix $W\in\mathbb{F}^{n\times n}$ such
that $B=WAW^{-1}$.

We write \textquotedblleft$A\sim B$\textquotedblright\ for \textquotedblleft%
$A$ is similar to $B$\textquotedblright.
\end{definition}

\begin{example}
The matrix $\left(
\begin{array}
[c]{cc}%
1 & 1\\
1 & 1
\end{array}
\right)  $ is similar to the matrix $\left(
\begin{array}
[c]{cc}%
2 & 0\\
0 & 0
\end{array}
\right)  $, since $\left(
\begin{array}
[c]{cc}%
2 & 0\\
0 & 0
\end{array}
\right)  =W\left(
\begin{array}
[c]{cc}%
1 & 1\\
1 & 1
\end{array}
\right)  W^{-1}$ for the invertible matrix $W=\left(
\begin{array}
[c]{cc}%
1 & 1\\
-1 & 1
\end{array}
\right)  $. In other words, we have $\left(
\begin{array}
[c]{cc}%
1 & 1\\
1 & 1
\end{array}
\right)  \sim\left(
\begin{array}
[c]{cc}%
2 & 0\\
0 & 0
\end{array}
\right)  $.
\end{example}

The relation $\sim$\ is easily seen to be an equivalence
relation:\footnote{Algebraists will recognize the relation $\sim$\ (for
matrices in $\mathbb{F}^{n\times n}$) as just being the conjugacy relation in
the ring $\mathbb{F}^{n\times n}$ of all $n\times n$-matrices. (The meaning of
the word \textquotedblleft conjugacy\textquotedblright\ here has nothing to do
with conjugates of complex numbers or with the conjugate transpose!)}

\begin{proposition}
\label{prop.schurtri.similar.eqrel}Let $\mathbb{F}$ be a field. Then: \medskip

\textbf{(a)} Any matrix $A\in\mathbb{F}^{n\times n}$ is similar to itself.
\medskip

\textbf{(b)} If $A$ and $B$ are two matrices in $\mathbb{F}^{n\times n}$ such
that $A$ is similar to $B$, then $B$ is similar to $A$. \medskip

\textbf{(c)} If $A$, $B$ and $C$ are three matrices in $\mathbb{F}^{n\times
n}$ such that $A$ is similar to $B$ and such that $B$ is similar to $C$, then
$A$ is similar to $C$.
\end{proposition}

\begin{proof}
\textbf{(a)} This follows from $A=I_{n}AI_{n}^{-1}$. \medskip

\textbf{(b)} Let $A$ and $B$ be two matrices in $\mathbb{F}^{n\times n}$ such
that $A$ is similar to $B$. Thus, there exists an invertible matrix
$W\in\mathbb{F}^{n\times n}$ such that $B=WAW^{-1}$. Consider this $W$. From
$B=WAW^{-1}$, we obtain $BW=WA$, so that $W^{-1}BW=A$. Thus, $A=W^{-1}%
B\underbrace{W}_{=\left(  W^{-1}\right)  ^{-1}}=W^{-1}B\left(  W^{-1}\right)
^{-1}$. Since $W^{-1}$ is invertible, this shows that $B$ is similar to $A$.
This proves Proposition \ref{prop.schurtri.similar.eqrel} \textbf{(b)}.
\medskip

\textbf{(c)} Let $A$, $B$ and $C$ be three matrices in $\mathbb{F}^{n\times
n}$ such that $A$ is similar to $B$ and such that $B$ is similar to $C$. Thus,
there exists an invertible matrix $U\in\mathbb{F}^{n\times n}$ such that
$B=UAU^{-1}$ (since $A$ is similar to $B$), and there exists an invertible
matrix $V\in\mathbb{F}^{n\times n}$ such that $C=VBV^{-1}$ (since $B$ is
similar to $C$). Consider these $U$ and $V$.

The matrices $V$ and $U$ are invertible. Thus, so is their product $VU$, and
its inverse is $\left(  VU\right)  ^{-1}=U^{-1}V^{-1}$. (This is the famous
\textquotedblleft socks-and-shoes rule\textquotedblright\ for inverting
products or compositions.) Now,%
\[
C=V\underbrace{B}_{=UAU^{-1}}V^{-1}=VUA\underbrace{U^{-1}V^{-1}}_{=\left(
VU\right)  ^{-1}}=VUA\left(  VU\right)  ^{-1}.
\]
In other words, $C=WAW^{-1}$ for the invertible matrix $W=VU$ (since we know
that $VU$ is invertible). This shows that $A$ is similar to $C$. This proves
Proposition \ref{prop.schurtri.similar.eqrel} \textbf{(c)}.
\end{proof}

Since the relation $\sim$\ is symmetric (by Proposition
\ref{prop.schurtri.similar.eqrel} \textbf{(b)}), we can make the following definition:

\begin{definition}
Let $\mathbb{F}$ be a field. Let $A$ and $B$ be two matrices in $\mathbb{F}%
^{n\times n}$. We say that $A$ and $B$ are \emph{similar} if $A$ is similar to
$B$ (or, equivalently, $B$ is similar to $A$).
\end{definition}

Similar matrices have a lot in common. Here is a selection of invariants:

\begin{proposition}
\label{prop.schurtri.similar.same}Let $\mathbb{F}$ be a field. Let
$A\in\mathbb{F}^{n\times n}$ and $B\in\mathbb{F}^{n\times n}$ be two similar
matrices. Then: \medskip

\textbf{(a)} The matrices $A$ and $B$ have the same rank. \medskip

\textbf{(b)} The matrices $A$ and $B$ have the same nullity. \medskip

\textbf{(c)} The matrices $A$ and $B$ have the same determinant. \medskip

\textbf{(d)} The matrices $A$ and $B$ have the same characteristic polynomial.
\medskip

\textbf{(e)} The matrices $A$ and $B$ have the same eigenvalues, with the same
algebraic multiplicities and with the same geometric multiplicities. \medskip

\textbf{(f)} For any $k\in\mathbb{N}$, the matrix $A^{k}$ is similar to
$B^{k}$. \medskip

\textbf{(g)} For any $\lambda\in\mathbb{F}$, the matrix $\lambda I_{n}-A$ is
similar to $\lambda I_{n}-B$.\medskip

\textbf{(h)} For any $\lambda\in\mathbb{F}$, the matrix $A-\lambda I_{n}$ is
similar to $B-\lambda I_{n}$.
\end{proposition}

\begin{proof}
Since $A$ is similar to $B$, there exists an invertible matrix $W\in
\mathbb{F}^{n\times n}$ such that $B=WAW^{-1}$. Consider this $W$. \medskip

\textbf{(b)} Consider the kernels\footnote{Recall that \textquotedblleft
kernel\textquotedblright\ is a synonym for \textquotedblleft
nullspace\textquotedblright.} $\operatorname*{Ker}A$ and $\operatorname*{Ker}%
B$ of $A$ and $B$. For any $v\in\operatorname*{Ker}A$, we have $Wv\in
\operatorname*{Ker}B$ (because $v\in\operatorname*{Ker}A$ implies $Av=0$, so
that $\underbrace{B}_{=WAW^{-1}}Wv=WA\underbrace{W^{-1}W}_{=I_{n}%
}v=W\underbrace{Av}_{=0}=0$ and therefore $Wv\in\operatorname*{Ker}B$). Thus,
we have found a linear map%
\begin{align*}
\operatorname*{Ker}A  &  \rightarrow\operatorname*{Ker}B,\\
v  &  \mapsto Wv.
\end{align*}
This linear map is furthermore injective (because $W$ is invertible, so that
$Wu=Wv$ entails $u=v$). Hence, we obtain $\dim\left(  \operatorname*{Ker}%
A\right)  \leq\dim\left(  \operatorname*{Ker}B\right)  $. But $A$ and $B$ play
symmetric roles in our situation (since the relation \textquotedblleft
similar\textquotedblright\ is symmetric), so that we can use the same
reasoning to obtain $\dim\left(  \operatorname*{Ker}B\right)  \leq\dim\left(
\operatorname*{Ker}A\right)  $. Combining these two inequalities, we obtain
$\dim\left(  \operatorname*{Ker}A\right)  =\dim\left(  \operatorname*{Ker}%
B\right)  $. In other words, $A$ and $B$ have the same nullity. This proves
Proposition \ref{prop.schurtri.similar.same} \textbf{(b)}. \medskip

\textbf{(a)} The rank of an $n\times n$-matrix equals $n$ minus its nullity
(by the rank-nullity theorem). Hence, two $n\times n$-matrices that have the
same nullity must also have the same rank. Thus, Proposition
\ref{prop.schurtri.similar.same} \textbf{(a)} follows from Proposition
\ref{prop.schurtri.similar.same} \textbf{(b)}. \medskip

\textbf{(c)} From $B=WAW^{-1}$, we obtain%
\begin{align*}
\det B  &  =\det\left(  WAW^{-1}\right)  =\det W\cdot\det A\cdot
\underbrace{\det\left(  W^{-1}\right)  }_{=\left(  \det W\right)  ^{-1}}\\
&  =\det W\cdot\det A\cdot\left(  \det W\right)  ^{-1}=\det A.
\end{align*}
This proves Proposition \ref{prop.schurtri.similar.same} \textbf{(c)}.
\medskip

\textbf{(d)} The characteristic polynomial of an $n\times n$-matrix $M$ is
defined to be $\det\left(  tI_{n}-M\right)  $ (where $t$ is the
indeterminate)\footnote{At least this is our definition. As we already
mentioned in Remark \ref{rmk.schurtri.ch.conventions} \textbf{(a)}, another
popular definition is $\det\left(  M-tI_{n}\right)  $. However, the two
definitions differ only in a factor of $\left(  -1\right)  ^{n}$, so they
behave almost completely the same (and our argument works equally well for
either of them).}. Thus, we must show that $\det\left(  tI_{n}-A\right)
=\det\left(  tI_{n}-B\right)  $. However, we have%
\begin{align*}
t\underbrace{I_{n}}_{\substack{=WW^{-1}\\=WI_{n}W^{-1}}}-\underbrace{B}%
_{=WAW^{-1}}  &  =\underbrace{tWI_{n}}_{=W\left(  tI_{n}\right)  }%
W^{-1}-WAW^{-1}=W\left(  tI_{n}\right)  W^{-1}-WAW^{-1}\\
&  =W\left(  tI_{n}-A\right)  W^{-1}.
\end{align*}
Thus,%
\begin{align*}
\det\left(  tI_{n}-B\right)   &  =\det\left(  W\left(  tI_{n}-A\right)
W^{-1}\right)  =\det W\cdot\det\left(  tI_{n}-A\right)  \cdot\underbrace{\det
\left(  W^{-1}\right)  }_{=\left(  \det W\right)  ^{-1}}\\
&  =\det W\cdot\det\left(  tI_{n}-A\right)  \cdot\left(  \det W\right)
^{-1}=\det\left(  tI_{n}-A\right)  .
\end{align*}
Thus, $\det\left(  tI_{n}-A\right)  =\det\left(  tI_{n}-B\right)  $, and
Proposition \ref{prop.schurtri.similar.same} \textbf{(d)} is proven. \medskip

\textbf{(e)} The eigenvalues of a matrix, with their algebraic multiplicities,
are the roots of the characteristic polynomial. Thus, from Proposition
\ref{prop.schurtri.similar.same} \textbf{(d)}, we see that the matrices $A$
and $B$ have the same eigenvalues, with the same algebraic multiplicities. It
remains to show that the geometric multiplicities are also the same.

Let $\lambda$ be an eigenvalue of $A$ (and therefore also of $B$, as we have
just seen). The geometric multiplicity of $\lambda$ as an eigenvalue of $A$ is
$\dim\left(  \operatorname*{Ker}\left(  A-\lambda I_{n}\right)  \right)  $.
Likewise, the geometric multiplicity of $\lambda$ as an eigenvalue of $B$ is
$\dim\left(  \operatorname*{Ker}\left(  B-\lambda I_{n}\right)  \right)  $.
Hence, we must show that $\dim\left(  \operatorname*{Ker}\left(  A-\lambda
I_{n}\right)  \right)  =\dim\left(  \operatorname*{Ker}\left(  B-\lambda
I_{n}\right)  \right)  $.

We have%
\begin{align*}
\underbrace{B}_{=WAW^{-1}}-\lambda\underbrace{I_{n}}_{\substack{=WW^{-1}%
\\=WI_{n}W^{-1}}}  &  =WAW^{-1}-\underbrace{\lambda WI_{n}}_{=W\left(  \lambda
I_{n}\right)  }W^{-1}=WAW^{-1}-W\left(  \lambda I_{n}\right)  W^{-1}\\
&  =W\left(  A-\lambda I_{n}\right)  W^{-1}.
\end{align*}
This shows that the matrices $A-\lambda I_{n}$ and $B-\lambda I_{n}$ are
similar. Hence, Proposition \ref{prop.schurtri.similar.same} \textbf{(b)}
shows that these two matrices $A-\lambda I_{n}$ and $B-\lambda I_{n}$ have the
same nullity. In other words, $\dim\left(  \operatorname*{Ker}\left(
A-\lambda I_{n}\right)  \right)  =\dim\left(  \operatorname*{Ker}\left(
B-\lambda I_{n}\right)  \right)  $. This is exactly what we needed to show;
thus, Proposition \ref{prop.schurtri.similar.same} \textbf{(e)} is proven.
\medskip

\textbf{(f)} Let $k\in\mathbb{N}$. We claim that
\begin{equation}
B^{k}=WA^{k}W^{-1}. \label{pf.prop.schurtri.similar.same.f.1}%
\end{equation}
Once this is proved, it will clearly follow that $A^{k}$ is similar to $B^{k}$.

One way to prove $B^{k}=WA^{k}W^{-1}$ is as follows: From $B=WAW^{-1}$, we
obtain%
\begin{align*}
B^{k}  &  =\left(  WAW^{-1}\right)  ^{k}=WA\underbrace{W^{-1}\cdot W}_{=I_{n}%
}A\underbrace{W^{-1}\cdot W}_{=I_{n}}AW^{-1}\cdot\cdots\cdot
WA\underbrace{W^{-1}\cdot W}_{=I_{n}}AW^{-1}\\
&  =W\underbrace{AA\cdots A}_{k\text{ factors}}W^{-1}%
\ \ \ \ \ \ \ \ \ \ \left(  \text{since all the }W^{-1}\cdot W\text{'s in the
middle cancel out}\right) \\
&  =WA^{k}W^{-1}.
\end{align*}
(To be precise, this works for $k\geq1$; but the case $k=0$ is trivial.)

A less handwavy proof of (\ref{pf.prop.schurtri.similar.same.f.1}) would
proceed by induction on $k$. As it is completely straightforward, I leave it
to the reader. \medskip

\textbf{(g)} Let $\lambda\in\mathbb{F}$. Then,
\begin{align*}
\lambda\underbrace{I_{n}}_{\substack{=WW^{-1}\\=WI_{n}W^{-1}}}-\underbrace{B}%
_{=WAW^{-1}}  &  =\underbrace{\lambda WI_{n}}_{=W\left(  \lambda I_{n}\right)
}W^{-1}-WAW^{-1}=W\left(  \lambda I_{n}\right)  W^{-1}-WAW^{-1}\\
&  =W\left(  \lambda I_{n}-A\right)  W^{-1}.
\end{align*}
This shows that the matrices $\lambda I_{n}-A$ and $\lambda I_{n}-B$ are
similar. Thus, Proposition \ref{prop.schurtri.similar.same} \textbf{(g)} is
proven.\medskip

\textbf{(h)} This differs from part \textbf{(g)} only in that the subtrahend
and the minuend trade places. The proof is entirely analogous to part
\textbf{(g)}.
\end{proof}

Note that neither part \textbf{(a)}, nor part \textbf{(b)}, nor part
\textbf{(c)}, nor part \textbf{(d)}, nor part \textbf{(e)} of Proposition
\ref{prop.schurtri.similar.same} is an \textquotedblleft if and only
if\textquotedblright\ statement: One can find two $n\times n$-matrices (for
sufficiently large $n$) that have the same rank, nullity, determinant,
characteristic polynomial and eigenvalues but are not similar.\footnote{Some
of these examples are easy to find: For example, the matrices $\left(
\begin{array}
[c]{cc}%
0 & 1\\
0 & 0
\end{array}
\right)  $ and $\left(
\begin{array}
[c]{cc}%
0 & 0\\
0 & 0
\end{array}
\right)  $ have the same eigenvalues with the same algebraic multiplicities,
but are not similar.} Thus, proving the similarity of two matrices is not as
easy as comparing these data. We will later learn an algorithmic way to check
whether two matrices are similar.

\begin{exercise}
\label{exe.schurtri.similar.two-4x4s}\fbox{2} Prove that the two matrices
$\left(
\begin{array}
[c]{cccc}%
0 & 1 & 0 & 0\\
0 & 0 & 0 & 0\\
0 & 0 & 0 & 1\\
0 & 0 & 0 & 0
\end{array}
\right)  $ and $\left(
\begin{array}
[c]{cccc}%
0 & 1 & 0 & 0\\
0 & 0 & 1 & 0\\
0 & 0 & 0 & 0\\
0 & 0 & 0 & 0
\end{array}
\right)  $ are not similar.
\end{exercise}

\begin{exercise}
\label{exe.schurtri.similar.unitary-inv}\fbox{3} Let $A\in\mathbb{C}^{n\times
n}$ be a matrix that is similar to some unitary matrix. Prove that $A^{-1}$ is
similar to $A^{\ast}$.
\end{exercise}

\begin{remark}
\label{rmk.schurtri.similar.endo-vp}If you are used to thinking of matrices as
linear maps, then similarity is a rather natural concept: Two $n\times
n$-matrices $A\in\mathbb{F}^{n\times n}$ and $B\in\mathbb{F}^{n\times n}$ are
similar if and only if they represent one and the same endomorphism
$f:\mathbb{F}^{n}\rightarrow\mathbb{F}^{n}$ of $\mathbb{F}^{n}$ with respect
to two (possibly different) bases of $\mathbb{F}^{n}$. To be more precise, $A$
has to represent $f$ with respect to some basis of $\mathbb{F}^{n}$, while $B$
has to represent $f$ with respect to a further basis of $\mathbb{F}^{n}$
(possibly the same, but usually not).

This fact is not hard to prove. Indeed, if $A$ and $B$ represent the same
endomorphism $f$ with respect to two bases of $\mathbb{F}^{n}$, then we have
$B=WAW^{-1}$, where $W$ is the change-of-basis matrix between these two bases.
Conversely, if $A$ and $B$ are similar, then there exists some invertible
matrix $W$ satisfying $B=WAW^{-1}$, and then $A$ and $B$ represent the same
endomorphism $f$ with respect to two bases of $\mathbb{F}^{n}$ (namely, $B$
represents the endomorphism%
\begin{align*}
\mathbb{F}^{n}  &  \rightarrow\mathbb{F}^{n},\\
v  &  \mapsto Bv
\end{align*}
with respect to the standard basis $\left(  e_{1},e_{2},\ldots,e_{n}\right)
$, whereas $A$ represents the same endomorphism with respect to the basis
$\left(  We_{1},We_{2},\ldots,We_{n}\right)  $).

Knowing this fact, many properties of similar matrices -- including all parts
of Proposition \ref{prop.schurtri.similar.same} -- become essentially trivial:
One just needs to recall that things like rank, nullity, determinant,
eigenvalues etc. are properties of the endomorphism rather than properties of
the matrix.
\end{remark}

Two diagonal matrices are similar whenever they have the same diagonal entries
up to order. In other words:

\begin{proposition}
\label{prop.schurtri.similar.diag}Let $\mathbb{F}$ be a field. Let
$n\in\mathbb{N}$. Let $\lambda_{1},\lambda_{2},\ldots,\lambda_{n}\in
\mathbb{F}$. Let $\sigma$ be a permutation of $\left[  n\right]  $ (that is, a
bijective map from $\left[  n\right]  $ to $\left[  n\right]  $). Then,%
\[
\operatorname*{diag}\left(  \lambda_{1},\lambda_{2},\ldots,\lambda_{n}\right)
\sim\operatorname*{diag}\left(  \lambda_{\sigma\left(  1\right)  }%
,\lambda_{\sigma\left(  2\right)  },\ldots,\lambda_{\sigma\left(  n\right)
}\right)  .
\]

\end{proposition}

\begin{example}
For $n=3$, Proposition \ref{prop.schurtri.similar.diag} claims that
$\operatorname*{diag}\left(  \lambda_{1},\lambda_{2},\lambda_{3}\right)
\sim\operatorname*{diag}\left(  \lambda_{\sigma\left(  1\right)  }%
,\lambda_{\sigma\left(  2\right)  },\lambda_{\sigma\left(  3\right)  }\right)
$. For example, if $\sigma$ is the permutation of $\left[  3\right]  $ that
sends $1,2,3$ to $2,3,1$, respectively, then this is saying that
$\operatorname*{diag}\left(  \lambda_{1},\lambda_{2},\lambda_{3}\right)
\sim\operatorname*{diag}\left(  \lambda_{2},\lambda_{3},\lambda_{1}\right)  $.
In other words,%
\[
\left(
\begin{array}
[c]{ccc}%
\lambda_{1} & 0 & 0\\
0 & \lambda_{2} & 0\\
0 & 0 & \lambda_{3}%
\end{array}
\right)  \sim\left(
\begin{array}
[c]{ccc}%
\lambda_{3} & 0 & 0\\
0 & \lambda_{1} & 0\\
0 & 0 & \lambda_{2}%
\end{array}
\right)  .
\]

\end{example}

\begin{proof}
[Proof of Proposition \ref{prop.schurtri.similar.diag}.]Let $P_{\sigma}%
\in\mathbb{F}^{n\times n}$ be the permutation matrix of $\sigma$ (defined as
in Example \ref{exa.unitary.unitary.exas} \textbf{(d)}, but using the field
$\mathbb{F}$ instead of $\mathbb{C}$). We recall that the $\left(  i,j\right)
$-th entry of this matrix $P_{\sigma}$ is $%
\begin{cases}
1, & \text{if }i=\sigma\left(  j\right)  ;\\
0, & \text{if }i\neq\sigma\left(  j\right)
\end{cases}
$ for any $i,j\in\left[  n\right]  $.

Now, it is easy to see that%
\begin{equation}
\operatorname*{diag}\left(  \lambda_{1},\lambda_{2},\ldots,\lambda_{n}\right)
\cdot P_{\sigma}=P_{\sigma}\cdot\operatorname*{diag}\left(  \lambda
_{\sigma\left(  1\right)  },\lambda_{\sigma\left(  2\right)  },\ldots
,\lambda_{\sigma\left(  n\right)  }\right)  .
\label{pf.prop.schurtri.similar.diag.1}%
\end{equation}


[\textit{Proof of (\ref{pf.prop.schurtri.similar.diag.1}):} It is
straightforward to see that for any $i,j\in\left[  n\right]  $, both matrices
$\operatorname*{diag}\left(  \lambda_{1},\lambda_{2},\ldots,\lambda
_{n}\right)  \cdot P_{\sigma}$ and $P_{\sigma}\cdot\operatorname*{diag}\left(
\lambda_{\sigma\left(  1\right)  },\lambda_{\sigma\left(  2\right)  }%
,\ldots,\lambda_{\sigma\left(  n\right)  }\right)  $ have the same $\left(
i,j\right)  $-th entry, namely $%
\begin{cases}
\lambda_{i}, & \text{if }i=\sigma\left(  j\right)  ;\\
0, & \text{if }i\neq\sigma\left(  j\right)  .
\end{cases}
$\ \ Thus, these two matrices are equal. This proves
(\ref{pf.prop.schurtri.similar.diag.1}).] \medskip

Since the permutation matrix $P_{\sigma}$ is invertible, we can multiply both
sides of (\ref{pf.prop.schurtri.similar.diag.1}) by $P_{\sigma}^{-1}$ from the
right, and thus we obtain
\[
\operatorname*{diag}\left(  \lambda_{1},\lambda_{2},\ldots,\lambda_{n}\right)
=P_{\sigma}\cdot\operatorname*{diag}\left(  \lambda_{\sigma\left(  1\right)
},\lambda_{\sigma\left(  2\right)  },\ldots,\lambda_{\sigma\left(  n\right)
}\right)  \cdot P_{\sigma}^{-1}.
\]
This shows that $\operatorname*{diag}\left(  \lambda_{1},\lambda_{2}%
,\ldots,\lambda_{n}\right)  \sim\operatorname*{diag}\left(  \lambda
_{\sigma\left(  1\right)  },\lambda_{\sigma\left(  2\right)  },\ldots
,\lambda_{\sigma\left(  n\right)  }\right)  $. Thus, Proposition
\ref{prop.schurtri.similar.diag} is proven.
\end{proof}

Proposition \ref{prop.schurtri.similar.diag} actually has a converse: If two
diagonal matrices are similar, then they have the same diagonal entries up to
order. This follows easily from Proposition \ref{prop.schurtri.similar.same}
\textbf{(e)}, because the diagonal entries of a diagonal matrix are its
eigenvalues (with their algebraic multiplicities).

An analogue of Proposition \ref{prop.schurtri.similar.diag} holds for
block-diagonal matrices:

\begin{proposition}
\label{prop.schurtri.similar.block-diag}Let $\mathbb{F}$ be a field. Let
$n\in\mathbb{N}$. For each $i\in\left[  n\right]  $, let $A_{i}$ be an
$n_{i}\times n_{i}$-matrix (for some $n_{i}\in\mathbb{N}$). Let $\sigma$ be a
permutation of $\left[  n\right]  $ (that is, a bijective map from $\left[
n\right]  $ to $\left[  n\right]  $). Then,%
\[
\left(
\begin{array}
[c]{cccc}%
A_{1} & 0 & \cdots & 0\\
0 & A_{2} & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & A_{n}%
\end{array}
\right)  \sim\left(
\begin{array}
[c]{cccc}%
A_{\sigma\left(  1\right)  } & 0 & \cdots & 0\\
0 & A_{\sigma\left(  2\right)  } & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & A_{\sigma\left(  n\right)  }%
\end{array}
\right)  .
\]

\end{proposition}

\begin{proof}
This can be proved similarly to how we proved Proposition
\ref{prop.schurtri.similar.diag}, except that now, instead of the permutation
matrix $P_{\sigma}$, we need to use a \textquotedblleft block permutation
matrix\textquotedblright\ $\mathbf{P}_{\sigma}$. This matrix $\mathbf{P}%
_{\sigma}$ is defined to be the matrix that is written as%
\[
\left(
\begin{array}
[c]{cccc}%
P\left(  1,1\right)  & P\left(  1,2\right)  & \cdots & P\left(  1,n\right) \\
P\left(  2,1\right)  & P\left(  2,2\right)  & \cdots & P\left(  2,n\right) \\
\vdots & \vdots & \ddots & \vdots\\
P\left(  n,1\right)  & P\left(  n,2\right)  & \cdots & P\left(  n,n\right)
\end{array}
\right)
\]
in block-matrix notation, where the $\left(  i,j\right)  $-th block $P\left(
i,j\right)  $ is defined by\footnote{We let $0_{u\times v}$ denote the zero
matrix of size $u\times v$.}%
\[
P\left(  i,j\right)  :=%
\begin{cases}
I_{n_{i}}, & \text{if }i=\sigma\left(  j\right)  ;\\
0_{n_{i}\times n_{\sigma\left(  j\right)  }}, & \text{if }i\neq\sigma\left(
j\right)  .
\end{cases}
\]
For example, if $n=2$ and if $\sigma$ is the permutation of $\left[  2\right]
$ that swaps $1$ with $2$, and if $n_{1}=1$ and $n_{2}=2$, then $\mathbf{P}%
_{\sigma}=\left(
\begin{array}
[c]{cc}%
0 & I_{1}\\
I_{2} & 0
\end{array}
\right)  =\left(
\begin{array}
[c]{ccc}%
0 & 0 & 1\\
1 & 0 & 0\\
0 & 1 & 0
\end{array}
\right)  $. The formula analogous to (\ref{pf.prop.schurtri.similar.diag.1})
is%
\[
\left(
\begin{array}
[c]{cccc}%
A_{1} & 0 & \cdots & 0\\
0 & A_{2} & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & A_{n}%
\end{array}
\right)  \cdot\mathbf{P}_{\sigma}=\mathbf{P}_{\sigma}\cdot\left(
\begin{array}
[c]{cccc}%
A_{\sigma\left(  1\right)  } & 0 & \cdots & 0\\
0 & A_{\sigma\left(  2\right)  } & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & A_{\sigma\left(  n\right)  }%
\end{array}
\right)
\]
this time; its proof is easy with the help of Proposition
\ref{prop.blockmatrix.mult-uxv}.
\end{proof}

Similarity of block-diagonal matrices can also come from similarity of the
respective blocks:

\begin{proposition}
\label{prop.blockmatrix.simi-pres}Let $\mathbb{F}$ be a field. Let
$n\in\mathbb{N}$. For each $i\in\left[  n\right]  $, let $A_{i}$ and $B_{i}$
be two $n_{i}\times n_{i}$-matrices (for some $n_{i}\in\mathbb{N}$) satisfying
$A_{i}\sim B_{i}$. Then,%
\[
\left(
\begin{array}
[c]{cccc}%
A_{1} & 0 & \cdots & 0\\
0 & A_{2} & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & A_{n}%
\end{array}
\right)  \sim\left(
\begin{array}
[c]{cccc}%
B_{1} & 0 & \cdots & 0\\
0 & B_{2} & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & B_{n}%
\end{array}
\right)  .
\]

\end{proposition}

\begin{exercise}
\label{exe.blockmatrix.simi-pres}\fbox{2} Prove Proposition
\ref{prop.blockmatrix.simi-pres}.
\end{exercise}

\begin{exercise}
\fbox{1} Let $\mathbb{F}$ be a field. Let $A\in\mathbb{F}^{n\times n}$ and
$B\in\mathbb{F}^{n\times n}$ be two matrices such that $A\sim B$. \medskip

\textbf{(a)} Prove that $A^{T}\sim B^{T}$. (Recall that $C^{T}$ denotes the
transpose of a matrix $C$.) \medskip

\textbf{(b)} Assume that $\mathbb{F}=\mathbb{C}$. Prove that $A^{\ast}\sim
B^{\ast}$.
\end{exercise}

\subsection{Unitary similarity}

Unitary similarity is a more restrictive form of similarity, even though it is
not immediately obvious from its definition:

\begin{definition}
\label{def.schurtri.unisim.def}Let $A$ and $B$ be two matrices in
$\mathbb{C}^{n\times n}$. We say that $A$ is \emph{unitarily similar} to $B$
if there exists a unitary matrix $W\in\operatorname*{U}\nolimits_{n}\left(
\mathbb{C}\right)  $ such that $B=WAW^{\ast}$.

We write \textquotedblleft$A\overset{\operatorname*{us}}{\sim}B$%
\textquotedblright\ for \textquotedblleft$A$ is unitarily similar to
$B$\textquotedblright.
\end{definition}

\begin{example}
The matrix $\left(
\begin{array}
[c]{cc}%
1 & 1\\
1 & 1
\end{array}
\right)  $ is unitarily similar to the matrix $\left(
\begin{array}
[c]{cc}%
2 & 0\\
0 & 0
\end{array}
\right)  $, since $\left(
\begin{array}
[c]{cc}%
2 & 0\\
0 & 0
\end{array}
\right)  =W\left(
\begin{array}
[c]{cc}%
1 & 1\\
1 & 1
\end{array}
\right)  W^{\ast}$ for the unitary matrix $W=\dfrac{1}{\sqrt{2}}\left(
\begin{array}
[c]{cc}%
1 & 1\\
-1 & 1
\end{array}
\right)  $.
\end{example}

\begin{exercise}
\label{exe.schurtri.unisim.two2x2}\fbox{2} Prove that the matrix $\left(
\begin{array}
[c]{cc}%
1 & 1\\
0 & 2
\end{array}
\right)  $ is similar to the matrix $\left(
\begin{array}
[c]{cc}%
1 & 0\\
0 & 2
\end{array}
\right)  $, but not unitarily similar to it.
\end{exercise}

Just like the relation $\sim$, the relation $\overset{\operatorname*{us}%
}{\sim}$\ is an equivalence relation:

\begin{proposition}
\label{prop.schurtri.unisim.eqrel}\textbf{(a)} Any matrix $A\in\mathbb{C}%
^{n\times n}$ is unitarily similar to itself. \medskip

\textbf{(b)} If $A$ and $B$ are two matrices in $\mathbb{C}^{n\times n}$ such
that $A$ is unitarily similar to $B$, then $B$ is unitarily similar to $A$.
\medskip

\textbf{(c)} If $A$, $B$ and $C$ are three matrices in $\mathbb{C}^{n\times
n}$ such that $A$ is unitarily similar to $B$ and such that $B$ is unitarily
similar to $C$, then $A$ is unitarily similar to $C$.
\end{proposition}

\begin{proof}
This is very similar to the proof of Proposition
\ref{prop.schurtri.similar.eqrel}, and therefore left to the reader. (The only
new idea is to use Exercise \ref{exe.unitary.group}.)
\end{proof}

\begin{definition}
Let $A$ and $B$ be two matrices in $\mathbb{C}^{n\times n}$. We say that $A$
and $B$ are \emph{unitarily similar} if $A$ is unitarily similar to $B$ (or,
equivalently, $B$ is unitarily similar to $A$).
\end{definition}

As we promised, unitary similarity is a more restrictive version of similarity:

\begin{proposition}
\label{prop.schurtri.unisim.sim}Let $A$ and $B$ be two unitarily similar
matrices in $\mathbb{C}^{n\times n}$. Then, $A$ and $B$ are similar.
\end{proposition}

\begin{proof}
There exists a unitary matrix $W\in\operatorname*{U}\nolimits_{n}\left(
\mathbb{C}\right)  $ such that $B=WAW^{\ast}$ (since $A$ is unitarily similar
to $B$). Consider this $W$. The matrix $W$ is unitary, and thus (by the
implication $\mathcal{A}\Longrightarrow\mathcal{D}$ in Theorem
\ref{thm.unitary.unitary.eqs}) must be square and invertible and satisfy
$W^{-1}=W^{\ast}$. Hence, $B=WA\underbrace{W^{\ast}}_{=W^{-1}}=WAW^{-1}$. But
this shows that $A$ is similar to $B$. Thus, Proposition
\ref{prop.schurtri.unisim.sim} is proven.
\end{proof}

The following proposition is an analogue of Proposition
\ref{prop.blockmatrix.simi-pres} for unitary similarity:

\begin{proposition}
\label{prop.schurtri.block-unisimi}Let $n\in\mathbb{N}$. For each $i\in\left[
n\right]  $, let $A_{i}\in\mathbb{C}^{n_{i}\times n_{i}}$ and $B_{i}%
\in\mathbb{C}^{n_{i}\times n_{i}}$ be two $n_{i}\times n_{i}$-matrices (for
some $n_{i}\in\mathbb{N}$) satisfying $A_{i}\overset{\operatorname*{us}}{\sim
}B_{i}$. Then,%
\[
\left(
\begin{array}
[c]{cccc}%
A_{1} & 0 & \cdots & 0\\
0 & A_{2} & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & A_{n}%
\end{array}
\right)  \overset{\operatorname*{us}}{\sim}\left(
\begin{array}
[c]{cccc}%
B_{1} & 0 & \cdots & 0\\
0 & B_{2} & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & B_{n}%
\end{array}
\right)  .
\]

\end{proposition}

\begin{exercise}
\label{exe.schurtri.block-unisimi}\fbox{1} Prove Proposition
\ref{prop.schurtri.block-unisimi}.
\end{exercise}

We note further that the similarity in Proposition
\ref{prop.schurtri.similar.diag} can be upgraded to a unitary similarity if we
work over the field $\mathbb{C}$:

\begin{proposition}
\label{prop.schurtri.unisimi.diag}Let $n\in\mathbb{N}$. Let $\lambda
_{1},\lambda_{2},\ldots,\lambda_{n}\in\mathbb{C}$. Let $\sigma$ be a
permutation of $\left[  n\right]  $ (that is, a bijective map from $\left[
n\right]  $ to $\left[  n\right]  $). Then,%
\[
\operatorname*{diag}\left(  \lambda_{1},\lambda_{2},\ldots,\lambda_{n}\right)
\overset{\operatorname*{us}}{\sim}\operatorname*{diag}\left(  \lambda
_{\sigma\left(  1\right)  },\lambda_{\sigma\left(  2\right)  },\ldots
,\lambda_{\sigma\left(  n\right)  }\right)  .
\]

\end{proposition}

\begin{proof}
Let $P_{\sigma}\in\mathbb{F}^{n\times n}$ be the permutation matrix of
$\sigma$ (defined in Example \ref{exa.unitary.unitary.exas} \textbf{(d)}). In
the proof of Proposition \ref{prop.schurtri.similar.diag}, we have shown that
\[
\operatorname*{diag}\left(  \lambda_{1},\lambda_{2},\ldots,\lambda_{n}\right)
=P_{\sigma}\cdot\operatorname*{diag}\left(  \lambda_{\sigma\left(  1\right)
},\lambda_{\sigma\left(  2\right)  },\ldots,\lambda_{\sigma\left(  n\right)
}\right)  \cdot P_{\sigma}^{-1}.
\]
However, the matrix $P_{\sigma}$ is unitary (as we have already seen in
Example \ref{exa.unitary.unitary.exas} \textbf{(d)}). Hence, $P_{\sigma}%
^{-1}=P_{\sigma}^{\ast}$. Thus,%
\begin{align*}
\operatorname*{diag}\left(  \lambda_{1},\lambda_{2},\ldots,\lambda_{n}\right)
&  =P_{\sigma}\cdot\operatorname*{diag}\left(  \lambda_{\sigma\left(
1\right)  },\lambda_{\sigma\left(  2\right)  },\ldots,\lambda_{\sigma\left(
n\right)  }\right)  \cdot\underbrace{P_{\sigma}^{-1}}_{=P_{\sigma}^{\ast}}\\
&  =P_{\sigma}\cdot\operatorname*{diag}\left(  \lambda_{\sigma\left(
1\right)  },\lambda_{\sigma\left(  2\right)  },\ldots,\lambda_{\sigma\left(
n\right)  }\right)  \cdot P_{\sigma}^{\ast}.
\end{align*}
This shows that $\operatorname*{diag}\left(  \lambda_{1},\lambda_{2}%
,\ldots,\lambda_{n}\right)  \overset{\operatorname*{us}}{\sim}%
\operatorname*{diag}\left(  \lambda_{\sigma\left(  1\right)  },\lambda
_{\sigma\left(  2\right)  },\ldots,\lambda_{\sigma\left(  n\right)  }\right)
$. Thus, Proposition \ref{prop.schurtri.unisimi.diag} is proven.
\end{proof}

%

\[%
\begin{tabular}
[c]{||l||}\hline\hline
\textbf{Lecture 4 starts here.}\\\hline\hline
\end{tabular}
\]


\subsection{Schur triangularization}

\subsubsection{The theorems}

We are now ready for one more matrix decomposition: the so-called \emph{Schur
triangularization} (aka \emph{Schur decomposition}):

\begin{theorem}
[Schur triangularization theorem]\label{thm.schurtri.schurtri}Let
$A\in\mathbb{C}^{n\times n}$. Then, there exist a unitary matrix
$U\in\operatorname*{U}\nolimits_{n}\left(  \mathbb{C}\right)  $ and an
upper-triangular matrix $T\in\mathbb{C}^{n\times n}$ such that $A=UTU^{\ast}$.
In other words, $A$ is unitarily similar to some upper-triangular matrix.
\end{theorem}

The factorization $A=UTU^{\ast}$ in Theorem \ref{thm.schurtri.schurtri} (or,
to be more precise, the pair $\left(  U,T\right)  $) is called a \emph{Schur
triangularization} of $A$. It is usually not unique.

\begin{example}
Let $A=\left(
\begin{array}
[c]{cc}%
1 & 3\\
-3 & 7
\end{array}
\right)  \in\mathbb{C}^{2\times2}$. Then, a Schur triangularization of $A$ is
$A=UTU^{\ast}$, where%
\[
U=\dfrac{1}{\sqrt{2}}\left(
\begin{array}
[c]{cc}%
1 & -1\\
1 & 1
\end{array}
\right)  \ \ \ \ \ \ \ \ \ \ \text{and}\ \ \ \ \ \ \ \ \ \ T=\left(
\begin{array}
[c]{cc}%
4 & 6\\
0 & 4
\end{array}
\right)  .
\]
(We chose $A$ deliberately to obtain \textquotedblleft nice\textquotedblright%
\ matrices $U$ and $T$. The Schur triangularization of a typical $n\times
n$-matrix will be more complicated, involving roots of $n$-th degree polynomials.)
\end{example}

We shall prove a slightly stronger form of Theorem \ref{thm.schurtri.schurtri}:

\begin{theorem}
[Schur triangularization with prescribed diagonal]%
\label{thm.jnf.schurtri-diag}Let $A\in\mathbb{C}^{n\times n}$ be an $n\times
n$-matrix. Let $\lambda_{1},\lambda_{2},\ldots,\lambda_{n}$ be its eigenvalues
(listed with their algebraic multiplicities). Then, there exists an
upper-triangular matrix $T\in\mathbb{C}^{n\times n}$ such that
$A\overset{\operatorname*{us}}{\sim}T$ (this means \textquotedblleft$A$ is
unitarily similar to $T$\textquotedblright, as we recall) and such that the
diagonal entries of $T$ are $\lambda_{1},\lambda_{2},\ldots,\lambda_{n}$ in
this order.
\end{theorem}

\begin{example}
\label{exa.jnf.schurtri-diag.2x2}Let $A=\left(
\begin{array}
[c]{cc}%
1 & -2i\\
0 & 3
\end{array}
\right)  \in\mathbb{C}^{2\times2}$. Theorem \ref{thm.schurtri.schurtri} then
says that $A$ is unitarily similar to some upper-triangular matrix. But this
is trivial, since $A$ is already upper-triangular (and clearly unitarily
similar to itself). However, Theorem \ref{thm.jnf.schurtri-diag} can be used
to draw a less obvious conclusion. In fact, the eigenvalues of $A$ are $1$ and
$3$. Let us list them in the order $3,1$. Then, Theorem
\ref{thm.jnf.schurtri-diag} (applied to $n=2$ and $\left(  \lambda_{1}%
,\lambda_{2},\ldots,\lambda_{n}\right)  =\left(  3,1\right)  $) yields that
there exists an upper-triangular matrix $T\in\mathbb{C}^{2\times2}$ such that
$A\overset{\operatorname*{us}}{\sim}T$ and such that the diagonal entries of
$T$ are $3$ and $1$ in this order. Finding such a $T$ is not all that easy (in
particular, $A$ itself does not qualify, since its diagonal entries are $1$
and $3$ rather than $3$ and $1$). The answer is:
\[
U=\dfrac{1}{\sqrt{2}}\left(
\begin{array}
[c]{cc}%
-i & i\\
1 & 1
\end{array}
\right)  \ \ \ \ \ \ \ \ \ \ \text{and}\ \ \ \ \ \ \ \ \ \ T=\left(
\begin{array}
[c]{cc}%
3 & 2\\
0 & 1
\end{array}
\right)  .
\]
Here, $U$ is the unitary matrix satisfying $A=UTU^{\ast}$ (which confirms that
$A\overset{\operatorname*{us}}{\sim}T$).
\end{example}

Actually, the form of the matrix $T$ in this example is no accident; more
generally, we have:

\begin{exercise}
\label{exe.jnf.schurtri-diag.2x2}\fbox{5} Let $a,b,c\in\mathbb{C}$. Prove that%
\[
\left(
\begin{array}
[c]{cc}%
a & b\\
0 & c
\end{array}
\right)  \overset{\operatorname*{us}}{\sim}\left(
\begin{array}
[c]{cc}%
c & -\overline{b}\\
0 & a
\end{array}
\right)  .
\]

\end{exercise}

\subsubsection{The proofs}

The following lemma about characteristic polynomials will help us in our proof
of Theorem \ref{thm.jnf.schurtri-diag}:

\begin{lemma}
\label{lem.jnf.schurtri-diag.descend-poly}Let $\mathbb{F}$ be a field. Let
$n\in\mathbb{N}$. Let $p\in\mathbb{F}^{1\times n}$ be a row vector, and let
$B\in\mathbb{F}^{n\times n}$ be an $n\times n$-matrix. Let $\lambda
\in\mathbb{F}$ be a scalar. Let $C$ be the $\left(  n+1\right)  \times\left(
n+1\right)  $-matrix $\left(
\begin{array}
[c]{cc}%
\lambda & p\\
0 & B
\end{array}
\right)  \in\mathbb{F}^{\left(  n+1\right)  \times\left(  n+1\right)  }$
(written in block-matrix notation, where the \textquotedblleft$\lambda
$\textquotedblright\ stands for the $1\times1$-matrix $\left(
\begin{array}
[c]{c}%
\lambda
\end{array}
\right)  $, and where the \textquotedblleft$0$\textquotedblright\ stands for
the zero vector in $\mathbb{F}^{n\times1}$). Then,
\[
p_{C}=\left(  t-\lambda\right)  \cdot p_{B}.
\]

\end{lemma}

\begin{proof}
[Proof of Lemma \ref{lem.jnf.schurtri-diag.descend-poly}.]The definition of a
characteristic polynomial yields%
\[
p_{B}=\det\left(  tI_{n}-B\right)  \ \ \ \ \ \ \ \ \ \ \text{and}%
\ \ \ \ \ \ \ \ \ \ p_{C}=\det\left(  tI_{n+1}-C\right)  .
\]
However, from $I_{n+1}=\left(
\begin{array}
[c]{cc}%
1 & 0\\
0 & I_{n}%
\end{array}
\right)  $ and $C=\left(
\begin{array}
[c]{cc}%
\lambda & p\\
0 & B
\end{array}
\right)  $ we obtain
\[
tI_{n+1}-C=t\left(
\begin{array}
[c]{cc}%
1 & 0\\
0 & I_{n}%
\end{array}
\right)  -\left(
\begin{array}
[c]{cc}%
\lambda & p\\
0 & B
\end{array}
\right)  =\left(
\begin{array}
[c]{cc}%
t-\lambda & -p\\
0 & tI_{n}-B
\end{array}
\right)  .
\]
Thus,%
\[
\det\left(  tI_{n+1}-C\right)  =\det\left(
\begin{array}
[c]{cc}%
t-\lambda & -p\\
0 & tI_{n}-B
\end{array}
\right)  =\left(  t-\lambda\right)  \cdot\det\left(  tI_{n}-B\right)
\]
(here, we have applied Laplace expansion along the first column to compute the
determinant, noticing that this column has only one nonzero entry). Thus,%
\[
p_{C}=\det\left(  tI_{n+1}-C\right)  =\left(  t-\lambda\right)  \cdot
\underbrace{\det\left(  tI_{n}-B\right)  }_{=p_{B}}=\left(  t-\lambda\right)
\cdot p_{B}.
\]
This proves Lemma \ref{lem.jnf.schurtri-diag.descend-poly}.
\end{proof}

\begin{noncompile}
The following lemma was originally used in the above proof:

\begin{lemma}
Let $\mathbb{F}$ be a field. Let $X\in\mathbb{F}^{n\times n}$, $Y\in
\mathbb{F}^{n\times m}$ and $Z\in\mathbb{F}^{m\times m}$ be three matrices.
Then, $\det\left(
\begin{array}
[c]{cc}%
X & Y\\
0 & Z
\end{array}
\right)  =\det X\cdot\det Z.$
\end{lemma}

For example,%
\[
\det\left(
\begin{array}
[c]{cccc}%
a & b & c & d\\
a^{\prime} & b^{\prime} & c^{\prime} & d^{\prime}\\
0 & 0 & c^{\prime\prime} & d^{\prime\prime}\\
0 & 0 & c^{\prime\prime\prime} & d^{\prime\prime\prime}%
\end{array}
\right)  =\det\left(
\begin{array}
[c]{cc}%
a & b\\
a^{\prime} & b^{\prime}%
\end{array}
\right)  \cdot\det\left(
\begin{array}
[c]{cc}%
c^{\prime\prime} & d^{\prime\prime}\\
c^{\prime\prime\prime} & d^{\prime\prime\prime}%
\end{array}
\right)  .
\]

\end{noncompile}

Let us now prove Theorem \ref{thm.jnf.schurtri-diag}:

\begin{proof}
[Proof of Theorem \ref{thm.jnf.schurtri-diag}.]We proceed by induction on $n$:

\textit{Induction base:} For $n=0$, Theorem \ref{thm.jnf.schurtri-diag} holds
trivially\footnote{There is only one $0\times0$-matrix, and we take $T$ to be
this matrix.}.

\textit{Induction step:} Let $m$ be a positive integer. Assume (as the
induction hypothesis) that Theorem \ref{thm.jnf.schurtri-diag} is proved for
$n=m-1$. We must prove that Theorem \ref{thm.jnf.schurtri-diag} holds for
$n=m$.

So let $A\in\mathbb{C}^{m\times m}$ be an $m\times m$-matrix, and let
$\lambda_{1},\lambda_{2},\ldots,\lambda_{m}$ be its eigenvalues (listed with
their algebraic multiplicities). We must show that there exists an
upper-triangular matrix $T\in\mathbb{C}^{m\times m}$ such that
$A\overset{\operatorname*{us}}{\sim}T$ and such that the diagonal entries of
$T$ are $\lambda_{1},\lambda_{2},\ldots,\lambda_{m}$ in this order.

Since $\lambda_{1}$ is an eigenvalue of $A$, there exists at least one nonzero
$\lambda_{1}$-eigenvector $x$ of $A$. Pick any such $x$. Set $u_{1}:=\dfrac
{1}{\left\vert \left\vert x\right\vert \right\vert }x$. Then, $u_{1}$ is still
a $\lambda_{1}$-eigenvector of $A$, but additionally satisfies $\left\vert
\left\vert u_{1}\right\vert \right\vert =1$. Hence, the $1$-tuple $\left(
u_{1}\right)  $ of vectors in $\mathbb{C}^{m}$ is orthonormal.

Thus, Corollary \ref{cor.unitary.orthon-extend} (applied to $k=1$ and $n=m$)
shows that we can find $m-1$ vectors $u_{2},u_{3},\ldots,u_{m}\in
\mathbb{C}^{m}$ such that $\left(  u_{1},u_{2},\ldots,u_{m}\right)  $ is an
orthonormal basis of $\mathbb{C}^{m}$. Consider these $m-1$ vectors
$u_{2},u_{3},\ldots,u_{m}$.

Let $U\in\mathbb{C}^{m\times m}$ be the $m\times m$-matrix whose columns are
$u_{1},u_{2},\ldots,u_{m}$ in this order. Then, the columns of this matrix
form an orthonormal basis of $\mathbb{C}^{m}$. Hence, Theorem
\ref{thm.unitary.unitary.eqs} (specifically, the implication $\mathcal{E}%
\Longrightarrow\mathcal{A}$ in this theorem) yields that the matrix $U$ is
unitary. Therefore, $U^{\ast}=U^{-1}$. Moreover, since $U$ is unitary, we have
$UU^{\ast}=I_{m}$ and $U^{\ast}U=I_{m}$. Thus, $U^{\ast}$ is unitary as well.

Define an $m\times m$-matrix%
\[
C:=U^{\ast}AU\in\mathbb{C}^{m\times m}.
\]
Since $U^{\ast}$ is unitary, we have $A\overset{\operatorname*{us}}{\sim
}U^{\ast}A\left(  U^{\ast}\right)  ^{\ast}$. In other words,
$A\overset{\operatorname*{us}}{\sim}C$ (since $U^{\ast}A\underbrace{\left(
U^{\ast}\right)  ^{\ast}}_{=U}=U^{\ast}AU=C$). Hence, $A\sim C$ (by
Proposition \ref{prop.schurtri.unisim.sim}). Thus, Proposition
\ref{prop.schurtri.similar.same} \textbf{(d)} shows that the matrices $A$ and
$C$ have the same characteristic polynomial. In other words, $p_{A}=p_{C}$.

The definition of $U$ yields $U=\left(
\begin{array}
[c]{ccc}%
\mid &  & \mid\\
u_{1} & \cdots & u_{m}\\
\mid &  & \mid
\end{array}
\right)  $ and therefore%
\[
U^{\ast}=\left(
\begin{array}
[c]{ccc}%
\text{---} & u_{1}^{\ast} & \text{---}\\
& \vdots & \\
\text{---} & u_{m}^{\ast} & \text{---}%
\end{array}
\right)  \ \ \ \ \ \ \ \ \ \ \text{and}\ \ \ \ \ \ \ \ \ \ AU=\left(
\begin{array}
[c]{ccc}%
\mid &  & \mid\\
Au_{1} & \cdots & Au_{m}\\
\mid &  & \mid
\end{array}
\right)  .
\]
Multiplying the latter two equalities, we obtain%
\[
U^{\ast}AU=\left(
\begin{array}
[c]{cccc}%
u_{1}^{\ast}Au_{1} & u_{1}^{\ast}Au_{2} & \cdots & u_{1}^{\ast}Au_{m}\\
u_{2}^{\ast}Au_{1} & u_{2}^{\ast}Au_{2} & \cdots & u_{2}^{\ast}Au_{m}\\
\vdots & \vdots & \ddots & \vdots\\
u_{m}^{\ast}Au_{1} & u_{m}^{\ast}Au_{2} & \cdots & u_{m}^{\ast}Au_{m}%
\end{array}
\right)  .
\]
Since $C=U^{\ast}AU$, we can rewrite this as%
\begin{equation}
C=\left(
\begin{array}
[c]{cccc}%
u_{1}^{\ast}Au_{1} & u_{1}^{\ast}Au_{2} & \cdots & u_{1}^{\ast}Au_{m}\\
u_{2}^{\ast}Au_{1} & u_{2}^{\ast}Au_{2} & \cdots & u_{2}^{\ast}Au_{m}\\
\vdots & \vdots & \ddots & \vdots\\
u_{m}^{\ast}Au_{1} & u_{m}^{\ast}Au_{2} & \cdots & u_{m}^{\ast}Au_{m}%
\end{array}
\right)  . \label{pf.thm.jnf.schurtri-diag.3}%
\end{equation}
Hence,
\begin{align}
C_{1,1}  &  =u_{1}^{\ast}\underbrace{Au_{1}}_{\substack{=\lambda_{1}%
u_{1}\\\text{(since }u_{1}\text{ is a }\lambda_{1}\text{-eigenvector of
}A\text{)}}}=u_{1}^{\ast}\lambda_{1}u_{1}=\lambda_{1}\underbrace{u_{1}^{\ast
}u_{1}}_{\substack{=\left\langle u_{1},u_{1}\right\rangle =\left\vert
\left\vert u_{1}\right\vert \right\vert ^{2}=1\\\text{(since }\left\vert
\left\vert u_{1}\right\vert \right\vert =1\text{)}}}\nonumber\\
&  =\lambda_{1}. \label{pf.thm.jnf.schurtri-diag.4}%
\end{align}
Moreover, for each $i\in\left\{  2,3,\ldots,m\right\}  $, we have%
\begin{align*}
C_{i,1}  &  =u_{i}^{\ast}\underbrace{Au_{1}}_{\substack{=\lambda_{1}%
u_{1}\\\text{(since }u_{1}\text{ is a }\lambda_{1}\text{-eigenvector of
}A\text{)}}}\ \ \ \ \ \ \ \ \ \ \left(  \text{by
(\ref{pf.thm.jnf.schurtri-diag.3})}\right) \\
&  =u_{i}^{\ast}\lambda_{1}u_{1}=\lambda_{1}\underbrace{u_{i}^{\ast}u_{1}%
}_{\substack{=\left\langle u_{1},u_{i}\right\rangle =0\\\text{(since }\left(
u_{1},u_{2},\ldots,u_{m}\right)  \text{ is an orthonormal basis,}\\\text{and
thus }u_{1}\perp u_{i}\text{)}}}=0.
\end{align*}
Combining this with (\ref{pf.thm.jnf.schurtri-diag.4}), we see that the $1$-st
column of the matrix $C$ has entries $\lambda_{1},0,0,\ldots,0$ from top to
bottom. In other words,%
\[
C=\left(
\begin{array}
[c]{ccccc}%
\lambda_{1} & \ast & \ast & \cdots & \ast\\
0 & \ast & \ast & \cdots & \ast\\
0 & \ast & \ast & \cdots & \ast\\
\vdots & \vdots & \vdots & \ddots & \vdots\\
0 & \ast & \ast & \cdots & \ast
\end{array}
\right)  ,
\]
where each asterisk ($\ast$) means an entry that we don't know or don't care
about (in our case, both).

Let us write this in block-matrix notation:%
\begin{equation}
C=\left(
\begin{array}
[c]{cc}%
\lambda_{1} & p\\
0 & B
\end{array}
\right)  , \label{pf.thm.jnf.schurtri-diag.C=2x2}%
\end{equation}
where $p\in\mathbb{C}^{1\times\left(  m-1\right)  }$ is a row vector and
$B\in\mathbb{C}^{\left(  m-1\right)  \times\left(  m-1\right)  }$ is a
matrix\footnote{The \textquotedblleft$0$\textquotedblright\ here is actually
the zero vector in $\mathbb{C}^{m-1}$.}. Consider this $p$ and this $B$.

We shall now show that the eigenvalues of $B$ are $\lambda_{2},\lambda
_{3},\ldots,\lambda_{m}$. Indeed, Lemma
\ref{lem.jnf.schurtri-diag.descend-poly} (applied to $\mathbb{F}=\mathbb{C}$
and $n=m-1$ and $\lambda=\lambda_{1}$) yields $p_{C}=\left(  t-\lambda
_{1}\right)  \cdot p_{B}$ (because of (\ref{pf.thm.jnf.schurtri-diag.C=2x2})).
Hence,%
\[
p_{A}=p_{C}=\left(  t-\lambda_{1}\right)  \cdot p_{B}.
\]
On the other hand,%
\[
p_{A}=\left(  t-\lambda_{1}\right)  \left(  t-\lambda_{2}\right)
\cdots\left(  t-\lambda_{m}\right)
\]
(since $p_{A}$ is monic, and the roots of $p_{A}$ are precisely the
eigenvalues of $A$ with algebraic multiplicities, which we know are
$\lambda_{1},\lambda_{2},\ldots,\lambda_{m}$). Comparing these two equalities,
we obtain%
\[
\left(  t-\lambda_{1}\right)  \cdot p_{B}=\left(  t-\lambda_{1}\right)
\left(  t-\lambda_{2}\right)  \cdots\left(  t-\lambda_{m}\right)  .
\]
We can cancel the factor $t-\lambda_{1}$ from both sides of this equality
(since the polynomial ring over $\mathbb{C}$ has no zero-divisors). Thus, we
obtain%
\[
p_{B}=\left(  t-\lambda_{2}\right)  \left(  t-\lambda_{3}\right)
\cdots\left(  t-\lambda_{m}\right)  .
\]
In other words, the eigenvalues of the $\left(  m-1\right)  \times\left(
m-1\right)  $-matrix $B$ are $\lambda_{2},\lambda_{3},\ldots,\lambda_{m}$
(listed with their algebraic multiplicities).

Hence, by the induction hypothesis, we can apply Theorem
\ref{thm.jnf.schurtri-diag} to $m-1$, $B$ and $\lambda_{2},\lambda_{3}%
,\ldots,\lambda_{m}$ instead of $n$, $A$ and $\lambda_{1},\lambda_{2}%
,\ldots,\lambda_{n}$. As a result, we conclude that there exists an
upper-triangular matrix $S\in\mathbb{C}^{\left(  m-1\right)  \times\left(
m-1\right)  }$ such that $B\overset{\operatorname*{us}}{\sim}S$ and such that
the diagonal entries of $S$ are $\lambda_{2},\lambda_{3},\ldots,\lambda_{m}$
in this order. Consider this $S$.

We have $B\overset{\operatorname*{us}}{\sim}S$. In other words, there is a
unitary matrix $V\in\mathbb{C}^{\left(  m-1\right)  \times\left(  m-1\right)
}$ such that $S=VBV^{\ast}$. Consider this $V$.

Now, let%
\[
W:=\left(
\begin{array}
[c]{cc}%
1 & 0\\
0 & V
\end{array}
\right)  \in\mathbb{C}^{m\times m}\ \ \ \ \ \ \ \ \ \ \left(  \text{in
block-matrix notation}\right)  .
\]
This is a block-diagonal matrix, with $\left(
\begin{array}
[c]{c}%
1
\end{array}
\right)  $ and $V$ being its diagonal blocks. Hence, $W^{\ast}=\left(
\begin{array}
[c]{cc}%
1 & 0\\
0 & V^{\ast}%
\end{array}
\right)  $. Moreover, $W$ is a unitary matrix (since it is a block-diagonal
matrix whose diagonal blocks are unitary\footnote{We are using Proposition
\ref{prop.blockmatrix.unitary-diag} here.}). Hence,
$C\overset{\operatorname*{us}}{\sim}WCW^{\ast}$. Combining
$A\overset{\operatorname*{us}}{\sim}C$ with $C\overset{\operatorname*{us}%
}{\sim}WCW^{\ast}$, we obtain $A\overset{\operatorname*{us}}{\sim}WCW^{\ast}$
(by Proposition \ref{prop.schurtri.unisim.eqrel} \textbf{(c)}).

However,%
\begin{align*}
&  \underbrace{W}_{=\left(
\begin{array}
[c]{cc}%
1 & 0\\
0 & V
\end{array}
\right)  }\ \ \underbrace{C}_{=\left(
\begin{array}
[c]{cc}%
\lambda_{1} & p\\
0 & B
\end{array}
\right)  }\ \ \underbrace{W^{\ast}}_{=\left(
\begin{array}
[c]{cc}%
1 & 0\\
0 & V^{\ast}%
\end{array}
\right)  }\\
&  =\left(
\begin{array}
[c]{cc}%
1 & 0\\
0 & V
\end{array}
\right)  \left(
\begin{array}
[c]{cc}%
\lambda_{1} & p\\
0 & B
\end{array}
\right)  \left(
\begin{array}
[c]{cc}%
1 & 0\\
0 & V^{\ast}%
\end{array}
\right) \\
&  =\left(
\begin{array}
[c]{cc}%
1\cdot\lambda_{1}\cdot1 & 1\cdot p\cdot V^{\ast}\\
V\cdot0\cdot1 & V\cdot B\cdot V^{\ast}%
\end{array}
\right)  \ \ \ \ \ \ \ \ \ \ \left(
\begin{array}
[c]{c}%
\text{by using Proposition \ref{prop.blockmatrix.mult-2x2} twice}\\
\text{and simplifying the }0\text{ addends away}%
\end{array}
\right) \\
&  =\left(
\begin{array}
[c]{cc}%
\lambda_{1} & pV^{\ast}\\
0 & VBV^{\ast}%
\end{array}
\right)  =\left(
\begin{array}
[c]{cc}%
\lambda_{1} & pV^{\ast}\\
0 & S
\end{array}
\right)  \ \ \ \ \ \ \ \ \ \ \left(  \text{since }VBV^{\ast}=S\right)  .
\end{align*}
This matrix $WCW^{\ast}$ is therefore upper-triangular (since the bottom-left
block is a zero vector, and since the bottom-right block $S$ is
upper-triangular), and its diagonal entries are $\lambda_{1},\lambda
_{2},\ldots,\lambda_{m}$ in this order (because its first diagonal entry is
visibly $\lambda_{1}$, whereas its remaining diagonal entries are the diagonal
entries of $S$ and therefore are $\lambda_{2},\lambda_{3},\ldots,\lambda_{m}$
in this order\footnote{In fact, we have shown above that the diagonal entries
of $S$ are $\lambda_{2},\lambda_{3},\ldots,\lambda_{m}$ in this order.}).

Thus, there exists an upper-triangular matrix $T\in\mathbb{C}^{m\times m}$
such that $A\overset{\operatorname*{us}}{\sim}T$ and such that the diagonal
entries of $T$ are $\lambda_{1},\lambda_{2},\ldots,\lambda_{m}$ in this order
(namely, $WCW^{\ast}$ is such a matrix, because $A\overset{\operatorname*{us}%
}{\sim}WCW^{\ast}$ and because of what we just said).

Thus, we have shown that Theorem \ref{thm.jnf.schurtri-diag} holds for $n=m$.
This completes the induction step. The proof of Theorem
\ref{thm.jnf.schurtri-diag} is thus complete.
\end{proof}

\begin{proof}
[Proof of Theorem \ref{thm.schurtri.schurtri}.]Theorem
\ref{thm.schurtri.schurtri} follows from Theorem \ref{thm.jnf.schurtri-diag}
(and the definition of unitary similarity).
\end{proof}

\begin{exercise}
\label{exe.schurtri.schurtri.one2x2}\fbox{2} Find a Schur triangularization of
the matrix $\left(
\begin{array}
[c]{cc}%
1 & 0\\
i & 1
\end{array}
\right)  $.
\end{exercise}

\begin{exercise}
\label{exe.schurtri.schurtri.one3x3}\fbox{3} Find a Schur triangularization of
the matrix $\left(
\begin{array}
[c]{ccc}%
1 & 1 & 1\\
1 & 1 & 1\\
1 & 1 & 1
\end{array}
\right)  $.
\end{exercise}

\subsubsection{The diagonal entries of $T$}

One remark is in order about the $T$ in a Schur triangularization:

\begin{proposition}
\label{prop.schurtri.schurtri.T-diag}Let $A\in\mathbb{C}^{n\times n}$. Let
$\left(  U,T\right)  $ be a Schur triangularization of $A$. Then, the diagonal
entries of $T$ are the eigenvalues of $A$ (with their algebraic multiplicities).
\end{proposition}

Instead of proving this directly, let us show a more general result:

\begin{proposition}
\label{prop.schurtri.similar.T-diag}Let $\mathbb{F}$ be a field. Let
$A\in\mathbb{F}^{n\times n}$ and $T\in\mathbb{F}^{n\times n}$ be two similar
matrices. Assume that $T$ is upper-triangular. Then, the diagonal entries of
$T$ are the eigenvalues of $A$ (with their algebraic multiplicities).
\end{proposition}

\begin{proof}
[Proof of Proposition \ref{prop.schurtri.similar.T-diag}.]We have assumed that
$A$ and $T$ are similar. In other words, $T$ is similar to $A$. Thus,
Proposition \ref{prop.schurtri.similar.same} \textbf{(e)} shows that the
matrices $T$ and $A$ have the same eigenvalues with the same algebraic
multiplicities (and the same geometric multiplicities, but we don't need to
know this). In other words, the eigenvalues of $T$ are the eigenvalues of $A$
(with the same algebraic multiplicities).

However, the matrix $T$ is upper-triangular. Thus, the eigenvalues of $T$
(with algebraic multiplicities) are the diagonal entries of $T$ (this is a
well-known fact\footnote{Here is a \textit{proof:} The matrix $T$ is
upper-triangular; thus, it has the form
\[
T=\left(
\begin{array}
[c]{cccc}%
T_{1,1} & T_{1,2} & \cdots & T_{1,n}\\
0 & T_{2,2} & \cdots & T_{2,n}\\
\vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & T_{n,n}%
\end{array}
\right)  .
\]
Thus, if $t$ is an indeterminate, then
\begin{align*}
tI_{n}-T  &  =t\left(
\begin{array}
[c]{cccc}%
1 & 0 & \cdots & 0\\
0 & 1 & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & 1
\end{array}
\right)  -\left(
\begin{array}
[c]{cccc}%
T_{1,1} & T_{1,2} & \cdots & T_{1,n}\\
0 & T_{2,2} & \cdots & T_{2,n}\\
\vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & T_{n,n}%
\end{array}
\right) \\
&  =\left(
\begin{array}
[c]{cccc}%
t-T_{1,1} & -T_{1,2} & \cdots & -T_{1,n}\\
0 & t-T_{2,2} & \cdots & -T_{2,n}\\
\vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & t-T_{n,n}%
\end{array}
\right)  .
\end{align*}
This is still an upper-triangular matrix; thus, its determinant is the product
of its diagonal entries. That is, we have%
\[
\det\left(  tI_{n}-T\right)  =\left(  t-T_{1,1}\right)  \left(  t-T_{2,2}%
\right)  \cdots\left(  t-T_{n,n}\right)  .
\]
Therefore, $T_{1,1},T_{2,2},\ldots,T_{n,n}$ are the roots of the polynomial
$\det\left(  tI_{n}-T\right)  $ (with multiplicities).
\par
However, $\det\left(  tI_{n}-T\right)  $ is the characteristic polynomial of
$T$. Thus, the roots of this polynomial $\det\left(  tI_{n}-T\right)  $ are
the eigenvalues of $T$ (with algebraic multiplicities). Since we know that the
roots of this polynomial are $T_{1,1},T_{2,2},\ldots,T_{n,n}$, we thus
conclude that $T_{1,1},T_{2,2},\ldots,T_{n,n}$ are the eigenvalues of $T$
(with algebraic multiplicities). In other words, the diagonal entries of $T$
are the eigenvalues of $T$ (with algebraic multiplicities). Qed.}). Since we
know that the eigenvalues of $T$ are the eigenvalues of $A$ (with the same
algebraic multiplicities), we thus conclude that the eigenvalues of $A$ (with
algebraic multiplicities) are the diagonal entries of $T$. This proves
Proposition \ref{prop.schurtri.similar.T-diag}.
\end{proof}

\begin{proof}
[Proof of Proposition \ref{prop.schurtri.schurtri.T-diag}.]We assumed that
$\left(  U,T\right)  $ is a Schur triangularization of $A$. Hence, we have
$A=UTU^{\ast}$, and the matrix $U$ is unitary whereas the matrix $T$ is
upper-triangular. From $A=UTU^{\ast}$, we conclude that $T$ is unitarily
similar to $A$ (by Definition \ref{def.schurtri.unisim.def}, since $U$ is
unitary). Hence, $T$ is similar to $A$ (by Proposition
\ref{prop.schurtri.unisim.sim}). Therefore, Proposition
\ref{prop.schurtri.similar.T-diag} shows that the diagonal entries of $T$ are
the eigenvalues of $A$ (with their algebraic multiplicities). This proves
Proposition \ref{prop.schurtri.schurtri.T-diag}.
\end{proof}

We will see several applications of Theorem \ref{thm.schurtri.schurtri} in
this chapter. First, however, let us see some variants of Schur triangularization.

\subsubsection{Triangularization over an arbitrary field}

The first variant gives a partial answer to the following natural question:
What becomes of Theorem \ref{thm.schurtri.schurtri} if we replace $\mathbb{C}$
by an arbitrary field $\mathbb{F}$ ? Of course, Theorem
\ref{thm.schurtri.schurtri} does not even make sense for an arbitrary field
$\mathbb{F}$, since the notion of a unitary matrix is only defined over
$\mathbb{C}$. Even if we take this loss and replace \textquotedblleft
unitary\textquotedblright\ by merely \textquotedblleft
invertible\textquotedblright\ (so $U^{\ast}$ becomes $U^{-1}$, and
\textquotedblleft unitarily similar\textquotedblright\ just becomes
\textquotedblleft similar\textquotedblright), the theorem will still fail,
because (for example) the matrix $\left(
\begin{array}
[c]{cc}%
0 & -1\\
1 & 0
\end{array}
\right)  $ is not similar to any triangular matrix over the field $\mathbb{R}%
$. This is because its eigenvalues (which are $i$ and $-i$) are not real.

However, apart from these two issues, things go well. So we can state the
following weak version of Schur triangularization over an arbitrary field:

\begin{theorem}
[triangularization theorem]\label{thm.schurtri.invtri}Let $\mathbb{F}$ be a
field. Let $A\in\mathbb{F}^{n\times n}$. Assume that the characteristic
polynomial $p_{A}$ of $A$ factors as a product of $n$ linear factors over
$\mathbb{F}$ (so $A$ has $n$ eigenvalues in $\mathbb{F}$, counted with
algebraic multiplicities). Then, there exist an invertible matrix
$U\in\mathbb{F}^{n\times n}$ and an upper-triangular matrix $T\in
\mathbb{F}^{n\times n}$ such that $A=UTU^{-1}$. In other words, $A$ is similar
to some upper-triangular matrix.
\end{theorem}

This is an analogue of Theorem \ref{thm.schurtri.schurtri}; a corresponding
analogue of Theorem \ref{thm.jnf.schurtri-diag} also exists:

\begin{theorem}
[triangularization with prescribed diagonal]\label{thm.jnf.invtri-diag}Let
$\mathbb{F}$ be a field. Let $A\in\mathbb{F}^{n\times n}$. Assume that the
characteristic polynomial $p_{A}$ of $A$ factors as a product of $n$ linear
factors over $\mathbb{F}$ (so $A$ has $n$ eigenvalues in $\mathbb{F}$, counted
with algebraic multiplicities). Let $\lambda_{1},\lambda_{2},\ldots
,\lambda_{n}$ be the eigenvalues of $A$ (listed with their algebraic
multiplicities). Then, there exists an upper-triangular matrix $T\in
\mathbb{F}^{n\times n}$ such that $A\sim T$ (this means \textquotedblleft$A$
is similar to $T$\textquotedblright, as we recall) and such that the diagonal
entries of $T$ are $\lambda_{1},\lambda_{2},\ldots,\lambda_{n}$ in this order.
\end{theorem}

The proofs of Theorem \ref{thm.schurtri.invtri} and Theorem
\ref{thm.jnf.invtri-diag} are fairly similar to the above proofs of Theorem
\ref{thm.schurtri.schurtri} and Theorem \ref{thm.jnf.schurtri-diag}:

\begin{proof}
[Proof of Theorem \ref{thm.jnf.invtri-diag}.]We proceed by induction on $n$:

\textit{Induction base:} For $n=0$, Theorem \ref{thm.jnf.invtri-diag} holds
trivially\footnote{There is only one $0\times0$-matrix, and we take $T$ to be
this matrix.}.

\textit{Induction step:} Let $m$ be a positive integer. Assume (as the
induction hypothesis) that Theorem \ref{thm.jnf.invtri-diag} is proved for
$n=m-1$. We must prove that Theorem \ref{thm.jnf.invtri-diag} holds for $n=m$.

So let $A\in\mathbb{F}^{m\times m}$ be an $m\times m$-matrix whose
characteristic polynomial $p_{A}$ factors as a product of $m$ linear factors,
and let $\lambda_{1},\lambda_{2},\ldots,\lambda_{m}$ be the eigenvalues of $A$
(listed with their algebraic multiplicities). We must show that there exists
an upper-triangular matrix $T\in\mathbb{F}^{m\times m}$ such that $A\sim T$
and such that the diagonal entries of $T$ are $\lambda_{1},\lambda_{2}%
,\ldots,\lambda_{m}$ in this order.

Since $\lambda_{1}$ is an eigenvalue of $A$, there exists at least one nonzero
$\lambda_{1}$-eigenvector $x$ of $A$. Pick any such $x$. Set $u_{1}:=x$. The
$1$-tuple $\left(  u_{1}\right)  $ of vectors in $\mathbb{F}^{m}$ is then
linearly independent (since $u_{1}=x$ is nonzero).

It is well-known that any linearly independent tuple of vectors in
$\mathbb{F}^{m}$ can be extended to a basis of $\mathbb{F}^{m}$ (if you wish,
this is an analogue of Corollary \ref{cor.unitary.orthon-extend}). Thus, in
particular, the $1$-tuple $\left(  u_{1}\right)  $ of vectors in
$\mathbb{F}^{m}$ can be extended to a basis of $\mathbb{F}^{m}$. In other
words, we can find $m-1$ vectors $u_{2},u_{3},\ldots,u_{m}\in\mathbb{F}^{m}$
such that $\left(  u_{1},u_{2},\ldots,u_{m}\right)  $ is a basis of
$\mathbb{F}^{m}$. Consider these $m-1$ vectors $u_{2},u_{3},\ldots,u_{m}$.

Let $U\in\mathbb{F}^{m\times m}$ be the $m\times m$-matrix whose columns are
$u_{1},u_{2},\ldots,u_{m}$ in this order. Then, the columns of this matrix
form a basis of $\mathbb{F}^{m}$. Hence, the matrix $U$ is invertible.

Let $\left(  e_{1},e_{2},\ldots,e_{m}\right)  $ be the standard basis of the
$\mathbb{F}$-vector space $\mathbb{F}^{m}$. It is known that if $B$ is any
$m\times m$-matrix, then%
\begin{equation}
\text{the }1\text{-st column of }B\text{ is }Be_{1}.
\label{pf.thm.jnf.invtri-diag.col1}%
\end{equation}
Applying this to $B=U$, we see that the $1$-st column of $U$ is $Ue_{1}$.
Since we also know that the $1$-st column of $U$ is $u_{1}$ (by the definition
of $U$), we thus conclude that $Ue_{1}=u_{1}=x$. However, $Ax=\lambda_{1}x$
(since $x$ is a $\lambda_{1}$-eigenvector of $A$). In view of $Ue_{1}=x$, this
rewrites as $AUe_{1}=\lambda_{1}Ue_{1}$.

Define an $m\times m$-matrix%
\[
C:=U^{-1}AU\in\mathbb{F}^{m\times m}.
\]
Thus, $A\sim C$. Hence, Proposition \ref{prop.schurtri.similar.same}
\textbf{(d)} shows that the matrices $A$ and $C$ have the same characteristic
polynomial. In other words, $p_{A}=p_{C}$.

Furthermore, (\ref{pf.thm.jnf.invtri-diag.col1}) shows that the $1$-st column
of the matrix $C$ is%
\begin{align*}
Ce_{1}  &  =U^{-1}\underbrace{AUe_{1}}_{=\lambda_{1}Ue_{1}}%
\ \ \ \ \ \ \ \ \ \ \left(  \text{since }C=U^{-1}AU\right) \\
&  =U^{-1}\lambda_{1}Ue_{1}=\lambda_{1}e_{1}=\left(
\begin{array}
[c]{c}%
\lambda_{1}\\
0\\
0\\
\vdots\\
0
\end{array}
\right)  .
\end{align*}
In other words, the matrix $C$ has the form%
\[
C=\left(
\begin{array}
[c]{ccccc}%
\lambda_{1} & \ast & \ast & \cdots & \ast\\
0 & \ast & \ast & \cdots & \ast\\
0 & \ast & \ast & \cdots & \ast\\
\vdots & \vdots & \vdots & \ddots & \vdots\\
0 & \ast & \ast & \cdots & \ast
\end{array}
\right)  ,
\]
where each asterisk ($\ast$) means an entry that we don't know or don't care
about (in our case, both).

Let us write this in block-matrix notation:%
\begin{equation}
C=\left(
\begin{array}
[c]{cc}%
\lambda_{1} & p\\
0 & B
\end{array}
\right)  , \label{pf.thm.jnf.invtri-diag.C=2x2}%
\end{equation}
where $p\in\mathbb{F}^{1\times\left(  m-1\right)  }$ is a row vector and
$B\in\mathbb{F}^{\left(  m-1\right)  \times\left(  m-1\right)  }$ is a
matrix\footnote{The \textquotedblleft$0$\textquotedblright\ here is actually
the zero vector in $\mathbb{F}^{m-1}$.}. Consider this $p$ and this $B$.

We shall now show that the eigenvalues of $B$ are $\lambda_{2},\lambda
_{3},\ldots,\lambda_{m}$. Indeed, Lemma
\ref{lem.jnf.schurtri-diag.descend-poly} (applied to $n=m-1$ and
$\lambda=\lambda_{1}$) yields $p_{C}=\left(  t-\lambda_{1}\right)  \cdot
p_{B}$ (because of (\ref{pf.thm.jnf.invtri-diag.C=2x2})). Hence,%
\[
p_{A}=p_{C}=\left(  t-\lambda_{1}\right)  \cdot p_{B}.
\]
On the other hand,%
\[
p_{A}=\left(  t-\lambda_{1}\right)  \left(  t-\lambda_{2}\right)
\cdots\left(  t-\lambda_{m}\right)
\]
(since $p_{A}$ is monic, and the roots of $p_{A}$ are precisely the
eigenvalues of $A$ with algebraic multiplicities, which we know are
$\lambda_{1},\lambda_{2},\ldots,\lambda_{m}$). Comparing these two equalities,
we obtain%
\[
\left(  t-\lambda_{1}\right)  \cdot p_{B}=\left(  t-\lambda_{1}\right)
\left(  t-\lambda_{2}\right)  \cdots\left(  t-\lambda_{m}\right)  .
\]
We can cancel the factor $t-\lambda_{1}$ from both sides of this equality
(since the polynomial ring over $\mathbb{F}$ has no zero-divisors). Thus, we
obtain%
\[
p_{B}=\left(  t-\lambda_{2}\right)  \left(  t-\lambda_{3}\right)
\cdots\left(  t-\lambda_{m}\right)  .
\]
In other words, the eigenvalues of the $\left(  m-1\right)  \times\left(
m-1\right)  $-matrix $B$ are $\lambda_{2},\lambda_{3},\ldots,\lambda_{m}$
(listed with their algebraic multiplicities).

Hence, by the induction hypothesis, we can apply Theorem
\ref{thm.jnf.invtri-diag} to $m-1$, $B$ and $\lambda_{2},\lambda_{3}%
,\ldots,\lambda_{m}$ instead of $n$, $A$ and $\lambda_{1},\lambda_{2}%
,\ldots,\lambda_{n}$. As a result, we conclude that there exists an
upper-triangular matrix $S\in\mathbb{F}^{\left(  m-1\right)  \times\left(
m-1\right)  }$ such that $B\sim S$ and such that the diagonal entries of $S$
are $\lambda_{2},\lambda_{3},\ldots,\lambda_{m}$ in this order. Consider this
$S$.

We have $B\sim S$. In other words, there is a invertible matrix $V\in
\mathbb{F}^{\left(  m-1\right)  \times\left(  m-1\right)  }$ such that
$S=VBV^{-1}$. Consider this $V$.

Now, let%
\[
W:=\left(
\begin{array}
[c]{cc}%
1 & 0\\
0 & V
\end{array}
\right)  \in\mathbb{F}^{m\times m}\ \ \ \ \ \ \ \ \ \ \left(  \text{in
block-matrix notation}\right)  .
\]
This is a block-diagonal matrix, with $\left(
\begin{array}
[c]{c}%
1
\end{array}
\right)  $ and $V$ being its diagonal blocks. Hence, $W$ is invertible with
$W^{-1}=\left(
\begin{array}
[c]{cc}%
1 & 0\\
0 & V^{-1}%
\end{array}
\right)  $. Therefore, $C\sim WCW^{-1}$. Combining $A\sim C$ with $C\sim
WCW^{-1}$, we obtain $A\sim WCW^{-1}$ (by Proposition
\ref{prop.schurtri.similar.eqrel} \textbf{(c)}).

However,%
\begin{align*}
&  \underbrace{W}_{=\left(
\begin{array}
[c]{cc}%
1 & 0\\
0 & V
\end{array}
\right)  }\ \ \underbrace{C}_{=\left(
\begin{array}
[c]{cc}%
\lambda_{1} & p\\
0 & B
\end{array}
\right)  }\ \ \underbrace{W^{-1}}_{=\left(
\begin{array}
[c]{cc}%
1 & 0\\
0 & V^{-1}%
\end{array}
\right)  }\\
&  =\left(
\begin{array}
[c]{cc}%
1 & 0\\
0 & V
\end{array}
\right)  \left(
\begin{array}
[c]{cc}%
\lambda_{1} & p\\
0 & B
\end{array}
\right)  \left(
\begin{array}
[c]{cc}%
1 & 0\\
0 & V^{-1}%
\end{array}
\right) \\
&  =\left(
\begin{array}
[c]{cc}%
1\cdot\lambda_{1}\cdot1 & 1\cdot p\cdot V^{-1}\\
V\cdot0\cdot1 & V\cdot B\cdot V^{-1}%
\end{array}
\right)  \ \ \ \ \ \ \ \ \ \ \left(
\begin{array}
[c]{c}%
\text{by using Proposition \ref{prop.blockmatrix.mult-2x2} twice}\\
\text{and simplifying the }0\text{ addends away}%
\end{array}
\right) \\
&  =\left(
\begin{array}
[c]{cc}%
\lambda_{1} & pV^{-1}\\
0 & VBV^{-1}%
\end{array}
\right)  =\left(
\begin{array}
[c]{cc}%
\lambda_{1} & pV^{-1}\\
0 & S
\end{array}
\right)  \ \ \ \ \ \ \ \ \ \ \left(  \text{since }VBV^{-1}=S\right)  .
\end{align*}
This matrix $WCW^{-1}$ is therefore upper-triangular (since the bottom-left
block is a zero vector, and since the bottom-right block $S$ is
upper-triangular), and its diagonal entries are $\lambda_{1},\lambda
_{2},\ldots,\lambda_{m}$ in this order (because its first diagonal entry is
visibly $\lambda_{1}$, whereas its remaining diagonal entries are the diagonal
entries of $S$ and therefore are $\lambda_{2},\lambda_{3},\ldots,\lambda_{m}$
in this order\footnote{In fact, we have shown above that the diagonal entries
of $S$ are $\lambda_{2},\lambda_{3},\ldots,\lambda_{m}$ in this order.}).

Thus, there exists an upper-triangular matrix $T\in\mathbb{F}^{m\times m}$
such that $A\sim T$ and such that the diagonal entries of $T$ are $\lambda
_{1},\lambda_{2},\ldots,\lambda_{m}$ in this order (namely, $WCW^{-1}$ is such
a matrix, because $A\sim WCW^{-1}$ and because of what we just said).

Thus, we have shown that Theorem \ref{thm.jnf.invtri-diag} holds for $n=m$.
This completes the induction step. The proof of Theorem
\ref{thm.jnf.invtri-diag} is thus complete.
\end{proof}

\begin{proof}
[Proof of Theorem \ref{thm.schurtri.invtri}.]Theorem \ref{thm.schurtri.invtri}
follows from Theorem \ref{thm.jnf.invtri-diag} (and the definition of similarity).
\end{proof}

\subsection{Commuting matrices}

Next, we shall generalize Theorem \ref{thm.schurtri.schurtri} from the case of
a single matrix to the case of several matrices that pairwise commute.

\begin{definition}
Two $n\times n$-matrices $A$ and $B$ are said to \emph{commute} if $AB=BA$.
\end{definition}

Examples of commuting matrices are easy to find; e.g., any two powers of a
single matrix commute (i.e., we have $A^{k}A^{\ell}=A^{\ell}A^{k}$ for any
$n\times n$-matrix $A$ and any $k,\ell\in\mathbb{N}$). Also, any two diagonal
matrices (of the same size) commute. But there are many more situations in
which matrices commute. In this section, we shall extend Schur
triangularization from a single matrix to a family of commuting matrices.

First, we need a lemma (\cite[Lemma 1.3.19]{HorJoh13}) which says that any
family of pairwise commuting matrices in $\mathbb{C}^{n\times n}$ has a common eigenvector:

\begin{lemma}
\label{lem.schurtri.commute.1}Let $n>0$. Let $\mathcal{F}$ be a subset of
$\mathbb{C}^{n\times n}$ such that any two matrices in $\mathcal{F}$ commute
(i.e., any $A\in\mathcal{F}$ and $B\in\mathcal{F}$ satisfy $AB=BA$).

Then, there exists a nonzero vector $x\in\mathbb{C}^{n}$ such that $x$ is an
eigenvector of each $A\in\mathcal{F}$.
\end{lemma}

\begin{proof}
We shall use the following conventions:

\begin{itemize}
\item An $\mathcal{F}$\emph{-eigenvector} will mean a vector $x\in
\mathbb{C}^{n}$ such that $x$ is an eigenvector of each $A\in\mathcal{F}$.
Thus, our goal is to show that there exists a nonzero $\mathcal{F}$-eigenvector.

\item A \emph{subspace} shall mean a $\mathbb{C}$-vector subspace of
$\mathbb{C}^{n}$.

\item A subspace $W$ of $\mathbb{C}^{n}$ is said to be \emph{nontrivial} if it
contains a nonzero vector (i.e., if its dimension is $>0$).

\item A subspace $W$ of $\mathbb{C}^{n}$ is said to be $\mathcal{F}%
$\emph{-invariant} if every $A\in\mathcal{F}$ and every $w\in W$ satisfy
$Aw\in W$. (This means that applying a matrix $A\in\mathcal{F}$ to a vector
$w\in W$ gives a vector in $W$ again -- i.e., that there is no way to
\textquotedblleft escape\textquotedblright\ $W$ by applying matrices
$A\in\mathcal{F}$.)

[\textbf{Example:} If $n=2$ and $\mathcal{F}=\left\{  \left(
\begin{array}
[c]{cc}%
3 & a\\
0 & 2
\end{array}
\right)  \ \mid\ a\in\mathbb{R}\right\}  $, then $\operatorname*{span}\left(
e_{1}\right)  =\left\{  \left(
\begin{array}
[c]{c}%
x\\
0
\end{array}
\right)  \ \mid\ x\in\mathbb{C}\right\}  $ is an $\mathcal{F}$-invariant
subspace, because every $A=\left(
\begin{array}
[c]{cc}%
3 & a\\
0 & 2
\end{array}
\right)  \in\mathcal{F}$ and every $w=\left(
\begin{array}
[c]{c}%
x\\
0
\end{array}
\right)  \in\operatorname*{span}\left(  e_{1}\right)  $ satisfy%
\[
Aw=\left(
\begin{array}
[c]{cc}%
3 & a\\
0 & 2
\end{array}
\right)  \left(
\begin{array}
[c]{c}%
x\\
0
\end{array}
\right)  =\left(
\begin{array}
[c]{c}%
3x\\
0
\end{array}
\right)  \in\operatorname*{span}\left(  e_{1}\right)  .
\]
More generally, if the set $\mathcal{F}\subseteq\mathbb{C}^{n\times n}$
consists entirely of upper-triangular matrices, then each of the subspaces
\[
\operatorname*{span}\left(  e_{1},e_{2},\ldots,e_{k}\right)  =\left\{
x\in\mathbb{C}^{n}\ \mid\ \text{the last }n-k\text{ coordinates of }x\text{
are }0\right\}
\]
is $\mathcal{F}$-invariant, because the product $Ax$ of an upper-triangular
matrix $A\in\mathbb{C}^{n\times n}$ with a vector $x\in\operatorname*{span}%
\left(  e_{1},e_{2},\ldots,e_{k}\right)  $ is always a vector in
$\operatorname*{span}\left(  e_{1},e_{2},\ldots,e_{k}\right)  $.]
\end{itemize}

It is clear that the trivial subspace $\left\{  0\right\}  $ is $\mathcal{F}%
$-invariant; so is the whole space $\mathbb{C}^{n}$ itself. The latter shows
that there exists at least one nontrivial $\mathcal{F}$-invariant subspace
(namely, $\mathbb{C}^{n}$).

Now, we shall show the following crucial claim:

\begin{statement}
\textit{Claim 1:} Let $W$ be a nontrivial $\mathcal{F}$-invariant subspace.
Let $w\in W$. Assume that $w$ is not an $\mathcal{F}$-eigenvector. Then, there
exists a nontrivial $\mathcal{F}$-invariant subspace $V$ such that $V$ is a
\textbf{proper subset} of $W$.
\end{statement}

Roughly speaking, this claim is saying that if a nontrivial $\mathcal{F}%
$-invariant subspace contains a vector $w$ that is not an $\mathcal{F}%
$-eigenvector, then there is a smaller nontrivial $\mathcal{F}$-invariant
subspace inside it. This fact (once proved) allows us to start with any
nontrivial $\mathcal{F}$-invariant subspace (for instance, $\mathbb{C}^{n}$
itself), and then successively replace it by smaller and smaller subspaces
until we eventually find a nontrivial $\mathcal{F}$-invariant subspace that
consists entirely of $\mathcal{F}$-eigenvectors. This will then yield the
existence of a nonzero $\mathcal{F}$-eigenvector, and thus Lemma
\ref{lem.schurtri.commute.1} will be proved. (We shall walk through this
argument in more detail after proving Claim 1.)

So let us now prove Claim 1:

[\textit{Proof of Claim 1:} We know that $w$ is not an $\mathcal{F}%
$-eigenvector. In other words, there exists some $B\in\mathcal{F}$ such that
$w$ is not an eigenvector of $B$. Consider this $B$.

We have $Bv\in W$ for each $v\in W$ (since $W$ is $\mathcal{F}$-invariant).
Hence, the map%
\begin{align*}
f:W  &  \rightarrow W,\\
v  &  \mapsto Bv
\end{align*}
is well-defined. This map $f$ is furthermore $\mathbb{C}$-linear (for obvious
reasons). Also, $\dim W>0$ (because $W$ is nontrivial).

However, it is well-known that any linear map from a finite-dimensional
$\mathbb{C}$-vector space to itself has at least one nonzero eigenvector,
provided that this vector space has dimension $>0$. We thus conclude that the
linear map $f:W\rightarrow W$ has at least one nonzero
eigenvector\footnote{since $f$ is a $\mathbb{C}$-linear map from the
finite-dimensional $\mathbb{C}$-vector space $W$ to itself, and since $\dim
W>0$}. Pick such a nonzero eigenvector $x$, and let $\lambda\in\mathbb{C}$ be
the corresponding eigenvalue. Thus, $x\in W$ and $v\neq0$ and $f\left(
x\right)  =\lambda x$. Since $f\left(  x\right)  =Bx$ (by the definition of
$f$), we can rewrite the latter equality as $Bx=\lambda x$. In contrast,
$Bw\neq\lambda w$ (since $w$ is not an eigenvector of $B$).

Define a subset $V$ of $W$ by
\[
V:=\left\{  v\in W\ \mid\ Bv=\lambda v\right\}  .
\]
It is easy to see that $V$ is a subspace (since $B\left(  u+v\right)  =Bu+Bv$
and $B\left(  \mu v\right)  =\mu\cdot Bv$ for any $u,v\in W$ and $\mu
\in\mathbb{C}$). Furthermore, this subspace $V$ contains the nonzero vector
$x$ (since $Bx=\lambda x$), and thus is nontrivial. However, this subspace $V$
does not contain $w$ (because if it did, then we would have $Bw=\lambda w$ (by
the definition of $V$), which would contradict $Bw\neq\lambda w$). Thus,
$V\neq W$ (since $W$ does contain $w$). Therefore, $V$ is a \textbf{proper}
subset of $W$ (since $V$ is a subset of $W$).

Let us now show that this subspace $V$ is $\mathcal{F}$-invariant. Indeed, let
$A\in\mathcal{F}$ and $v\in V$ be arbitrary. We shall show that $Av\in V$.

We have $v\in V\subseteq W$ and therefore $Av\in W$ (since $W$ is
$\mathcal{F}$-invariant). Moreover, $Bv=\lambda v$ (since $v\in V$).
Furthermore, $B$ and $A$ commute (since any two matrices in $\mathcal{F}$
commute); thus, $BA=AB$. Hence, $BAv=A\underbrace{Bv}_{=\lambda v}%
=\lambda\cdot Av$. Thus, $Av$ is a vector $z\in W$ that satisfies $Bz=\lambda
z$ (since $Av\in W$). In other words, $Av\in V$ (by the definition of $V$).

Forget that we fixed $A$ and $v$. We thus have shown that every $A\in
\mathcal{F}$ and every $v\in V$ satisfy $Av\in V$. In other words, the
subspace $V$ is $\mathcal{F}$-invariant.

Thus, we have found a nontrivial $\mathcal{F}$-invariant subspace $V$ such
that $V$ is a \textbf{proper subset} of $W$. This proves Claim 1.] \medskip

Now, we can complete the proof of Lemma \ref{lem.schurtri.commute.1} (using
the strategy we outlined above):

We must prove that there exists a nonzero $\mathcal{F}$-eigenvector. Assume
the contrary. Thus, there exists no nonzero $\mathcal{F}$-eigenvector.

We recursively construct a sequence $\left(  V_{0},V_{1},V_{2},V_{3}%
,\ldots\right)  $ of nontrivial $\mathcal{F}$-invariant subspaces as follows:

\begin{itemize}
\item We begin by setting $V_{0}:=\mathbb{C}^{n}$. (This subspace is indeed
$\mathcal{F}$-invariant, and furthermore is nontrivial because $n>0$.)

\item For each $i\in\mathbb{N}$, if the nontrivial $\mathcal{F}$-invariant
subspace $V_{i}$ has already been constructed, we define the next subspace
$V_{i+1}$ as follows: We pick an arbitrary nonzero vector $w\in V_{i}$. (Such
a $w$ exists, since $V_{i}$ is nontrivial.) This $w$ cannot be an
$\mathcal{F}$-eigenvector (since there exists no nonzero $\mathcal{F}%
$-eigenvector). Hence, Claim 1 (applied to $W=V_{i}$) yields that there exists
a nontrivial $\mathcal{F}$-invariant subspace $V$ such that $V$ is a
\textbf{proper subset} of $V_{i}$. We choose such a $V$ and define
$V_{i+1}:=V$.
\end{itemize}

Thus, we obtain a sequence $\left(  V_{0},V_{1},V_{2},V_{3},\ldots\right)  $
of subspaces such that each subspace $V_{i+1}$ in this sequence is a
\textbf{proper subset} of the preceding subspace $V_{i}$. Hence, for each
$i\in\mathbb{N}$, we have $\dim\left(  V_{i+1}\right)  <\dim\left(
V_{i}\right)  $ (since $V_{i}$ and $V_{i+1}$ are finite-dimensional vector
spaces). Equivalently, for each $i\in\mathbb{N}$, we have $\dim\left(
V_{i}\right)  >\dim\left(  V_{i+1}\right)  $. In other words,%
\[
\dim\left(  V_{0}\right)  >\dim\left(  V_{1}\right)  >\dim\left(
V_{2}\right)  >\cdots.
\]
Hence, the sequence $\left(  \dim\left(  V_{0}\right)  ,\ \dim\left(
V_{1}\right)  ,\ \dim\left(  V_{2}\right)  ,\ \ldots\right)  $ is a strictly
decreasing infinite sequence of nonnegative integers. However, this is absurd,
since there exists no strictly decreasing infinite sequence of nonnegative
integers\footnote{This is an example of a
\href{https://en.wikipedia.org/wiki/Proof_by_infinite_descent}{\textquotedblleft
proof by infinite descent\textquotedblright}.}. Thus, we have obtained a
contradiction. This proves Lemma \ref{lem.schurtri.commute.1}.
\end{proof}

We can now generalize Theorem \ref{thm.schurtri.schurtri} to families of
commuting matrices:

\begin{theorem}
\label{thm.schurtri.commute.schurtri}Let $\mathcal{F}$ be a subset of
$\mathbb{C}^{n\times n}$ such that any two matrices in $\mathcal{F}$ commute
(i.e., any $A\in\mathcal{F}$ and $B\in\mathcal{F}$ satisfy $AB=BA$).

Then, there exists a unitary matrix $U\in\operatorname*{U}\nolimits_{n}\left(
\mathbb{C}\right)  $ such that for each $A\in\mathcal{F}$, the matrix
$UAU^{\ast}$ is upper-triangular.
\end{theorem}

\begin{proof}
This can be proved by an induction on $n$, similarly to Theorem
\ref{thm.jnf.schurtri-diag}. But now, instead of picking an eigenvector $x$ of
a single matrix $A$, we pick a common eigenvector for all matrices in
$\mathcal{F}$. The existence of such an eigenvector is guaranteed by Lemma
\ref{lem.schurtri.commute.1}.

Here is the argument in more detail:

We proceed by induction on $n$:

\textit{Induction base:} For $n=0$, Theorem
\ref{thm.schurtri.commute.schurtri} holds trivially\footnote{because any
$0\times0$-matrix is upper-triangular}.

\textit{Induction step:} Let $m$ be a positive integer. Assume (as the
induction hypothesis) that Theorem \ref{thm.schurtri.commute.schurtri} is
proved for $n=m-1$. We must prove that Theorem
\ref{thm.schurtri.commute.schurtri} holds for $n=m$.

So let $\mathcal{F}$ be a subset of $\mathbb{C}^{m\times m}$ such that any two
matrices in $\mathcal{F}$ commute (i.e., any $A\in\mathcal{F}$ and
$B\in\mathcal{F}$ satisfy $AB=BA$). We must show that there exists a unitary
matrix $Q\in\operatorname*{U}\nolimits_{m}\left(  \mathbb{C}\right)  $ such
that for each $A\in\mathcal{F}$, the matrix $QAQ^{\ast}$ is upper-triangular.

Lemma \ref{lem.schurtri.commute.1} (applied to $n=m$) shows that there exists
a nonzero vector $x\in\mathbb{C}^{m}$ such that $x$ is an eigenvector of each
$A\in\mathcal{F}$. Pick any such $x$. Set $u_{1}:=\dfrac{1}{\left\vert
\left\vert x\right\vert \right\vert }x$. Then, $u_{1}$ is still an eigenvector
of each $A\in\mathcal{F}$, but additionally satisfies $\left\vert \left\vert
u_{1}\right\vert \right\vert =1$. Hence, the $1$-tuple $\left(  u_{1}\right)
$ of vectors in $\mathbb{C}^{m}$ is orthonormal.

Thus, Corollary \ref{cor.unitary.orthon-extend} (applied to $k=1$ and $n=m$)
shows that we can find $m-1$ vectors $u_{2},u_{3},\ldots,u_{m}\in
\mathbb{C}^{m}$ such that $\left(  u_{1},u_{2},\ldots,u_{m}\right)  $ is an
orthonormal basis of $\mathbb{C}^{m}$. Consider these $m-1$ vectors
$u_{2},u_{3},\ldots,u_{m}$.

Let $U\in\mathbb{C}^{m\times m}$ be the $m\times m$-matrix whose columns are
$u_{1},u_{2},\ldots,u_{m}$ in this order. Then, the columns of this matrix
form an orthonormal basis of $\mathbb{C}^{m}$. Hence, Theorem
\ref{thm.unitary.unitary.eqs} (specifically, the implication $\mathcal{E}%
\Longrightarrow\mathcal{A}$ in this theorem) yields that the matrix $U$ is
unitary. Therefore, $U^{\ast}=U^{-1}$. Moreover, since $U$ is unitary, we have
$UU^{\ast}=I_{m}$ and $U^{\ast}U=I_{m}$. Thus, the matrix $U^{\ast}$ is unitary.

For each $A\in\mathcal{F}$, we now define an $m\times m$-matrix%
\[
C_{A}:=U^{\ast}AU\in\mathbb{C}^{m\times m}.
\]


Now, let $A\in\mathcal{F}$ be arbitrary. We know that $u_{1}$ is an
eigenvector of $A$; let $\lambda_{A}$ be the corresponding eigenvalue. Now, it
is not hard to show\footnote{Indeed, this is essentially a carbon copy of the
proof of (\ref{pf.thm.jnf.schurtri-diag.C=2x2}) in our above proof of Theorem
\ref{thm.jnf.schurtri-diag} (except that $C$, $\lambda_{1}$, $p$ and $B$ are
now called $C_{A}$, $\lambda_{A}$, $p_{A}$ and $B_{A}$); thus, we leave the
details to the reader.} that the matrix $C_{A}$ can be written in block-matrix
notation as%
\begin{equation}
C_{A}=\left(
\begin{array}
[c]{cc}%
\lambda_{A} & p_{A}\\
0 & B_{A}%
\end{array}
\right)  , \label{pf.thm.schurtri.commute.schurtri.C=2x2}%
\end{equation}
where $p_{A}\in\mathbb{C}^{1\times\left(  m-1\right)  }$ is a row vector and
$B_{A}\in\mathbb{C}^{\left(  m-1\right)  \times\left(  m-1\right)  }$ is a
matrix\footnote{The \textquotedblleft$0$\textquotedblright\ here is actually
the zero vector in $\mathbb{C}^{m-1}$.}. Consider this $p_{A}$ and this
$B_{A}$.

Forget that we fixed $A$. Thus, for each $A\in\mathcal{F}$, we have defined a
complex number $\lambda_{A}$, a row vector $p_{A}\in\mathbb{C}^{1\times\left(
m-1\right)  }$ and a matrix $B_{A}\in\mathbb{C}^{\left(  m-1\right)
\times\left(  m-1\right)  }$ such that
(\ref{pf.thm.schurtri.commute.schurtri.C=2x2}) holds.

Let $A\in\mathcal{F}$ and $A^{\prime}\in\mathcal{F}$ be arbitrary. We are
going to show that $B_{AA^{\prime}}=B_{A}B_{A^{\prime}}$.

Indeed, we first observe that the definitions of $C_{A}$ and $C_{A^{\prime}}$
yield%
\begin{align}
\underbrace{C_{A}}_{=U^{\ast}AU}\ \ \underbrace{C_{A^{\prime}}}_{=U^{\ast
}A^{\prime}U}  &  =U^{\ast}A\underbrace{UU^{\ast}}_{=I_{m}}A^{\prime}%
U=U^{\ast}AA^{\prime}U\nonumber\\
&  =C_{AA^{\prime}} \label{pf.thm.schurtri.commute.schurtri.CACA'=}%
\end{align}
(by the definition of $C_{AA^{\prime}}$). However,
(\ref{pf.thm.schurtri.commute.schurtri.C=2x2}) yields%
\begin{align*}
C_{A}  &  =\left(
\begin{array}
[c]{cc}%
\lambda_{A} & p_{A}\\
0 & B_{A}%
\end{array}
\right)  ,\ \ \ \ \ \ \ \ \ \ C_{A^{\prime}}=\left(
\begin{array}
[c]{cc}%
\lambda_{A^{\prime}} & p_{A^{\prime}}\\
0 & B_{A^{\prime}}%
\end{array}
\right)  ,\\
\text{and}\ \ \ \ \ \ \ \ \ \ C_{AA^{\prime}}  &  =\left(
\begin{array}
[c]{cc}%
\lambda_{AA^{\prime}} & p_{AA^{\prime}}\\
0 & B_{AA^{\prime}}%
\end{array}
\right)  .
\end{align*}
In view of this, we can rewrite (\ref{pf.thm.schurtri.commute.schurtri.CACA'=}%
) as%
\[
\left(
\begin{array}
[c]{cc}%
\lambda_{A} & p_{A}\\
0 & B_{A}%
\end{array}
\right)  \left(
\begin{array}
[c]{cc}%
\lambda_{A^{\prime}} & p_{A^{\prime}}\\
0 & B_{A^{\prime}}%
\end{array}
\right)  =\left(
\begin{array}
[c]{cc}%
\lambda_{AA^{\prime}} & p_{AA^{\prime}}\\
0 & B_{AA^{\prime}}%
\end{array}
\right)  .
\]
Hence,%
\[
\left(
\begin{array}
[c]{cc}%
\lambda_{AA^{\prime}} & p_{AA^{\prime}}\\
0 & B_{AA^{\prime}}%
\end{array}
\right)  =\left(
\begin{array}
[c]{cc}%
\lambda_{A} & p_{A}\\
0 & B_{A}%
\end{array}
\right)  \left(
\begin{array}
[c]{cc}%
\lambda_{A^{\prime}} & p_{A^{\prime}}\\
0 & B_{A^{\prime}}%
\end{array}
\right)  =\left(
\begin{array}
[c]{cc}%
\lambda_{A}\lambda_{A^{\prime}} & \lambda_{A}p_{A^{\prime}}+p_{A}B_{A^{\prime
}}\\
0 & B_{A}B_{A^{\prime}}%
\end{array}
\right)
\]
(by applying Proposition \ref{prop.blockmatrix.mult-2x2} and simplifying).
Comparing the bottom-right blocks of the block matrices on both sides, we
obtain $B_{AA^{\prime}}=B_{A}B_{A^{\prime}}$.

Similarly, $B_{A^{\prime}A}=B_{A^{\prime}}B_{A}$. However, $AA^{\prime
}=A^{\prime}A$ (since any two matrices in $\mathcal{F}$ commute). Thus,
$B_{AA^{\prime}}=B_{A^{\prime}A}$. In view of $B_{AA^{\prime}}=B_{A}%
B_{A^{\prime}}$ and $B_{A^{\prime}A}=B_{A^{\prime}}B_{A}$, we can rewrite this
as
\begin{equation}
B_{A}B_{A^{\prime}}=B_{A^{\prime}}B_{A}.
\label{pf.thm.schurtri.commute.schurtri.BABA'comm}%
\end{equation}


Forget that we fixed $A$ and $A^{\prime}$. We thus have proved
(\ref{pf.thm.schurtri.commute.schurtri.BABA'comm}) for all $A\in\mathcal{F}$
and $A^{\prime}\in\mathcal{F}$.

Define a set%
\[
\mathcal{F}^{\prime}:=\left\{  B_{A}\ \mid\ A\in\mathcal{F}\right\}
\subseteq\mathbb{C}^{\left(  m-1\right)  \times\left(  m-1\right)  }.
\]
Then, (\ref{pf.thm.schurtri.commute.schurtri.BABA'comm}) shows that any two
matrices in $\mathcal{F}^{\prime}$ commute.

Hence, by the induction hypothesis, we can apply Theorem
\ref{thm.schurtri.commute.schurtri} to $m-1$ and $\mathcal{F}^{\prime}$
instead of $n$ and $\mathcal{F}$. As a result, we conclude that there exists a
unitary matrix $V\in\operatorname*{U}\nolimits_{m-1}\left(  \mathbb{C}\right)
$ such that for each $B\in\mathcal{F}^{\prime}$, the matrix $VBV^{\ast}$ is
upper-triangular. Consider this $V$.

Now, let%
\[
W:=\left(
\begin{array}
[c]{cc}%
1 & 0\\
0 & V
\end{array}
\right)  \in\mathbb{C}^{m\times m}\ \ \ \ \ \ \ \ \ \ \left(  \text{in
block-matrix notation}\right)  .
\]
This is a block-diagonal matrix, with $\left(
\begin{array}
[c]{c}%
1
\end{array}
\right)  $ and $V$ being its diagonal blocks. Hence, $W^{\ast}=\left(
\begin{array}
[c]{cc}%
1 & 0\\
0 & V^{\ast}%
\end{array}
\right)  $. Moreover, $W$ is a unitary matrix (since it is a block-diagonal
matrix whose diagonal blocks are unitary\footnote{We are using Proposition
\ref{prop.blockmatrix.unitary-diag} here.}).

Now, let $Q$ be the matrix $WU^{\ast}$. Then, $Q$ is unitary (by Exercise
\ref{exe.unitary.group} \textbf{(b)}, since $W$ and $U^{\ast}$ are unitary).
We shall show that for each $A\in\mathcal{F}$, the matrix $QAQ^{\ast}$ is upper-triangular.

Indeed, let $A\in\mathcal{F}$. Then, $B_{A}\in\mathcal{F}^{\prime}$ (by the
definition of $\mathcal{F}^{\prime}$). However, we know that for each
$B\in\mathcal{F}^{\prime}$, the matrix $VBV^{\ast}$ is upper-triangular.
Applying this to $B=B_{A}$, we conclude that the matrix $VB_{A}V^{\ast}$ is
upper-triangular. On the other hand, from $Q=WU^{\ast}$, we obtain%
\begin{align*}
QAQ^{\ast}  &  =WU^{\ast}A\underbrace{\left(  WU^{\ast}\right)  ^{\ast}%
}_{=\left(  U^{\ast}\right)  ^{\ast}W^{\ast}=UW^{\ast}}=W\underbrace{U^{\ast
}AU}_{\substack{=C_{A}\\\text{(by the definition of }C_{A}\text{)}}}W^{\ast}\\
&  =\underbrace{W}_{=\left(
\begin{array}
[c]{cc}%
1 & 0\\
0 & V
\end{array}
\right)  }\ \ \underbrace{C_{A}}_{\substack{=\left(
\begin{array}
[c]{cc}%
\lambda_{A} & p_{A}\\
0 & B_{A}%
\end{array}
\right)  \\\text{(by (\ref{pf.thm.schurtri.commute.schurtri.C=2x2}))}%
}}\ \ \underbrace{W^{\ast}}_{=\left(
\begin{array}
[c]{cc}%
1 & 0\\
0 & V^{\ast}%
\end{array}
\right)  }\\
&  =\left(
\begin{array}
[c]{cc}%
1 & 0\\
0 & V
\end{array}
\right)  \left(
\begin{array}
[c]{cc}%
\lambda_{A} & p_{A}\\
0 & B_{A}%
\end{array}
\right)  \left(
\begin{array}
[c]{cc}%
1 & 0\\
0 & V^{\ast}%
\end{array}
\right) \\
&  =\left(
\begin{array}
[c]{cc}%
1\cdot\lambda_{A}\cdot1 & 1\cdot p_{A}\cdot V^{\ast}\\
V\cdot0\cdot1 & V\cdot B_{A}\cdot V^{\ast}%
\end{array}
\right)  \ \ \ \ \ \ \ \ \ \ \left(
\begin{array}
[c]{c}%
\text{by using Proposition \ref{prop.blockmatrix.mult-2x2} twice and}\\
\text{simplifying the }0\text{ addends away}%
\end{array}
\right) \\
&  =\left(
\begin{array}
[c]{cc}%
\lambda_{A} & p_{A}V^{\ast}\\
0 & VB_{A}V^{\ast}%
\end{array}
\right)  .
\end{align*}
This matrix $QAQ^{\ast}$ is therefore upper-triangular (since its bottom-right
block $VB_{A}V^{\ast}$ is upper-triangular).

Forget that we fixed $A$. We thus have shown that for each $A\in\mathcal{F}$,
the matrix $QAQ^{\ast}$ is upper-triangular. Since $Q$ is unitary, we thus
have found a unitary matrix $Q\in\operatorname*{U}\nolimits_{m}\left(
\mathbb{C}\right)  $ such that for each $A\in\mathcal{F}$, the matrix
$QAQ^{\ast}$ is upper-triangular.

Thus, we have shown that Theorem \ref{thm.schurtri.commute.schurtri} holds for
$n=m$. This completes the induction step. The proof of Theorem
\ref{thm.schurtri.commute.schurtri} is thus complete.
\end{proof}

%

\[%
\begin{tabular}
[c]{||l||}\hline\hline
\textbf{Lecture 5 starts here.}\\\hline\hline
\end{tabular}
\]


\subsection{Normal matrices}

We next define a fairly wide class of matrices with complex entries that
contains several of our familiar classes as subsets:

\begin{definition}
\label{def.schurtri.normal.normal}A square matrix $A\in\mathbb{C}^{n\times n}$
is said to be \emph{normal} if $AA^{\ast}=A^{\ast}A$.
\end{definition}

In other words, a square matrix is normal if it commutes with its own
conjugate transpose. This is not the most intuitive notion (nor is the word
\textquotedblleft normal\textquotedblright\ particularly expressive), so we
shall give some examples:

\begin{example}
\textbf{(a)} Let $A=\left(
\begin{array}
[c]{cc}%
1 & -1\\
1 & 1
\end{array}
\right)  \in\mathbb{C}^{2\times2}$. Then, the matrix $A$ is normal. Indeed,
its conjugate transpose is $A^{\ast}=\left(
\begin{array}
[c]{cc}%
1 & 1\\
-1 & 1
\end{array}
\right)  $, and it is easily seen that $AA^{\ast}=A^{\ast}A=2I_{2}$. \medskip

\textbf{(b)} Let $B=\left(
\begin{array}
[c]{cc}%
0 & i\\
0 & 0
\end{array}
\right)  \in\mathbb{C}^{2\times2}$. Then, the matrix $B$ is not normal.
Indeed, $B^{\ast}=\left(
\begin{array}
[c]{cc}%
0 & 0\\
-i & 0
\end{array}
\right)  $ and thus $BB^{\ast}\neq B^{\ast}B$, as can easily be verified.
\medskip

\textbf{(c)} Let $a,b\in\mathbb{C}$ be arbitrary, and let $C=\left(
\begin{array}
[c]{cc}%
a & b\\
b & a
\end{array}
\right)  \in\mathbb{C}^{2\times2}$. Then, $C$ is normal. Indeed, $C^{\ast
}=\left(
\begin{array}
[c]{cc}%
\overline{a} & \overline{b}\\
\overline{b} & \overline{a}%
\end{array}
\right)  $, so that it is easy to check that both $CC^{\ast}$ and $C^{\ast}C$
equal $\left(
\begin{array}
[c]{cc}%
a & b\\
b & a
\end{array}
\right)  \left(
\begin{array}
[c]{cc}%
\overline{a} & \overline{b}\\
\overline{b} & \overline{a}%
\end{array}
\right)  =\left(
\begin{array}
[c]{cc}%
a\overline{a}+b\overline{b} & a\overline{b}+b\overline{a}\\
a\overline{b}+b\overline{a} & a\overline{a}+b\overline{b}%
\end{array}
\right)  $.
\end{example}

As we promised, several familiar classes of matrices are normal. We recall a definition:

\begin{definition}
\label{def.schurtri.herm}A square matrix $H\in\mathbb{C}^{n\times n}$ is said
to be \emph{Hermitian} if and only if $H^{\ast}=H$.
\end{definition}

For example, the matrix $\left(
\begin{array}
[c]{cc}%
1 & i\\
-i & 2
\end{array}
\right)  $ is Hermitian. Any real symmetric matrix (i.e., any symmetric matrix
with real entries) is Hermitian as well.

In contrast, a square matrix $S\in\mathbb{C}^{n\times n}$ is skew-Hermitian if
and only if $S^{\ast}=-S$ (by Definition \ref{def.unitary.skew-herm}).
Finally, a square matrix $U\in\mathbb{C}^{n\times n}$ is unitary if and only
if $UU^{\ast}=U^{\ast}U=I_{n}$ (by Theorem \ref{thm.unitary.unitary.eqs},
equivalence $\mathcal{A}\Longleftrightarrow\mathcal{C}$). Having recalled all
these concepts, we can state the following:

\begin{proposition}
\label{prop.schurtri.normal.classes}\textbf{(a)} Every Hermitian matrix
$H\in\mathbb{C}^{n\times n}$ is normal. \medskip

\textbf{(b)} Every skew-Hermitian matrix $S\in\mathbb{C}^{n\times n}$ is
normal. \medskip

\textbf{(c)} Every unitary matrix $U\in\mathbb{C}^{n\times n}$ is normal.
\medskip

\textbf{(d)} Every diagonal matrix $D\in\mathbb{C}^{n\times n}$ is normal.
\end{proposition}

\begin{proof}
\textbf{(a)} Let $H\in\mathbb{C}^{n\times n}$ be a Hermitian matrix. Then,
$H^{\ast}=H$ (by the definition of \textquotedblleft
Hermitian\textquotedblright). Hence, $H\underbrace{H^{\ast}}_{=H}%
=\underbrace{H}_{=H^{\ast}}H=H^{\ast}H$. In other words, $H$ is normal. This
proves Proposition \ref{prop.schurtri.normal.classes} \textbf{(a)}. \medskip

\textbf{(b)} This is analogous to part \textbf{(a)}, except for a minus sign
that appears and disappears again. \medskip

\textbf{(c)} This is clear, since $UU^{\ast}=U^{\ast}U=I_{n}$ entails
$UU^{\ast}=U^{\ast}U$. \medskip

\textbf{(d)} Let $D\in\mathbb{C}^{n\times n}$ be a diagonal matrix. Write $D$
in the form
\[
D=\operatorname*{diag}\left(  \lambda_{1},\lambda_{2},\ldots,\lambda
_{n}\right)  \ \ \ \ \ \ \ \ \ \ \text{for some }\lambda_{1},\lambda
_{2},\ldots,\lambda_{n}\in\mathbb{C}.
\]
Then, $D^{\ast}=\operatorname*{diag}\left(  \overline{\lambda_{1}}%
,\overline{\lambda_{2}},\ldots,\overline{\lambda_{n}}\right)  $. Hence,
\begin{align*}
DD^{\ast}  &  =\operatorname*{diag}\left(  \lambda_{1}\overline{\lambda_{1}%
},\lambda_{2}\overline{\lambda_{2}},\ldots,\lambda_{n}\overline{\lambda_{n}%
}\right)  \ \ \ \ \ \ \ \ \ \ \text{and}\\
D^{\ast}D  &  =\operatorname*{diag}\left(  \overline{\lambda_{1}}\lambda
_{1},\overline{\lambda_{2}}\lambda_{2},\ldots,\overline{\lambda_{n}}%
\lambda_{n}\right)  .
\end{align*}
The right hand sides of these two equalities are equal (since $\lambda
_{i}\overline{\lambda_{i}}=\overline{\lambda_{i}}\lambda_{i}$ for each
$i\in\left[  n\right]  $). Thus, the left hand sides must too be equal. In
other words, $DD^{\ast}=D^{\ast}D$. This means that $D$ is normal. This proves
Proposition \ref{prop.schurtri.normal.classes} \textbf{(d)}.
\end{proof}

Unlike the unitary matrices, the normal matrices are not closed under multiplication:

\begin{exercise}
\label{exe.schurtri.normal.not-additive}\fbox{2} Find two normal matrices
$A,B\in\mathbb{C}^{2\times2}$ such that neither $A+B$ nor $AB$ is normal.
\end{exercise}

Here are three more ways to construct normal matrices out of existing normal matrices:

\begin{proposition}
\label{prop.schurtri.normal.conj}Let $A\in\mathbb{C}^{n\times n}$ be a normal
matrix. \medskip

\textbf{(a)} If $\lambda\in\mathbb{C}$ is arbitrary, then the matrix $\lambda
I_{n}+A$ is normal. \medskip

\textbf{(b)} If $U\in\mathbb{C}^{n\times n}$ is a unitary matrix, then the
matrix $UAU^{\ast}$ is normal. \medskip

\textbf{(c)} The matrix $A^{\ast}$ is normal.
\end{proposition}

\begin{proof}
We have $AA^{\ast}=A^{\ast}A$ (since $A$ is normal). \medskip

\textbf{(a)} Let $\lambda\in\mathbb{C}$ be arbitrary. Then, Proposition
\ref{prop.unitary.(AB)*} \textbf{(a)} yields%
\[
\left(  \lambda I_{n}+A\right)  ^{\ast}=\underbrace{\left(  \lambda
I_{n}\right)  ^{\ast}}_{\substack{=\overline{\lambda}I_{n}\\\text{(this is
easily seen directly, or}\\\text{obtained from Proposition
\ref{prop.unitary.(AB)*} \textbf{(b)})}}}+A^{\ast}=\overline{\lambda}%
I_{n}+A^{\ast}.
\]
Hence,
\begin{align*}
\left(  \lambda I_{n}+A\right)  \underbrace{\left(  \lambda I_{n}+A\right)
^{\ast}}_{=\overline{\lambda}I_{n}+A^{\ast}}  &  =\left(  \lambda
I_{n}+A\right)  \left(  \overline{\lambda}I_{n}+A^{\ast}\right) \\
&  =\lambda\overline{\lambda}I_{n}+\lambda A^{\ast}+\overline{\lambda
}A+AA^{\ast}.
\end{align*}
A similar computation shows that%
\[
\left(  \lambda I_{n}+A\right)  ^{\ast}\left(  \lambda I_{n}+A\right)
=\overline{\lambda}\lambda I_{n}+\lambda A^{\ast}+\overline{\lambda}A+A^{\ast
}A.
\]
The right hand sides of these two equalities are equal (since $\lambda
\overline{\lambda}=\overline{\lambda}\lambda$ and $AA^{\ast}=A^{\ast}A$).
Hence, so are the left hand sides. In other words, $\left(  \lambda
I_{n}+A\right)  \left(  \lambda I_{n}+A\right)  ^{\ast}=\left(  \lambda
I_{n}+A\right)  ^{\ast}\left(  \lambda I_{n}+A\right)  $. In other words, the
matrix $\lambda I_{n}+A$ is normal. This proves Proposition
\ref{prop.schurtri.normal.conj} \textbf{(a)}. \medskip

\textbf{(b)} Let $U\in\mathbb{C}^{n\times n}$ be a unitary matrix. Thus,
$UU^{\ast}=U^{\ast}U=I_{n}$ (by the $\mathcal{A}\Longleftrightarrow
\mathcal{C}$ part of Theorem \ref{thm.unitary.unitary.eqs}). Now, applying
Proposition \ref{prop.unitary.(AB)*} \textbf{(c)} twice, we see that $\left(
XYZ\right)  ^{\ast}=Z^{\ast}Y^{\ast}X^{\ast}$ for any three $n\times
n$-matrices $X,Y,Z$. Hence,%
\[
\left(  UAU^{\ast}\right)  ^{\ast}=\underbrace{\left(  U^{\ast}\right)
^{\ast}}_{\substack{=U\\\text{(by Proposition \ref{prop.unitary.(AB)*}
\textbf{(d)})}}}A^{\ast}U^{\ast}=UA^{\ast}U^{\ast}.
\]
Hence,%
\[
\left(  UAU^{\ast}\right)  \underbrace{\left(  UAU^{\ast}\right)  ^{\ast}%
}_{=UA^{\ast}U^{\ast}}=\left(  UAU^{\ast}\right)  \left(  UA^{\ast}U^{\ast
}\right)  =UA\underbrace{U^{\ast}U}_{=I_{n}}A^{\ast}U^{\ast}=UAA^{\ast}%
U^{\ast}.
\]
A similar computation shows that%
\[
\left(  UAU^{\ast}\right)  ^{\ast}\left(  UAU^{\ast}\right)  =UA^{\ast
}AU^{\ast}.
\]
The right hand sides of these two equalities are equal (since $AA^{\ast
}=A^{\ast}A$). Hence, so are the left hand sides. In other words, $\left(
UAU^{\ast}\right)  \left(  UAU^{\ast}\right)  ^{\ast}=\left(  UAU^{\ast
}\right)  ^{\ast}\left(  UAU^{\ast}\right)  $. In other words, the matrix
$UAU^{\ast}$ is normal. This proves Proposition
\ref{prop.schurtri.normal.conj} \textbf{(b)}. \medskip

\textbf{(c)} This is left to the reader.
\end{proof}

Here is another normality-preserving way to transform matrices:

\begin{definition}
\label{def.schurtri.normal.p(A)}Let $\mathbb{F}$ be a field. Let
$A\in\mathbb{F}^{n\times n}$ be a square matrix. Let $p\left(  x\right)  $ be
a polynomial in a single indeterminate $x$ with coefficients in $\mathbb{F}$.
Write $p\left(  x\right)  $ in the form $p\left(  x\right)  =a_{0}x^{0}%
+a_{1}x^{1}+\cdots+a_{d}x^{d}$, where $a_{0},a_{1},\ldots,a_{d}\in\mathbb{F}$.

Then, $p\left(  A\right)  $ denotes the matrix $a_{0}A^{0}+a_{1}A^{1}%
+\cdots+a_{d}A^{d}\in\mathbb{F}^{n\times n}$.
\end{definition}

For instance, if $p\left(  x\right)  =x^{3}-2x^{2}+1$, then $p\left(
A\right)  =A^{3}-2A^{2}+A^{0}=A^{3}-2A^{2}+I_{n}$.

\begin{proposition}
\label{prop.schurtri.normal.p(A)nor}Let $A\in\mathbb{C}^{n\times n}$ be a
normal matrix. Let $p\left(  x\right)  $ be a polynomial in a single
indeterminate $x$ with coefficients in $\mathbb{C}$. Then, the matrix
$p\left(  A\right)  $ is normal.
\end{proposition}

\begin{exercise}
\label{exe.schurtri.normal.p(A)nor}\fbox{3} Prove Proposition
\ref{prop.schurtri.normal.p(A)nor}.
\end{exercise}

\begin{exercise}
\label{exe.schurtri.normal.isometry}\fbox{2} Generalizing Proposition
\ref{prop.schurtri.normal.conj} \textbf{(b)}, we might claim the following:

Let $A\in\mathbb{C}^{k\times k}$ be a normal matrix. Let $U\in\mathbb{C}%
^{n\times k}$ be an isometry. Then, the matrix $UAU^{\ast}$ is normal.

Is this generalization correct?
\end{exercise}

\begin{exercise}
\label{exe.schurtri.normal.A*x}\fbox{4} Let $A\in\mathbb{C}^{n\times n}$ be a
normal matrix. Prove the following: \medskip

\textbf{(a)} We have $\left\vert \left\vert Ax\right\vert \right\vert
=\left\vert \left\vert A^{\ast}x\right\vert \right\vert $ for each
$x\in\mathbb{C}^{n}$. \medskip

\textbf{(b)} We have $\operatorname*{Ker}A=\operatorname*{Ker}\left(  A^{\ast
}\right)  $. \medskip

\textbf{(c)} Let $\lambda\in\mathbb{C}$. Then, the $\lambda$-eigenvectors of
$A$ are the $\overline{\lambda}$-eigenvectors of $A^{\ast}$.
\end{exercise}

\begin{exercise}
\label{exe.schurtri.normal.fp}\fbox{4} \textbf{(a)} Let $A\in\mathbb{C}%
^{n\times n}$ and $B\in\mathbb{C}^{m\times m}$ be two normal matrices, and
$X\in\mathbb{C}^{n\times m}$. Prove that $AX=XB$ if and only if $A^{\ast
}X=XB^{\ast}$. (This is known as the (finite) \emph{Fuglede--Putnam
theorem}.)\medskip

\textbf{(b)} Let $A\in\mathbb{C}^{n\times n}$ and $X\in\mathbb{C}^{n\times n}$
be two matrices such that $A$ is normal. Prove that $X$ commutes with $A$ if
and only if $X$ commutes with $A^{\ast}$. \medskip

[\textbf{Hint:} For part \textbf{(a)}, set $C:=AX-XB$ and $D:=A^{\ast
}X-XB^{\ast}$. Use Exercise \ref{exe.trace.TrAB} to show that
$\operatorname*{Tr}\left(  C^{\ast}C\right)  =\operatorname*{Tr}\left(
D^{\ast}D\right)  $. Conclude using Exercise \ref{exe.trace.A*A=0}
\textbf{(b)}. Finally, observe that part \textbf{(b)} is a particular case of
part \textbf{(a)}.]
\end{exercise}

\begin{exercise}
\fbox{5} Let $A\in\mathbb{C}^{n\times n}$ and $B\in\mathbb{C}^{n\times n}$ be
two normal matrices such that $AB=BA$. Prove that the matrices $A+B$ and $AB$
are normal. \medskip

[\textbf{Hint:} Use Exercise \ref{exe.schurtri.normal.fp} \textbf{(b)}.]
\end{exercise}

\begin{exercise}
\fbox{4} Let $A\in\mathbb{C}^{n\times n}$. \medskip

\textbf{(a)} Show that there is a \textbf{unique} pair $\left(  R,C\right)  $
of Hermitian matrices $R$ and $C$ such that $A=R+iC$. \medskip

\textbf{(b)} Consider this pair $\left(  R,C\right)  $. Show that $A$ is
normal if and only if $R$ and $C$ commute (that is, $RC=CR$). \medskip

[\textbf{Hint:} For part \textbf{(a)}, apply the \textquotedblleft conjugate
transpose\textquotedblright\ operation to $A=R+iC$ to obtain $A^{\ast}=R-iC$.]
\end{exercise}

We will now prove an innocent-looking property of normal matrices that will
turn out crucial in characterizing them:

\begin{lemma}
\label{lem.schurtri.normal.tri}Let $T\in\mathbb{C}^{n\times n}$ be a
triangular matrix. Then, $T$ is normal if and only if $T$ is diagonal.
\end{lemma}

\begin{proof}
The \textquotedblleft if\textquotedblright\ direction follows from Proposition
\ref{prop.schurtri.normal.classes} \textbf{(d)}. Thus, it remains to prove the
\textquotedblleft only if\textquotedblright\ direction.

So let us assume that $T$ is normal. We shall show that $T$ is diagonal.

The matrix $T$ is normal; thus, $T^{\ast}$ is normal as well (by Proposition
\ref{prop.schurtri.normal.conj} \textbf{(c)}). Since $T$ is normal, we have
$TT^{\ast}=T^{\ast}T$.

We have assumed that $T$ is triangular. WLOG assume that $T$ is
upper-triangular (because otherwise, we can replace $T$ by $T^{\ast}$). In
other words,%
\begin{equation}
T_{i,j}=0\ \ \ \ \ \ \ \ \ \ \text{for all }i,j\in\left[  n\right]  \text{
satisfying }i>j. \label{pf.lem.schurtri.normal.tri.tria}%
\end{equation}


We must prove that the matrix $T$ is diagonal. Assume the contrary. Thus, $T$
has a nonzero off-diagonal entry\footnote{An \textquotedblleft off-diagonal
entry\textquotedblright\ means an entry that does not lie on the diagonal.}.
Let $i$ be the smallest element of $\left[  n\right]  $ such that the $i$-th
row of $T$ contains a nonzero off-diagonal entry. Hence, the $i$-th row of $T$
contains a nonzero off-diagonal entry, but the $1$-st, $2$-nd, $\ldots$,
$\left(  i-1\right)  $-st rows of $T$ contain no such entries.

The definition of the product of two matrices yields that the $\left(
i,i\right)  $-th entry of $T^{\ast}T$ is%
\begin{align*}
\left(  T^{\ast}T\right)  _{i,i}  &  =\sum_{k=1}^{n}\underbrace{\left(
T^{\ast}\right)  _{i,k}}_{\substack{=\overline{T_{k,i}}\\\text{(by the
definition of }T^{\ast}\text{)}}}T_{k,i}=\sum_{k=1}^{n}\overline{T_{k,i}%
}T_{k,i}\\
&  =\sum_{k=1}^{i}\overline{T_{k,i}}T_{k,i}+\sum_{k=i+1}^{n}\overline{T_{k,i}%
}\underbrace{T_{k,i}}_{\substack{=0\\\text{(by
(\ref{pf.lem.schurtri.normal.tri.tria}), applied to }k\text{ and
}i\\\text{instead of }i\text{ and }j\text{)}}}\\
&  =\sum_{k=1}^{i}\overline{T_{k,i}}T_{k,i}+\sum_{k=i+1}^{n}\overline{T_{k,i}%
}0=\sum_{k=1}^{i}\underbrace{\overline{T_{k,i}}T_{k,i}}_{\substack{=\left\vert
T_{k,i}\right\vert ^{2}\\\text{(since }\overline{z}z=\left\vert z\right\vert
^{2}\text{ for each }z\in\mathbb{C}\text{)}}}\\
&  =\sum_{k=1}^{i}\left\vert T_{k,i}\right\vert ^{2}.
\end{align*}
A similar computation yields%
\[
\left(  TT^{\ast}\right)  _{i,i}=\sum_{k=i}^{n}\left\vert T_{i,k}\right\vert
^{2}.
\]
The left hand sides of these two equalities are equal (since $T^{\ast
}T=TT^{\ast}$). Thus, the right hand sides are equal as well. In other words,
we have%
\[
\sum_{k=1}^{i}\left\vert T_{k,i}\right\vert ^{2}=\sum_{k=i}^{n}\left\vert
T_{i,k}\right\vert ^{2}.
\]
Both sides of this equality are sums with their $k=i$ addend equal to
$\left\vert T_{i,i}\right\vert ^{2}$. Thus, if we subtract $\left\vert
T_{i,i}\right\vert ^{2}$ from this equality, then both sums lose their $k=i$
addends, and we are left with%
\begin{equation}
\sum_{k=1}^{i-1}\left\vert T_{k,i}\right\vert ^{2}=\sum_{k=i+1}^{n}\left\vert
T_{i,k}\right\vert ^{2}. \label{pf.lem.schurtri.normal.tri.sum=sum}%
\end{equation}


Now, recall that the $1$-st, $2$-nd, $\ldots$, $\left(  i-1\right)  $-st rows
of $T$ contain no nonzero off-diagonal entries. In other words, if
$k\in\left[  i-1\right]  $, then $T_{k,j}=0$ for each $j\neq k$. Hence, in
particular, if $k\in\left[  i-1\right]  $, then $T_{k,i}=0$ (since $i\neq k$).
Therefore, $\sum_{k=1}^{i-1}\left\vert \underbrace{T_{k,i}}_{=0}\right\vert
^{2}=\sum_{k=1}^{i-1}0^{2}=0$. Comparing this with
(\ref{pf.lem.schurtri.normal.tri.sum=sum}), we obtain%
\[
\sum_{k=i+1}^{n}\left\vert T_{i,k}\right\vert ^{2}=0.
\]
Therefore, all addends $\left\vert T_{i,i+1}\right\vert ^{2},\left\vert
T_{i,i+2}\right\vert ^{2},\ldots,\left\vert T_{i,n}\right\vert ^{2}$ in this
sum are $0$ (because a sum of nonnegative reals can only be $0$ if all its
addends are $0$). In other words, all the numbers $T_{i,i+1},T_{i,i+2}%
,\ldots,T_{i,n}$ are $0$. Since all the numbers $T_{i,1},T_{i,2}%
,\ldots,T_{i,i-1}$ are $0$ as well (by (\ref{pf.lem.schurtri.normal.tri.tria}%
)), we thus conclude that all the numbers $T_{i,1},T_{i,2},\ldots,T_{i,n}$ are
$0$ except for (possibly) $T_{i,i}$. In other words, the $i$-th row of $T$
contains no nonzero off-diagonal entries. This contradicts the definition of
$i$. Hence, we have obtained a contradiction, and our proof of Lemma
\ref{lem.schurtri.normal.tri} is complete.
\end{proof}

\begin{exercise}
\label{lem.schurtri.normal.tri.2}\fbox{3} \textbf{(a)} Let $T\in
\mathbb{C}^{n\times n}$ be an upper-triangular matrix. Prove that%
\[
\sum_{i=1}^{m}\left(  TT^{\ast}-T^{\ast}T\right)  _{i,i}=\sum_{i=1}%
^{m}\ \ \sum_{j=m+1}^{n}\left\vert T_{i,j}\right\vert ^{2}%
\]
for each $m\in\left\{  0,1,\ldots,n\right\}  $. \medskip

\textbf{(b)} Use this to give a direct proof (i.e., not a proof by
contradiction) of Lemma \ref{lem.schurtri.normal.tri}.
\end{exercise}

For the next exercise, we recall the notion of a \emph{nilpotent matrix}:

\begin{definition}
Let $\mathbb{F}$ be a field. A square matrix $A\in\mathbb{F}^{n\times n}$ is
said to be \emph{nilpotent} if there exists some nonnegative integer $m$ such
that $A^{m}=0$.
\end{definition}

For example, the matrix $\left(
\begin{array}
[c]{cc}%
6 & 9\\
-4 & -6
\end{array}
\right)  $ is nilpotent, since $\left(
\begin{array}
[c]{cc}%
6 & 9\\
-4 & -6
\end{array}
\right)  ^{2}=0$. Also, every strictly upper-triangular matrix and every
strictly lower-triangular matrix is nilpotent.

\begin{exercise}
\label{exe.schurtri.normal.nilp}\fbox{2} Let $A\in\mathbb{C}^{n\times n}$ be a
normal matrix that is nilpotent. Prove that $A=0$.
\end{exercise}

Let us state another useful property of polynomials applied to matrices
(Definition \ref{def.schurtri.normal.p(A)}):

\begin{exercise}
\label{exe.schurtri.spec-map}\fbox{3} Let $A\in\mathbb{C}^{n\times n}$ be any
matrix. Let $\lambda_{1},\lambda_{2},\ldots,\lambda_{n}$ be the eigenvalues of
$A$ (listed with their algebraic multiplicities). Let $p\left(  x\right)  $ be
a polynomial in a single indeterminate $x$ with coefficients in $\mathbb{C}$.

Prove that the eigenvalues of the matrix $p\left(  A\right)  $ are $p\left(
\lambda_{1}\right)  ,p\left(  \lambda_{2}\right)  ,\ldots,p\left(  \lambda
_{n}\right)  $ (listed with their algebraic multiplicities).

(This is known as the \emph{spectral mapping theorem}.)
\end{exercise}

\subsection{The spectral theorem}

\subsubsection{The spectral theorem for normal matrices}

We are now ready to state the main theorem about normal matrices, the
so-called \emph{spectral theorem}:

\begin{theorem}
[spectral theorem for normal matrices]\label{thm.schurtri.normal.spectral}Let
$A\in\mathbb{C}^{n\times n}$ be a normal matrix. Then: \medskip

\textbf{(a)} There exists a unitary matrix $U\in\operatorname*{U}%
\nolimits_{n}\left(  \mathbb{C}\right)  $ and a diagonal matrix $D\in
\mathbb{C}^{n\times n}$ such that%
\[
A=UDU^{\ast}.
\]
In other words, $A$ is unitarily similar to a diagonal matrix. \medskip

\textbf{(b)} Let $U\in\operatorname*{U}\nolimits_{n}\left(  \mathbb{C}\right)
$ be a unitary matrix, and $D\in\mathbb{C}^{n\times n}$ be a diagonal matrix
such that $A=UDU^{\ast}$. Then, the diagonal entries of $D$ are the
eigenvalues of $A$. Moreover, the columns of $U$ are eigenvectors of $A$.
Thus, there exists an orthonormal basis of $\mathbb{C}^{n}$ consisting of
eigenvectors of $A$.
\end{theorem}

\begin{proof}
\textbf{(a)} Theorem \ref{thm.schurtri.schurtri} yields that there exist a
unitary matrix $U\in\operatorname*{U}\nolimits_{n}\left(  \mathbb{C}\right)  $
and an upper-triangular matrix $T\in\mathbb{C}^{n\times n}$ such that
$A=UTU^{\ast}$. Consider these $U$ and $T$.

Since $U$ is unitary, we have $U^{\ast}U=I_{n}$ and $UU^{\ast}=I_{n}$. The
matrix $U^{\ast}$ is unitary as well (since $U^{\ast}\underbrace{\left(
U^{\ast}\right)  ^{\ast}}_{=U}=U^{\ast}U=I_{n}$ and $\underbrace{\left(
U^{\ast}\right)  ^{\ast}}_{=U}U^{\ast}=UU^{\ast}=I_{n}$). Hence, Proposition
\ref{prop.schurtri.normal.conj} \textbf{(b)} (applied to $U^{\ast}$ instead of
$U$) yields that the matrix $U^{\ast}A\left(  U^{\ast}\right)  ^{\ast}$ is
normal. Since%
\[
U^{\ast}\underbrace{A}_{=UTU^{\ast}}\ \ \underbrace{\left(  U^{\ast}\right)
^{\ast}}_{=U}=\underbrace{U^{\ast}U}_{=I_{n}}T\underbrace{U^{\ast}U}_{=I_{n}%
}=I_{n}TI_{n}=T,
\]
this rewrites as follows: The matrix $T$ is normal. Since $T$ is
upper-triangular, we thus conclude by Lemma \ref{lem.schurtri.normal.tri} that
$T$ is diagonal. Hence, if we set $D=T$, then we have constructed a unitary
matrix $U\in\operatorname*{U}\nolimits_{n}\left(  \mathbb{C}\right)  $ and a
diagonal matrix $D\in\mathbb{C}^{n\times n}$ such that $A=UDU^{\ast}$. Hence,
$A$ is unitarily similar to a diagonal matrix. This proves Theorem
\ref{thm.schurtri.normal.spectral} \textbf{(a)}. \medskip

\textbf{(b)} From $A=UDU^{\ast}$, we see that the matrix $D$ is unitarily
similar to $A$. Hence, $D$ is similar to $A$ (by Proposition
\ref{prop.schurtri.unisim.sim}). Moreover, the matrix $D$ is upper-triangular
(since it is diagonal). Thus, Proposition \ref{prop.schurtri.similar.T-diag}
(applied to $\mathbb{F}=\mathbb{C}$ and $T=D$) yields that the diagonal
entries of $T$ are the eigenvalues of $A$ (with their algebraic multiplicities).

Next, we shall show that the columns of $U$ are eigenvectors of $A$. Indeed,
from $A=UDU^{\ast}$, we obtain
\[
AU=UD\underbrace{U^{\ast}U}_{\substack{=I_{n}\\\text{(since }U\text{ is
unitary)}}}=UD.
\]


Now, let $\lambda_{1},\lambda_{2},\ldots,\lambda_{n}$ be the diagonal entries
of the diagonal matrix $D$. Let $i\in\left[  n\right]  $. Then, $De_{i}%
=\lambda_{i}e_{i}$ (where $\left(  e_{1},e_{2},\ldots,e_{n}\right)  $ denotes
the standard basis of $\mathbb{C}^{n}$), since $D$ is a diagonal matrix whose
$i$-th diagonal entry is $e_{i}$. Therefore,%
\[
\underbrace{AU}_{=UD}e_{i}=U\underbrace{De_{i}}_{=\lambda_{i}e_{i}}%
=\lambda_{i}\cdot Ue_{i}.
\]
This shows that $Ue_{i}$ is an eigenvector of $A$ (for eigenvalue $\lambda
_{i}$). Since $Ue_{i}$ is the $i$-th column of $U$, we can rewrite this as
follows: The $i$-th column of $U$ is an eigenvector of $A$.

Forget that we fixed $i$. We thus have shown that for each $i\in\left[
n\right]  $, the $i$-th column of $U$ is an eigenvector of $A$. In other
words, the columns of $U$ are eigenvectors of $A$.

It remains to prove that there exists an orthonormal basis of $\mathbb{C}^{n}$
consisting of eigenvectors of $A$. However, this is now easy: The matrix $U$
is unitary. Thus, the columns of $U$ form an orthonormal basis of
$\mathbb{C}^{n}$ (by the implication $\mathcal{A}\Longrightarrow\mathcal{E}$
in Theorem \ref{thm.unitary.unitary.eqs}). This basis consists of eigenvectors
of $A$ (since the columns of $U$ are eigenvectors of $A$). Thus, there exists
an orthonormal basis of $\mathbb{C}^{n}$ consisting of eigenvectors of $A$
(namely, this basis). This concludes the proof of Theorem
\ref{thm.schurtri.normal.spectral} \textbf{(b)}.
\end{proof}

The decomposition $A=UDU^{\ast}$ in Theorem
\ref{thm.schurtri.commute.schurtri} (or, to be more precise, the pair $\left(
U,D\right)  $) is called a \emph{spectral decomposition} of $A$. It is not
unique (e.g., we can replace $U$ by $\lambda U$ whenever $\lambda\in
\mathbb{C}$ satisfies $\left\vert \lambda\right\vert =1$; this does not change
$UDU^{\ast}$). We can actually choose the order of the diagonal entries of $D$
at will, as the following simple corollary shows:

\begin{corollary}
\label{cor.schurtri.normal.given-order}Let $A\in\mathbb{C}^{n\times n}$ be a
normal matrix. Let $\lambda_{1},\lambda_{2},\ldots,\lambda_{n}$ be the $n$
eigenvalues of $A$ (listed with algebraic multiplicities, in an arbitrary
order). Then, there exists a spectral decomposition $\left(  U,D\right)  $ of
$A$ with $D=\operatorname*{diag}\left(  \lambda_{1},\lambda_{2},\ldots
,\lambda_{n}\right)  $. Thus,%
\begin{equation}
A\overset{\operatorname*{us}}{\sim}\operatorname*{diag}\left(  \lambda
_{1},\lambda_{2},\ldots,\lambda_{n}\right)  .
\label{eq.cor.schurtri.normal.given-order.us}%
\end{equation}

\end{corollary}

\begin{proof}
Let $L=\operatorname*{diag}\left(  \lambda_{1},\lambda_{2},\ldots,\lambda
_{n}\right)  $. This is clearly a diagonal matrix.

Theorem \ref{thm.schurtri.normal.spectral} \textbf{(a)} yields that there
exist a unitary matrix $U\in\operatorname*{U}\nolimits_{n}\left(
\mathbb{C}\right)  $ and a diagonal matrix $D\in\mathbb{C}^{n\times n}$ such
that $A=UDU^{\ast}$. Consider these $U$ and $D$, and denote them by $W$ and
$F$ (since they are not yet the $U$ and $D$ that we are looking for). Thus,
$W\in\operatorname*{U}\nolimits_{n}\left(  \mathbb{C}\right)  $ is a unitary
matrix and $F\in\mathbb{C}^{n\times n}$ is a diagonal matrix such that%
\[
A=WFW^{\ast}.
\]


The definition of unitary similarity yields $A\overset{\operatorname*{us}%
}{\sim}F$ (since $W$ is unitary and $A=WFW^{\ast}$). However, the diagonal
entries of $F$ are the eigenvalues of $A$ (by Theorem
\ref{thm.schurtri.normal.spectral} \textbf{(b)}, applied to $U=W$ and $D=F$).
Since the eigenvalues of $A$ are $\lambda_{1},\lambda_{2},\ldots,\lambda_{n}$,
this shows that the diagonal entries of $F$ are $\lambda_{1},\lambda
_{2},\ldots,\lambda_{n}$ in some order. In other words, there exists a
permutation $\sigma$ of $\left[  n\right]  $ such that the diagonal entries of
$F$ are $\lambda_{\sigma\left(  1\right)  },\lambda_{\sigma\left(  2\right)
},\ldots,\lambda_{\sigma\left(  n\right)  }$. Consider this $\sigma$.

The matrix $F$ is a diagonal matrix, and its diagonal entries are
$\lambda_{\sigma\left(  1\right)  },\lambda_{\sigma\left(  2\right)  }%
,\ldots,\lambda_{\sigma\left(  n\right)  }$. In other words,
$F=\operatorname*{diag}\left(  \lambda_{\sigma\left(  1\right)  }%
,\lambda_{\sigma\left(  2\right)  },\ldots,\lambda_{\sigma\left(  n\right)
}\right)  $.

However, Proposition \ref{prop.schurtri.unisimi.diag} yields
$\operatorname*{diag}\left(  \lambda_{1},\lambda_{2},\ldots,\lambda
_{n}\right)  \overset{\operatorname*{us}}{\sim}\operatorname*{diag}\left(
\lambda_{\sigma\left(  1\right)  },\lambda_{\sigma\left(  2\right)  }%
,\ldots,\lambda_{\sigma\left(  n\right)  }\right)  $. This rewrites as
$L\overset{\operatorname*{us}}{\sim}F$ (since $L=\operatorname*{diag}\left(
\lambda_{1},\lambda_{2},\ldots,\lambda_{n}\right)  $ and
$F=\operatorname*{diag}\left(  \lambda_{\sigma\left(  1\right)  }%
,\lambda_{\sigma\left(  2\right)  },\ldots,\lambda_{\sigma\left(  n\right)
}\right)  $). In other words, there exists a unitary matrix $Q\in
\operatorname*{U}\nolimits_{n}\left(  \mathbb{C}\right)  $ such that%
\[
F=QLQ^{\ast}.
\]
Consider this $Q$. Now, the matrix $WQ$ is unitary (by Exercise
\ref{exe.unitary.group} \textbf{(b)}, since $W$ and $Q$ are unitary), and we
have%
\[
A=W\underbrace{F}_{=QLQ^{\ast}}W^{\ast}=WQL\underbrace{Q^{\ast}W^{\ast}%
}_{=\left(  WQ\right)  ^{\ast}}=WQL\left(  WQ\right)  ^{\ast}=\left(
WQ\right)  \cdot L\cdot\left(  WQ\right)  ^{\ast}.
\]
This shows that $\left(  WQ,L\right)  $ is a spectral decomposition of $A$
(since $WQ$ is unitary and $L$ is diagonal). This spectral decomposition
satisfies $L=\operatorname*{diag}\left(  \lambda_{1},\lambda_{2}%
,\ldots,\lambda_{n}\right)  $. Thus, there exists a spectral decomposition
$\left(  U,D\right)  $ of $A$ with $D=\operatorname*{diag}\left(  \lambda
_{1},\lambda_{2},\ldots,\lambda_{n}\right)  $ (namely, $\left(  WQ,L\right)
$). This furthermore shows that $A\overset{\operatorname*{us}}{\sim
}\operatorname*{diag}\left(  \lambda_{1},\lambda_{2},\ldots,\lambda
_{n}\right)  $. Corollary \ref{cor.schurtri.normal.given-order} is thus proven.
\end{proof}

Note that Theorem \ref{thm.schurtri.normal.spectral} \textbf{(b)} has a
converse, which helps finding spectral decompositions in practice if one
doesn't want to go through the trouble of Schur triangularization:

\begin{proposition}
\label{prop.schurtri.normal.converse}Let $A\in\mathbb{C}^{n\times n}$. Let
$U\in\operatorname*{U}\nolimits_{n}\left(  \mathbb{C}\right)  $ be a unitary
matrix and $D\in\mathbb{C}^{n\times n}$ a diagonal matrix. Assume that for
each $i\in\left[  n\right]  $, we have $AU_{\bullet,i}=D_{i,i}U_{\bullet,i}$
(that is, the $i$-th column of $U$ is an eigenvector of $A$ for the eigenvalue
$D_{i,i}$). Then, $A=UDU^{\ast}$, so that $\left(  U,D\right)  $ is a spectral
decomposition of $A$.
\end{proposition}

\begin{exercise}
\fbox{2} Prove Proposition \ref{prop.schurtri.normal.converse}.
\end{exercise}

\begin{exercise}
\label{exe.schurtri.normal.examples}\fbox{5} \textbf{(a)} Find a spectral
decomposition of the normal matrix $\left(
\begin{array}
[c]{cc}%
1 & 1+i\\
1+i & 1
\end{array}
\right)  $. \medskip

\textbf{(b)} Find a spectral decomposition of the Hermitian matrix $\left(
\begin{array}
[c]{cc}%
0 & -i\\
i & 0
\end{array}
\right)  $. \medskip

\textbf{(c)} Find a spectral decomposition of the skew-Hermitian matrix
$\left(
\begin{array}
[c]{cc}%
0 & i\\
i & 0
\end{array}
\right)  $. \medskip

\textbf{(d)} Find a spectral decomposition of the unitary matrix $\dfrac
{1}{\sqrt{2}}\left(
\begin{array}
[c]{cc}%
1 & 1\\
1 & -1
\end{array}
\right)  $.
\end{exercise}

\begin{exercise}
\fbox{2} Describe all spectral decompositions of the $n\times n$ identity
matrix $I_{n}$.
\end{exercise}

Only normal matrices can have a spectral decomposition. Indeed, if some
$n\times n$-matrix $A\in\mathbb{C}^{n\times n}$ can be written as
$A=UDU^{\ast}$ for some unitary $U$ and some diagonal $D$, then $D$ is normal
(by Proposition \ref{prop.schurtri.normal.classes} \textbf{(d)}), and
therefore $A$ is normal (by Proposition \ref{prop.schurtri.normal.conj}
\textbf{(b)}, applied to $D$ instead of $A$). Thus, we obtain the following
characterization of normal matrices:

\begin{corollary}
\label{cor.schurtri.normal.normal-iff}An $n\times n$-matrix $A\in
\mathbb{C}^{n\times n}$ is normal if and only if it is unitarily similar to a
diagonal matrix.
\end{corollary}

\begin{proof}
$\Longrightarrow:$ Assume that $A$ is normal. Then, Theorem
\ref{thm.schurtri.normal.spectral} \textbf{(a)} shows that $A$ is unitarily
similar to a diagonal matrix. This proves the \textquotedblleft%
$\Longrightarrow$\textquotedblright\ direction of Corollary
\ref{cor.schurtri.normal.normal-iff}. \medskip

$\Longleftarrow:$ Assume that $A$ is unitarily similar to a diagonal matrix.
In other words, $A=UDU^{\ast}$ for some unitary matrix $U\in\operatorname*{U}%
\nolimits_{n}\left(  \mathbb{C}\right)  $ and some diagonal matrix
$D\in\mathbb{C}^{n\times n}$. Consider these $U$ and $D$. The matrix $D$ is
normal (by Proposition \ref{prop.schurtri.normal.classes} \textbf{(d)}).
Hence, the matrix $UDU^{\ast}$ is normal (by Proposition
\ref{prop.schurtri.normal.conj} \textbf{(b)}, applied to $D$ instead of $A$).
In other words, the matrix $A$ is normal (since $A=UDU^{\ast}$). This proves
the \textquotedblleft$\Longleftarrow$\textquotedblright\ direction of
Corollary \ref{cor.schurtri.normal.normal-iff}.
\end{proof}

\begin{exercise}
\fbox{3} Let $A\in\mathbb{C}^{n\times n}$ and $B\in\mathbb{C}^{n\times n}$ be
two normal matrices such that $A\sim B$. Prove that
$A\overset{\operatorname*{us}}{\sim}B$.
\end{exercise}

\subsubsection{The spectral theorem for Hermitian matrices}

The spectral decompositions of a Hermitian matrix have a special property:

\begin{proposition}
\label{prop.schurtri.normal.hermitian-spec}Let $A\in\mathbb{C}^{n\times n}$ be
a Hermitian matrix, and let $\left(  U,D\right)  $ be a spectral decomposition
of $A$. Then, the diagonal entries of $D$ are real.
\end{proposition}

\begin{proof}
The definition of a spectral decomposition yields that $U$ is unitary and $D$
is diagonal and $A=UDU^{\ast}$. However, since $A$ is Hermitian, we have
$A^{\ast}=A$. In view of $A=UDU^{\ast}$, this rewrites as $\left(  UDU^{\ast
}\right)  ^{\ast}=UDU^{\ast}$. Hence,
\[
UDU^{\ast}=\left(  UDU^{\ast}\right)  ^{\ast}=\underbrace{\left(  U^{\ast
}\right)  ^{\ast}}_{=U}D^{\ast}U^{\ast}=UD^{\ast}U^{\ast}.
\]
Since the matrix $U$ is unitary, we can cancel both $U$ and $U^{\ast}$ from
this equality\footnote{Indeed, the matrix $U$ is unitary; therefore, $U$ is
invertible, and its inverse is $U^{-1}=U^{\ast}$. Hence, $U^{\ast}$ is also
invertible (being the inverse of $U$). Thus, we can multiply both sides of the
equality $UDU^{\ast}=UD^{\ast}U^{\ast}$ from the left by $U^{-1}$ and from the
right by $\left(  U^{\ast}\right)  ^{-1}$; as a result, we obtain $D=D^{\ast}%
$.}, and thus obtain $D=D^{\ast}$. However, if $\lambda$ is a diagonal entry
of $D$, then the corresponding diagonal entry of $D^{\ast}$ must be
$\overline{\lambda}$, and therefore we obtain $\lambda=\overline{\lambda}$
(since $D=D^{\ast}$ shows that these two entries are equal). Thus, each
diagonal entry $\lambda$ of $D$ satisfies $\lambda=\overline{\lambda}$ and
therefore $\lambda\in\mathbb{R}$ (because a complex number $z$ satisfying
$z=\overline{z}$ must automatically satisfy $z\in\mathbb{R}$). In other words,
the diagonal entries of $D$ are real. This proves Proposition
\ref{prop.schurtri.normal.hermitian-spec}.
\end{proof}

This allows us to characterize Hermitian matrices in a similar way as normal
matrices were characterized by Corollary \ref{cor.schurtri.normal.normal-iff}:

\begin{corollary}
\label{cor.schurtri.normal.hermitian-iff}An $n\times n$-matrix $A\in
\mathbb{C}^{n\times n}$ is Hermitian if and only if it is unitarily similar to
a diagonal matrix with real entries.
\end{corollary}

\begin{proof}
$\Longrightarrow:$ Assume that $A$ is Hermitian. Then, $A$ is normal (by
Proposition \ref{prop.schurtri.normal.classes} \textbf{(a)}). Hence, Theorem
\ref{thm.schurtri.normal.spectral} \textbf{(a)} shows that $A$ is unitarily
similar to a diagonal matrix. In other words, $A=UDU^{\ast}$ for some unitary
matrix $U\in\operatorname*{U}\nolimits_{n}\left(  \mathbb{C}\right)  $ and
some diagonal matrix $D\in\mathbb{C}^{n\times n}$. Consider these $U$ and $D$.
Clearly, $A$ is unitarily similar to $D$. Moreover, $\left(  U,D\right)  $ is
a spectral decomposition of $A$ (by the definition of a spectral
decomposition). Hence, Proposition \ref{prop.schurtri.normal.hermitian-spec}
yields that the diagonal entries of $D$ are real. Thus, $D$ is a diagonal
matrix with real entries. Hence, $A$ is unitarily similar to a diagonal matrix
with real entries (since $A$ is unitarily similar to $D$). This proves the
\textquotedblleft$\Longrightarrow$\textquotedblright\ direction of Corollary
\ref{cor.schurtri.normal.hermitian-iff}. \medskip

$\Longleftarrow:$ Assume that $A$ is unitarily similar to a diagonal matrix
with real entries. In other words, $A=UDU^{\ast}$ for some unitary matrix
$U\in\operatorname*{U}\nolimits_{n}\left(  \mathbb{C}\right)  $ and some
diagonal matrix $D\in\mathbb{C}^{n\times n}$ that has real entries. Consider
these $U$ and $D$. The matrix $D$ is a diagonal matrix with real entries;
thus, it is easy to see that $D^{\ast}=D$ (since the diagonal entries of $D$
are real and thus remain unchanged and unmoved in $D^{\ast}$, whereas all
other entries of $D$ are $0$). Now, from $A=UDU^{\ast}$, we obtain
\[
A^{\ast}=\left(  UDU^{\ast}\right)  ^{\ast}=\underbrace{\left(  U^{\ast
}\right)  ^{\ast}}_{=U}\ \ \underbrace{D^{\ast}}_{=D}U^{\ast}=UDU^{\ast}=A.
\]
In other words, the matrix $A$ is Hermitian. This proves the \textquotedblleft%
$\Longleftarrow$\textquotedblright\ direction of Corollary
\ref{cor.schurtri.normal.hermitian-iff}.
\end{proof}

\subsubsection{The spectral theorem for skew-Hermitian matrices}

Similarly, we can handle skew-Hermitian matrices:

\begin{proposition}
\label{prop.schurtri.normal.skewherm-spec}Let $A\in\mathbb{C}^{n\times n}$ be
a skew-Hermitian matrix, and let $\left(  U,D\right)  $ be a spectral
decomposition of $A$. Then, the diagonal entries of $D$ are purely imaginary.
\end{proposition}

\begin{corollary}
\label{cor.schurtri.normal.skewherm-iff}An $n\times n$-matrix $A\in
\mathbb{C}^{n\times n}$ is skew-Hermitian if and only if it is unitarily
similar to a diagonal matrix with purely imaginary entries.
\end{corollary}

\begin{exercise}
\label{exe.schurtri.normal.skewherm}\fbox{3} Prove Proposition
\ref{prop.schurtri.normal.skewherm-spec} and Corollary
\ref{cor.schurtri.normal.skewherm-iff}.
\end{exercise}

\subsubsection{The spectral theorem for unitary matrices}

Likewise, we can handle unitary matrices:

\begin{proposition}
\label{prop.schurtri.normal.unitary-spec}Let $A\in\mathbb{C}^{n\times n}$ be a
unitary matrix, and let $\left(  U,D\right)  $ be a spectral decomposition of
$A$. Then, each of the diagonal entries of $D$ has absolute value $1$.
\end{proposition}

\begin{corollary}
\label{cor.schurtri.normal.unitary-iff}An $n\times n$-matrix $A\in
\mathbb{C}^{n\times n}$ is unitary if and only if it is unitarily similar to a
diagonal matrix whose all diagonal entries have absolute value $1$.
\end{corollary}

\begin{exercise}
\label{exe.schurtri.normal.unitary}\fbox{2} Prove Proposition
\ref{prop.schurtri.normal.unitary-spec} and Corollary
\ref{cor.schurtri.normal.unitary-iff}.
\end{exercise}

\begin{exercise}
\label{exe.schurtri.normal.F}\fbox{2} Prove the following generalization of
Theorem \ref{thm.schurtri.normal.spectral}:

Let $\mathcal{F}$ be a subset of $\mathbb{C}^{n\times n}$ such that any matrix
in $\mathcal{F}$ is normal, and such that any two matrices in $\mathcal{F}$
commute (i.e., any $A\in\mathcal{F}$ and $B\in\mathcal{F}$ satisfy $AB=BA$).

Then, there exists a unitary matrix $U\in\operatorname*{U}\nolimits_{n}\left(
\mathbb{C}\right)  $ such that for each $A\in\mathcal{F}$, the matrix
$UAU^{\ast}$ is diagonal.
\end{exercise}

%

\[%
\begin{tabular}
[c]{||l||}\hline\hline
\textbf{Lecture 6 starts here.}\\\hline\hline
\end{tabular}
\]


\subsection{The Cayley--Hamilton theorem}

We will now state the famous Cayley--Hamilton theorem, and to prove it at
least for matrices with complex entries. This will serve as a reminder of an
important theorem (which will soon be used), and also as an illustration of
how Schur triangularization can be applied.

In Definition \ref{def.schurtri.normal.p(A)}, we have learnt how to substitute
a square matrix into a polynomial. Something peculiar happens when a matrix is
substituted into its own characteristic polynomial:

\begin{theorem}
[Cayley--Hamilton theorem]\label{thm.schurtri.ch.ch}Let $\mathbb{F}$ be a
field. Let $A\in\mathbb{F}^{n\times n}$ be an $n\times n$-matrix. Then,%
\[
p_{A}\left(  A\right)  =0.
\]
(The \textquotedblleft$0$\textquotedblright\ on the right hand side here means
the zero matrix $0_{2\times2}$.)
\end{theorem}

\begin{example}
Let $n=2$ and $A=\left(
\begin{array}
[c]{cc}%
a & b\\
c & d
\end{array}
\right)  $. Then, as we know from Example \ref{exa.schurtri.ch.pA.2x2}, we
have%
\[
p_{A}=t^{2}-\left(  a+d\right)  t+\left(  ad-bc\right)  .
\]
Thus,%
\begin{align*}
p_{A}\left(  A\right)   &  =A^{2}-\left(  a+d\right)  A+\left(  ad-bc\right)
I_{2}\\
&  =\left(
\begin{array}
[c]{cc}%
a & b\\
c & d
\end{array}
\right)  ^{2}-\left(  a+d\right)  \left(
\begin{array}
[c]{cc}%
a & b\\
c & d
\end{array}
\right)  +\left(  ad-bc\right)  \left(
\begin{array}
[c]{cc}%
1 & 0\\
0 & 1
\end{array}
\right) \\
&  =\left(
\begin{array}
[c]{cc}%
0 & 0\\
0 & 0
\end{array}
\right)  =0.
\end{align*}
Thus, we have verified Theorem \ref{thm.schurtri.ch.ch} for $n=2$.
\end{example}

\begin{remark}
\label{rmk.schurtri.ch.not-that-easy}It is tempting to \textquotedblleft
prove\textquotedblright\ Theorem \ref{thm.schurtri.ch.ch} by arguing that
$p_{A}\left(  A\right)  =\det\left(  AI_{n}-A\right)  $ holds
\textquotedblleft by substituting $A$ for $t$ into $p_{A}=\det\left(
tI_{n}-A\right)  $\textquotedblright. Unfortunately, such an argument is
unjustified. Indeed, $tI_{n}-A$ is a matrix whose entries are polynomials in
$t$. If you substitute $A$ for $t$ into it, it will become a matrix whose
entries are matrices. This poses two problems: First, it is unclear how to
take the determinant of such a matrix; second, this matrix is not $AI_{n}-A$.
For example, for $n=2$, substituting $A$ for $t$ in $tI_{n}-A$ gives%
\[
\left(
\begin{array}
[c]{cc}%
\left(
\begin{array}
[c]{cc}%
a & b\\
c & d
\end{array}
\right)  -a & -b\\
-c & \left(
\begin{array}
[c]{cc}%
a & b\\
c & d
\end{array}
\right)  -d
\end{array}
\right)  ,
\]
which can be made sense of (if we treat the $a,b,c,d$ as multiples of $I_{2}%
$), but which is certainly not the same as $AI_{n}-A$ (which is the zero
matrix). There \textbf{is} a correct proof of the Cayley--Hamilton theorem
along the lines of \textquotedblleft substituting $A$ for $t$%
\textquotedblright, but it requires a lot of additional work (see
\url{https://math.stackexchange.com/questions/1141648/} for some discussion of this).
\end{remark}

Various proofs of Theorem \ref{thm.schurtri.ch.ch} are found across the
literature; see \cite[after Theorem 2.6]{trach} for a list of references
(Theorem \ref{thm.schurtri.ch.ch} is \cite[Theorem 2.5]{trach}). I can
particularly recommend the algebraic proofs given in \cite[Chapter Five,
Section IV, Lemma 1.9]{Heffer20}, \cite[\S 4, Theorem 1]{Mate16} and
\cite{Shurma15}, and the combinatorial proof shown in \cite{Straub83} and
\cite[\S 3]{Zeilbe}. Here, however, I will show a proof of Theorem
\ref{thm.schurtri.ch.ch} in the particular case when $\mathbb{F}=\mathbb{C}$.

This proof will rely on two lemmas. The first collects some useful properties
of the application of polynomials to matrices:

\begin{lemma}
\label{lem.p(A).multiplicative}Let $n\in\mathbb{N}$. Let $\mathbb{F}$ be a
field. Let $A\in\mathbb{F}^{n\times n}$ be an $n\times n$-matrix. The word
\textquotedblleft polynomial\textquotedblright\ shall mean \textquotedblleft
polynomial in an indeterminate $t$ with coefficients in $\mathbb{F}%
$\textquotedblright. Then: \medskip

\textbf{(a)} If $f$ and $g$ are two polynomials, then $\left(  fg\right)
\left(  A\right)  =f\left(  A\right)  \cdot g\left(  A\right)  $. \medskip

\textbf{(b)} If $f_{1},f_{2},\ldots,f_{k}$ are several polynomials, then
$\left(  f_{1}f_{2}\cdots f_{k}\right)  \left(  A\right)  =f_{1}\left(
A\right)  \cdot f_{2}\left(  A\right)  \cdot\cdots\cdot f_{k}\left(  A\right)
$. \medskip

\textbf{(c)} If $f$ is a polynomial, and $W\in\mathbb{F}^{n\times n}$ is an
invertible matrix, then $f\left(  WAW^{-1}\right)  =W\cdot f\left(  A\right)
\cdot W^{-1}$.
\end{lemma}

\begin{proof}
[Proof of Lemma \ref{lem.p(A).multiplicative}.]\textbf{(a)} Let $f$ and $g$ be
two polynomials. Write $f$ in the form $f=\sum_{i=0}^{p}f_{i}t^{i}$ for some
coefficients $f_{0},f_{1},\ldots,f_{p}\in\mathbb{F}$. Write $g$ in the form
$g=\sum_{j=0}^{q}g_{j}t^{j}$ for some coefficients $g_{0},g_{1},\ldots
,g_{q}\in\mathbb{F}$. Definition \ref{def.schurtri.normal.p(A)} yields%
\[
f\left(  A\right)  =\sum_{i=0}^{p}f_{i}A^{i}\ \ \ \ \ \ \ \ \ \ \left(
\text{since }f=\sum_{i=0}^{p}f_{i}t^{i}\right)
\]
and%
\[
g\left(  A\right)  =\sum_{j=0}^{q}g_{j}A^{j}\ \ \ \ \ \ \ \ \ \ \left(
\text{since }g=\sum_{j=0}^{q}g_{j}t^{j}\right)  .
\]
Multiplying these two equalities, we obtain%
\begin{align}
f\left(  A\right)  \cdot g\left(  A\right)   &  =\left(  \sum_{i=0}^{p}%
f_{i}A^{i}\right)  \cdot\left(  \sum_{j=0}^{q}g_{j}A^{j}\right)  =\sum
_{i=0}^{p}\ \ \sum_{j=0}^{q}f_{i}g_{j}\underbrace{A^{i}A^{j}}_{=A^{i+j}%
}\nonumber\\
&  =\sum_{i=0}^{p}\ \ \sum_{j=0}^{q}f_{i}g_{j}A^{i+j}.
\label{pf.lem.p(A).multiplicative.a.3}%
\end{align}


Multiplying the equalities $f=\sum_{i=0}^{p}f_{i}t^{i}$ and $g=\sum_{j=0}%
^{q}g_{j}t^{j}$, we obtain%
\[
fg=\left(  \sum_{i=0}^{p}f_{i}t^{i}\right)  \left(  \sum_{j=0}^{q}g_{j}%
t^{j}\right)  =\sum_{k=0}^{p+q}\left(  \sum_{\substack{i\in\left\{
0,1,\ldots,k\right\}  ;\\i\leq p;\ k-i\leq q}}f_{i}g_{k-i}\right)  t^{k}%
\]
(by the definition of the product of two polynomials). Hence, Definition
\ref{def.schurtri.normal.p(A)} yields%
\begin{align*}
\left(  fg\right)  \left(  A\right)   &  =\sum_{k=0}^{p+q}\left(
\sum_{\substack{i\in\left\{  0,1,\ldots,k\right\}  ;\\i\leq p;\ k-i\leq
q}}f_{i}g_{k-i}\right)  A^{k}=\underbrace{\sum_{k=0}^{p+q}\ \ \sum
_{\substack{i\in\left\{  0,1,\ldots,k\right\}  ;\\i\leq p;\ k-i\leq q}%
}}_{\substack{=\sum_{i=0}^{p}\ \ \sum_{k=i}^{i+q}\\\text{(because both of
these double sums}\\\text{are summing over all pairs }\left(  k,i\right)
\text{ of}\\\text{nonnegative integers satisfying }i\leq k\\\text{and }i\leq
p\text{ and }k\leq i+q\text{)}}}f_{i}g_{k-i}A^{k}\\
&  =\sum_{i=0}^{p}\ \ \sum_{k=i}^{i+q}f_{i}g_{k-i}A^{k}=\sum_{i=0}^{p}%
\ \ \sum_{j=0}^{q}f_{i}\underbrace{g_{\left(  i+j\right)  -i}}_{=g_{j}}%
A^{i+j}\\
&  \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \left(  \text{here, we have
substituted }i+j\text{ for }k\text{ in the inner sum}\right) \\
&  =\sum_{i=0}^{p}\ \ \sum_{j=0}^{q}f_{i}g_{j}A^{i+j}=f\left(  A\right)  \cdot
g\left(  A\right)  \ \ \ \ \ \ \ \ \ \ \left(  \text{by
(\ref{pf.lem.p(A).multiplicative.a.3})}\right)  .
\end{align*}
This proves Lemma \ref{lem.p(A).multiplicative} \textbf{(a)}. \medskip

\textbf{(b)} Lemma \ref{lem.p(A).multiplicative} \textbf{(b)} follows by
induction on $k$. (The base case relies on $1\left(  A\right)  =I_{n}$, where
$1$ denotes the constant polynomial $1$. The induction step uses Lemma
\ref{lem.p(A).multiplicative} \textbf{(a)}.) \medskip

\textbf{(c)} Let $f$ be a polynomial, and let $W\in\mathbb{F}^{n\times n}$ be
an invertible matrix. Let $B:=WAW^{-1}$. Write the polynomial $f$ in the form
$f=\sum_{k=0}^{p}f_{k}t^{k}$ for some coefficients $f_{0},f_{1},\ldots
,f_{p}\in\mathbb{F}$. Thus, Definition \ref{def.schurtri.normal.p(A)} yields%
\begin{align}
f\left(  A\right)   &  =\sum_{k=0}^{p}f_{k}A^{k}\ \ \ \ \ \ \ \ \ \ \text{and}%
\label{pf.lem.p(A).multiplicative.c.1}\\
f\left(  B\right)   &  =\sum_{k=0}^{p}f_{k}B^{k}.
\label{pf.lem.p(A).multiplicative.c.2}%
\end{align}


However, for each $k\in\mathbb{N}$, we have
\begin{equation}
B^{k}=WA^{k}W^{-1} \label{pf.lem.p(A).multiplicative.c.3}%
\end{equation}
(indeed, this is precisely the equality
(\ref{pf.prop.schurtri.similar.same.f.1}), which we have proved long ago).
Therefore, (\ref{pf.lem.p(A).multiplicative.c.2}) becomes%
\begin{align*}
f\left(  B\right)   &  =\sum_{k=0}^{p}f_{k}\underbrace{B^{k}}%
_{\substack{=WA^{k}W^{-1}\\\text{(by (\ref{pf.lem.p(A).multiplicative.c.3}))}%
}}=\sum_{k=0}^{p}f_{k}WA^{k}W^{-1}\\
&  =W\cdot\sum_{k=0}^{p}f_{k}A^{k}W^{-1}=W\cdot\underbrace{\left(  \sum
_{k=0}^{p}f_{k}A^{k}\right)  }_{\substack{=f\left(  A\right)  \\\text{(by
(\ref{pf.lem.p(A).multiplicative.c.1}))}}}\cdot W^{-1}=W\cdot f\left(
A\right)  \cdot W^{-1}.
\end{align*}
This proves Lemma \ref{lem.p(A).multiplicative} \textbf{(c)}.
\end{proof}

The second lemma is an easy but neat property of triangular matrices:

\begin{lemma}
\label{lem.uptri.prod=0}Let $n\in\mathbb{N}$. Let $\mathbb{F}$ be a field. Let
$T_{1},T_{2},\ldots,T_{n}$ be $n$ upper-triangular $n\times n$-matrices.
Assume that for each $i\in\left[  n\right]  $, the $i$-th diagonal entry of
the matrix $T_{i}$ is $0$ (that is, we have $\left(  T_{i}\right)  _{i,i}=0$).
Then,%
\[
T_{1}T_{2}\cdots T_{n}=0.
\]
(The $0$ on the right hand side here is the zero matrix.)
\end{lemma}

\begin{example}
For $n=3$, Theorem \ref{lem.uptri.prod=0} is saying the following: If
$T_{1},T_{2},T_{3}$ are three $3\times3$-matrices of the form%
\[
T_{1}=\left(
\begin{array}
[c]{ccc}%
0 & \ast & \ast\\
0 & \ast & \ast\\
0 & 0 & \ast
\end{array}
\right)  ,\ \ \ \ \ \ \ \ \ \ T_{2}=\left(
\begin{array}
[c]{ccc}%
\ast & \ast & \ast\\
0 & 0 & \ast\\
0 & 0 & \ast
\end{array}
\right)  ,\ \ \ \ \ \ \ \ \ \ T_{3}=\left(
\begin{array}
[c]{ccc}%
\ast & \ast & \ast\\
0 & \ast & \ast\\
0 & 0 & 0
\end{array}
\right)
\]
(where each asterisk \textquotedblleft$\ast$\textquotedblright\ stands for an
arbitrary entry -- not necessarily equal to the other asterisk entries), then
$T_{1}T_{2}T_{3}=0$.
\end{example}

\begin{proof}
[Proof of Lemma \ref{lem.uptri.prod=0}.]We claim that%
\begin{equation}
\text{the first }k\text{ columns of the matrix }T_{1}T_{2}\cdots T_{k}\text{
are }0 \label{pf.lem.uptri.prod=0.goal}%
\end{equation}
for each $k\in\left\{  0,1,\ldots,n\right\}  $.

[\textit{Proof of (\ref{pf.lem.uptri.prod=0.goal}):} We shall prove
(\ref{pf.lem.uptri.prod=0.goal}) by induction on $k$:

\textit{Base case:} The first $0$ columns of any matrix are $0$ (indeed, this
is vacuously true). Thus, (\ref{pf.lem.uptri.prod=0.goal}) holds for $k=0$.

\textit{Induction step:} Let $p\in\left[  n\right]  $. Assume that
(\ref{pf.lem.uptri.prod=0.goal}) holds for $k=p-1$. We must prove that
(\ref{pf.lem.uptri.prod=0.goal}) holds for $k=p$.

Let $A=T_{1}T_{2}\cdots T_{p-1}$ and $B=T_{p}$. Thus,%
\begin{equation}
AB=\left(  T_{1}T_{2}\cdots T_{p-1}\right)  T_{p}=T_{1}T_{2}\cdots T_{p}.
\label{pf.lem.uptri.prod=0.1}%
\end{equation}


We have assumed that (\ref{pf.lem.uptri.prod=0.goal}) holds for $k=p-1$. In
other words, the first $p-1$ columns of the matrix $T_{1}T_{2}\cdots T_{p-1}$
are $0$. In other words, the first $p-1$ columns of the matrix $A$ are $0$
(since $A=T_{1}T_{2}\cdots T_{p-1}$). In other words,%
\begin{equation}
A_{i,j}=0\ \ \ \ \ \ \ \ \ \ \text{for each }i\in\left[  n\right]  \text{ and
}j\in\left[  p-1\right]  . \label{pf.lem.uptri.prod=0.2}%
\end{equation}


On the other hand, the assumption of Lemma \ref{lem.uptri.prod=0} yields that
the $p$-th diagonal entry of the matrix $T_{p}$ is $0$. In other words,
$\left(  T_{p}\right)  _{p,p}=0$. In other words, $B_{p,p}=0$ (since $B=T_{p}%
$). Moreover, the matrix $T_{p}$ is upper-triangular (by the assumption of
Lemma \ref{lem.uptri.prod=0}). In other words, the matrix $B$ is
upper-triangular (since $B=T_{p}$). In other words,%
\begin{equation}
B_{i,j}=0\ \ \ \ \ \ \ \ \ \ \text{for each }i,j\in\left[  n\right]  \text{
satisfying }i>j. \label{pf.lem.uptri.prod=0.3}%
\end{equation}


Now, let $i\in\left[  n\right]  $ and $j\in\left[  p\right]  $ be arbitrary.
We shall show that $\left(  AB\right)  _{i,j}=0$.

Indeed, the definition of the product of two matrices yields%
\begin{align}
\left(  AB\right)  _{i,j}  &  =\sum_{k=1}^{n}A_{i,k}B_{k,j}=\sum_{k=1}%
^{p-1}\underbrace{A_{i,k}}_{\substack{=0\\\text{(by
(\ref{pf.lem.uptri.prod=0.2}), applied to }k\text{ instead of }j\\\text{(since
}k\in\left[  p-1\right]  \text{))}}}B_{k,j}+\sum_{k=p}^{n}A_{i,k}%
B_{k,j}\nonumber\\
&  =\underbrace{\sum_{k=1}^{p-1}0B_{k,j}}_{=0}+\sum_{k=p}^{n}A_{i,k}%
B_{k,j}=\sum_{k=p}^{n}A_{i,k}B_{k,j}. \label{pf.lem.uptri.prod=0.4}%
\end{align}


If $p>j$, then this becomes%
\[
\left(  AB\right)  _{i,j}=\sum_{k=p}^{n}A_{i,k}\underbrace{B_{k,j}%
}_{\substack{=0\\\text{(by (\ref{pf.lem.uptri.prod=0.3}), applied to }k\text{
instead of }i\\\text{(since }k\geq p>j\text{))}}}=\sum_{k=p}^{n}A_{i,k}0=0,
\]
and therefore $\left(  AB\right)  _{i,j}=0$ has been proved in this case.
Hence, for the rest of the proof of $\left(  AB\right)  _{i,j}=0$, we WLOG
assume that we don't have $p>j$. Thus, $j\geq p$, so that $j=p$ (since
$j\in\left[  p\right]  $). In other words, $p=j$. Now,
(\ref{pf.lem.uptri.prod=0.4}) becomes%
\begin{align*}
\left(  AB\right)  _{i,j}  &  =\sum_{k=p}^{n}A_{i,k}B_{k,j}=A_{i,p}%
\underbrace{B_{p,j}}_{\substack{=B_{p,p}\\\text{(since }j=p\text{)}}%
}+\sum_{k=p+1}^{n}A_{i,k}\underbrace{B_{k,j}}_{\substack{=0\\\text{(by
(\ref{pf.lem.uptri.prod=0.3}), applied to }k\text{ instead of }i\\\text{(since
}k\geq p+1>p=j\text{))}}}\\
&  \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \left(  \text{here, we have split
off the addend for }k=p\text{ from the sum}\right) \\
&  =A_{i,p}\underbrace{B_{p,p}}_{=0}+\underbrace{\sum_{k=p+1}^{n}A_{i,k}%
0}_{=0}=0.
\end{align*}
Thus, we have proved that $\left(  AB\right)  _{i,j}=0$.

Forget that we fixed $i$ and $j$. We thus have shown that $\left(  AB\right)
_{i,j}=0$ for all $i\in\left[  n\right]  $ and $j\in\left[  p\right]  $. In
other words, the first $p$ columns of the matrix $AB$ are $0$. In view of
(\ref{pf.lem.uptri.prod=0.1}), we can rewrite this as follows: The first $p$
columns of the matrix $T_{1}T_{2}\cdots T_{p}$ are $0$. In other words,
(\ref{pf.lem.uptri.prod=0.goal}) holds for $k=p$. This completes the induction
step. Thus, (\ref{pf.lem.uptri.prod=0.goal}) is proven.] \medskip

Now, we can apply (\ref{pf.lem.uptri.prod=0.goal}) to $k=n$, and conclude that
the first $n$ columns of the matrix $T_{1}T_{2}\cdots T_{n}$ is $0$. Since
this matrix $T_{1}T_{2}\cdots T_{n}$ has only $n$ columns, this means that all
of its columns are $0$. In other words, the entire matrix $T_{1}T_{2}\cdots
T_{n}$ is $0$. This proves Lemma \ref{lem.uptri.prod=0}.
\end{proof}

\begin{proof}
[Proof of Theorem \ref{thm.schurtri.ch.ch} for $\mathbb{F}=\mathbb{C}$.]Assume
that $\mathbb{F}=\mathbb{C}$. The Schur triangularization theorem (Theorem
\ref{thm.schurtri.schurtri}) shows that $A$ is unitarily similar to an
upper-triangular matrix. Hence, $A$ is similar to an upper-triangular matrix
(because unitarily similar matrices always are similar). In other words, there
exist an invertible matrix $U$ and an upper-triangular matrix $T$ such that
$A=UTU^{-1}$. Consider these $U$ and $T$.

From $A=UTU^{-1}$, we obtain
\[
p_{A}\left(  A\right)  =p_{A}\left(  UTU^{-1}\right)  =U\cdot p_{A}\left(
T\right)  \cdot U^{-1}%
\]
(by Lemma \ref{lem.p(A).multiplicative} \textbf{(c)}, applied to $p_{A}$, $U$
and $T$ instead of $f$, $W$ and $A$). Hence, in order to prove that
$p_{A}\left(  A\right)  =0$, it will suffice to show that $p_{A}\left(
T\right)  =0$.

Now, let $\lambda_{1},\lambda_{2},\ldots,\lambda_{n}$ be the diagonal entries
of $T$. Then, by Proposition \ref{prop.schurtri.similar.T-diag}, these
diagonal entries $\lambda_{1},\lambda_{2},\ldots,\lambda_{n}$ are the
eigenvalues of $A$ (with algebraic multiplicities). Hence,%
\[
p_{A}=\left(  t-\lambda_{1}\right)  \left(  t-\lambda_{2}\right)
\cdots\left(  t-\lambda_{n}\right)
\]
(since $p_{A}$ is monic, and the roots of $p_{A}$ are precisely the
eigenvalues of $A$ with algebraic multiplicities). Therefore,%
\begin{align}
p_{A}\left(  T\right)   &  =\left(  \left(  t-\lambda_{1}\right)  \left(
t-\lambda_{2}\right)  \cdots\left(  t-\lambda_{n}\right)  \right)  \left(
T\right) \nonumber\\
&  =\left(  T-\lambda_{1}I_{n}\right)  \left(  T-\lambda_{2}I_{n}\right)
\cdots\left(  T-\lambda_{n}I_{n}\right)
\label{pf.thm.schurtri.ch.ch.F=C.pA(T)=}%
\end{align}
(by Lemma \ref{lem.p(A).multiplicative} \textbf{(b)}, applied to $n$, $T$ and
$t-\lambda_{i}$ instead of $k$, $A$ and $f_{i}$).

We have%
\begin{equation}
T_{i,i}=\lambda_{i}\ \ \ \ \ \ \ \ \ \ \text{for each }i\in\left[  n\right]
\label{pf.thm.schurtri.ch.ch.F=C.Tii=}%
\end{equation}
(since $\lambda_{1},\lambda_{2},\ldots,\lambda_{n}$ are the diagonal entries
of $T$).

However, the $n$ matrices $T-\lambda_{1}I_{n},T-\lambda_{2}I_{n}%
,\ldots,T-\lambda_{n}I_{n}$ are upper-triangular (since they are linear
combinations of the upper-triangular matrices $T$ and $I_{n}$). Moreover, for
each $i\in\left[  n\right]  $, the $i$-th diagonal entry of the matrix
$T-\lambda_{i}I_{n}$ is $0$ (because this entry is $\left(  T-\lambda_{i}%
I_{n}\right)  _{i,i}=\underbrace{T_{i,i}}_{\substack{=\lambda_{i}\\\text{(by
(\ref{pf.thm.schurtri.ch.ch.F=C.Tii=}))}}}-\lambda_{i}\underbrace{\left(
I_{n}\right)  _{i,i}}_{=1}=\lambda_{i}-\lambda_{i}=0$). Thus, Lemma
\ref{lem.uptri.prod=0} (applied to $T_{i}=T-\lambda_{i}I_{n}$) yields%
\[
\left(  T-\lambda_{1}I_{n}\right)  \left(  T-\lambda_{2}I_{n}\right)
\cdots\left(  T-\lambda_{n}I_{n}\right)  =0.
\]
In view of (\ref{pf.thm.schurtri.ch.ch.F=C.pA(T)=}), this rewrites as
$p_{A}\left(  T\right)  =0$. As explained above, this entails $p_{A}\left(
A\right)  =0$. Thus, Theorem \ref{thm.schurtri.ch.ch} is proved under the
assumption that $\mathbb{F}=\mathbb{C}$.
\end{proof}

The Cayley--Hamilton theorem has an interesting consequence: it yields that
the inverse of an invertible matrix can be written as a polynomial applied to
this matrix. (However, the specific polynomial that needs to be applied
depends on this matrix.) In more detail:

\begin{exercise}
\label{exe.schurtri.ch.inverse-poly}\fbox{3} Let $\mathbb{F}$ be a field. Let
$n$ be a positive integer. Let $A\in\mathbb{F}^{n\times n}$ be an invertible
matrix with entries in $\mathbb{F}$. Prove that there exists a polynomial $f$
of degree $n-1$ in the single indeterminate $t$ over $\mathbb{F}$ such that
$A^{-1}=f\left(  A\right)  $.
\end{exercise}

For example, for $n=2$, we have $A^{-1}=uI_{2}-vA$ with $u=\dfrac
{\operatorname*{Tr}A}{\det A}$ and $v=\dfrac{1}{\det A}$. \medskip

Another consequence of Cayley--Hamilton is that the powers of a given square
matrix $A\in\mathbb{F}^{n\times n}$ span a vector space of dimension $\leq n$:

\begin{exercise}
\label{exe.schurtri.ch.powers-span}\fbox{3} Let $\mathbb{F}$ be a field. Let
$A\in\mathbb{F}^{n\times n}$ be a square matrix with entries in $\mathbb{F}$.
Prove that for any nonnegative integer $k$, the power $A^{k}$ can be written
as an $\mathbb{F}$-linear combination of the first $n$ powers $A^{0}%
,A^{1},\ldots,A^{n-1}$.
\end{exercise}

Yet another rather curious consequence is an application to linearly recurrent
sequences. We recall what these are:

\begin{definition}
Let $a_{1},a_{2},\ldots,a_{k}$ be $k$ numbers. A sequence $\left(  x_{0}%
,x_{1},x_{2},\ldots\right)  $ of numbers is said to be $\left(  a_{1}%
,a_{2},\ldots,a_{k}\right)  $\emph{-recurrent} if each integer $i\geq k$
satisfies%
\[
x_{i}=a_{1}x_{i-1}+a_{2}x_{i-2}+\cdots+a_{k}x_{i-k}.
\]

\end{definition}

For instance, the famous Fibonacci sequence $\left(  f_{0},f_{1},f_{2}%
,\ldots\right)  $ (defined by the starting values $f_{0}=0$ and $f_{1}=1$ and
the recurrence $f_{i}=f_{i-1}+f_{i-2}$) is $\left(  1,1\right)  $-recurrent
(by its very definition). Now, it is a simple exercise to check that the
\textquotedblleft even-indexed Fibonacci sequence\textquotedblright\ $\left(
f_{0},f_{2},f_{4},f_{6},\ldots\right)  $ and the \textquotedblleft odd-indexed
Fibonacci sequence\textquotedblright\ $\left(  f_{1},f_{3},f_{5},f_{7}%
,\ldots\right)  $ themselves follow a simple recursion; to wit, they are both
$\left(  3,-1\right)  $-recurrent (check this!). Likewise, the
\textquotedblleft multiples-of-$3$-indexed Fibonacci
sequence\textquotedblright\ $\left(  f_{0},f_{3},f_{6},f_{9},\ldots\right)  $
as well as its companions $\left(  f_{1},f_{4},f_{7},f_{10},\ldots\right)  $
and $\left(  f_{2},f_{5},f_{8},f_{11},\ldots\right)  $ are $\left(
4,1\right)  $-recurrent. This generalizes:

\begin{exercise}
\label{exe.schurtri.ch.lin-rec-kd}\fbox{5} Let $a_{1},a_{2},\ldots,a_{k}$ be
$k$ numbers. Let $\left(  x_{0},x_{1},x_{2},\ldots\right)  $ be any $\left(
a_{1},a_{2},\ldots,a_{k}\right)  $-recurrent sequence of numbers. Let $d$ be a
positive integer. Show that there exist $k$ integers $b_{1},b_{2},\ldots
,b_{k}$ such that each $i\geq kd$ satisfies%
\[
x_{i}=b_{1}x_{i-d}+b_{2}x_{i-2d}+\cdots+b_{k}x_{i-kd}.
\]
(This means that the sequences $\left(  x_{0+u},x_{d+u},x_{2d+u}%
,x_{3d+u},\ldots\right)  $ are $\left(  b_{1},b_{2},\ldots,b_{k}\right)
$-recurrent for all $u\geq0$.)

[\textbf{Hint:} For each $j\geq0$, define the column vector $v_{j}$ by
$v_{j}=\left(
\begin{array}
[c]{c}%
x_{j}\\
x_{j+1}\\
\vdots\\
x_{j+k-1}%
\end{array}
\right)  \in\mathbb{R}^{k}$. Let $A$ be the $k\times k$-matrix $\left(
\begin{array}
[c]{ccccc}%
0 & 1 & 0 & \cdots & 0\\
0 & 0 & 1 & \cdots & 0\\
\vdots & \vdots & \vdots & \ddots & \vdots\\
0 & 0 & 0 & \cdots & 1\\
a_{k} & a_{k-1} & a_{k-2} & \cdots & a_{1}%
\end{array}
\right)  \in\mathbb{R}^{k\times k}$. Start by showing that $Av_{j}=v_{j+1}$
for each $j\geq0$.]
\end{exercise}

\subsection{Sylvester's equation}

We shall next see another application of the Cayley--Hamilton theorem. First,
a notation:

\begin{definition}
\label{def.schurtri.syl.spec}Let $A\in\mathbb{C}^{n\times n}$. Then, the
\emph{spectrum} of $A$ is defined to be the set of all eigenvalues of $A$.
This spectrum is denoted by $\sigma\left(  A\right)  $.
\end{definition}

Some authors write $\operatorname*{spec}A$ instead of $\sigma\left(  A\right)
$. (Some also define it to be a multiset rather than a set; however, the set
suffices for our purposes.)

We now claim the following:

\begin{theorem}
\label{thm.schurtri.syl.equivalence}Let $A\in\mathbb{C}^{n\times n}$ be an
$n\times n$-matrix, and let $B\in\mathbb{C}^{m\times m}$ be an $m\times
m$-matrix (both with complex entries). Let $C\in\mathbb{C}^{n\times m}$ be an
$n\times m$-matrix. Then, the following two statements are equivalent:

\begin{itemize}
\item $\mathcal{U}$: There is a \textbf{unique} matrix $X\in\mathbb{C}%
^{n\times m}$ such that $AX-XB=C$.

\item $\mathcal{V}$: We have $\sigma\left(  A\right)  \cap\sigma\left(
B\right)  =\varnothing$.
\end{itemize}
\end{theorem}

\begin{example}
Let us take $n=1$ and $m=1$, and see what Theorem
\ref{thm.schurtri.syl.equivalence} becomes. In this case, the matrices $A$,
$B$ and $C$ are $1\times1$-matrices, so we can view them as scalars. Let us
therefore write $a$, $b$ and $c$ for them. Then, Theorem
\ref{thm.schurtri.syl.equivalence} says that the following two statements are equivalent:

\begin{itemize}
\item $\mathcal{U}$: There is a \textbf{unique} complex number $x$ such that
$ax-xb=c$.

\item $\mathcal{V}$: We have $\left\{  a\right\}  \cap\left\{  b\right\}
=\varnothing$ (that is, $a\neq b$).
\end{itemize}

This is not surprising, because the linear equation $ax-xb=c$ has a unique
solution (namely, $x=\dfrac{c}{a-b}$) when $a\neq b$, and otherwise has either
none or infinitely many solutions.
\end{example}

The equation $AX-XB=C$ in Theorem \ref{thm.schurtri.syl.equivalence} is known
as \emph{Sylvester's equation}. It is much harder than the superficially
similar equations $AX-BX=C$ and $XA-XB=C$ (see Exercise
\ref{exe.schurtri.syl.AX-BX} for the first of these). In fact, since the $X$
is on different sides in $AX$ and in $XB$, it cannot be factored out from
$AX-XB$ (matrices do not generally commute).

\begin{exercise}
\label{exe.schurtri.syl.AX-BX}\fbox{2} Let $A\in\mathbb{C}^{n\times m}$,
$B\in\mathbb{C}^{n\times m}$ and $C\in\mathbb{C}^{n\times p}$ be three complex
matrices. Prove that there exists a matrix $X\in\mathbb{C}^{m\times p}$ such
that $AX-BX=C$ if and only if each column of $C$ belongs to the image (=
column space) of $A-B$.
\end{exercise}

We shall prove only the $\mathcal{V}\Longrightarrow\mathcal{U}$ part of
Theorem \ref{thm.schurtri.syl.equivalence}; the opposite direction will be
left as an exercise (Exercise \ref{exe.schurtri.syl.sigA-B} \textbf{(b)}). Our
proof of $\mathcal{V}\Longrightarrow\mathcal{U}$ will rely on the following lemma:

\begin{lemma}
\label{lem.schurtri.syl.AX=XB}Let $\mathbb{F}$ be a field. Let $A\in
\mathbb{F}^{n\times n}$, $B\in\mathbb{F}^{m\times m}$ and $X\in\mathbb{F}%
^{n\times m}$ be three matrices such that $AX=XB$. Then: \medskip

\textbf{(a)} We have $A^{k}X=XB^{k}$ for each $k\in\mathbb{N}$. \medskip

\textbf{(b)} Let $p$ be a polynomial in a single indeterminate $x$ with
coefficients in $\mathbb{F}$. Then, $p\left(  A\right)  X=Xp\left(  B\right)
$.
\end{lemma}

\begin{proof}
[Proof of Lemma \ref{lem.schurtri.syl.AX=XB}.]\textbf{(a)} Intuitively, this
is easy: For instance, if $k=4$, then this is saying that $A^{4}X=XB^{4}$, but
this follows from
\[
A^{4}B=AAA\underbrace{AB}_{=BA}=AA\underbrace{AB}_{=BA}A=A\underbrace{AB}%
_{=BA}AA=\underbrace{AB}_{=BA}AAA=BAAAA=BA^{4}.
\]


Formally, Lemma \ref{lem.schurtri.syl.AX=XB} \textbf{(a)} is proved by
induction on $k$:

\textit{Induction base:} We have $A^{0}X=XB^{0}$, since both sides of this
equation equal $X$. Thus, Lemma \ref{lem.schurtri.syl.AX=XB} \textbf{(a)}
holds for $k=0$.

\textit{Induction step:} Let $\ell\in\mathbb{N}$. Assume (as the induction
hypothesis) that Lemma \ref{lem.schurtri.syl.AX=XB} \textbf{(a)} holds for
$k=\ell$. We must prove that Lemma \ref{lem.schurtri.syl.AX=XB} \textbf{(a)}
holds for $k=\ell+1$.

We have assumed that Lemma \ref{lem.schurtri.syl.AX=XB} \textbf{(a)} holds for
$k=\ell$. In other words, $A^{\ell}X=XB^{\ell}$. Thus,%
\[
\underbrace{A^{\ell+1}}_{=AA^{\ell}}X=A\underbrace{A^{\ell}X}_{=XB^{\ell}%
}=\underbrace{AX}_{=XB}B^{\ell}=X\underbrace{BB^{\ell}}_{=B^{\ell+1}}%
=XB^{\ell+1}.
\]
In other words, Lemma \ref{lem.schurtri.syl.AX=XB} \textbf{(a)} holds for
$k=\ell+1$. This completes the induction step; thus, Lemma
\ref{lem.schurtri.syl.AX=XB} \textbf{(a)} is proven. \medskip

\textbf{(b)} Write the polynomial $p$ in the form $p\left(  x\right)
=\sum_{k=0}^{d}p_{k}x^{k}$ for some coefficients $p_{0},p_{1},\ldots,p_{d}%
\in\mathbb{F}$. Then, Definition \ref{def.schurtri.normal.p(A)} yields%
\[
p\left(  A\right)  =\sum_{k=0}^{d}p_{k}A^{k}\ \ \ \ \ \ \ \ \ \ \text{and}%
\ \ \ \ \ \ \ \ \ \ p\left(  B\right)  =\sum_{k=0}^{d}p_{k}B^{k}.
\]
Hence,%
\begin{align*}
p\left(  A\right)  X  &  =\left(  \sum_{k=0}^{d}p_{k}A^{k}\right)
X=\sum_{k=0}^{d}p_{k}\underbrace{A^{k}X}_{\substack{=XB^{k}\\\text{(by Lemma
\ref{lem.schurtri.syl.AX=XB} \textbf{(a)})}}}=\sum_{k=0}^{d}p_{k}%
XB^{k}\ \ \ \ \ \ \ \ \ \ \text{and}\\
Xp\left(  B\right)   &  =X\sum_{k=0}^{d}p_{k}B^{k}=\sum_{k=0}^{d}p_{k}XB^{k}.
\end{align*}
Comparing these two equalities, we find $p\left(  A\right)  X=Xp\left(
B\right)  $. Thus, Lemma \ref{lem.schurtri.syl.AX=XB} \textbf{(b)} is proven.
\end{proof}

\begin{proof}
[Proof of the $\mathcal{V}\Longrightarrow\mathcal{U}$ part of Theorem
\ref{thm.schurtri.syl.equivalence}.]First, we observe that the matrix space
$\mathbb{C}^{n\times m}$ is itself a $\mathbb{C}$-vector space of dimension
$nm$.

Consider the map%
\begin{align*}
L:\mathbb{C}^{n\times m}  &  \rightarrow\mathbb{C}^{n\times m},\\
X  &  \mapsto AX-XB.
\end{align*}
This map $L$ is linear, because for any $\alpha,\beta\in\mathbb{C}$ and any
$X,Y\in\mathbb{C}^{n\times m}$, we have%
\begin{align*}
L\left(  \alpha X+\beta Y\right)   &  =A\left(  \alpha X+\beta Y\right)
-\left(  \alpha X+\beta Y\right)  B\\
&  =\alpha AX+\beta AY-\alpha XB-\beta YB\\
&  =\alpha\underbrace{\left(  AX-XB\right)  }_{=L\left(  X\right)  }%
+\beta\underbrace{\left(  AY-YB\right)  }_{=L\left(  Y\right)  }=\alpha
L\left(  X\right)  +\beta L\left(  Y\right)  .
\end{align*}


Now, assume that statement $\mathcal{V}$ holds. That is, we have
$\sigma\left(  A\right)  \cap\sigma\left(  B\right)  =\varnothing$. We shall
now show that $\operatorname*{Ker}L=0$. This will then yield that $L$ is bijective.

Indeed, let $X\in\operatorname*{Ker}L$. Thus, $X\in\mathbb{C}^{n\times m}$ and
$L\left(  X\right)  =0$. However, the definition of $L$ yields $L\left(
X\right)  =AX-XB$. Therefore, $AX-XB=L\left(  X\right)  =0$. In other words,
$AX=XB$. Hence, we can apply Lemma \ref{lem.schurtri.syl.AX=XB}.

Thus, Lemma \ref{lem.schurtri.syl.AX=XB} \textbf{(b)} (applied to $p=p_{A}$)
yields $p_{A}\left(  A\right)  X=Xp_{A}\left(  B\right)  $. However, Theorem
\ref{thm.schurtri.ch.ch} \textbf{(a)} yields $p_{A}\left(  A\right)  =0$, so
that $p_{A}\left(  A\right)  X=0X=0$. Comparing this with $p_{A}\left(
A\right)  X=Xp_{A}\left(  B\right)  $, we obtain $Xp_{A}\left(  B\right)  =0$.

We shall show that the matrix $p_{A}\left(  B\right)  $ is invertible. Indeed,
Theorem \ref{thm.schurtri.ch.fta-cons} \textbf{(a)} shows that the polynomial
$p_{A}$ factors into $n$ linear terms:%
\begin{equation}
p_{A}=\left(  t-\lambda_{1}\right)  \left(  t-\lambda_{2}\right)
\cdots\left(  t-\lambda_{n}\right)  ,
\label{pf.thm.schurtri.syl.equivalence.pA=prod}%
\end{equation}
where $\lambda_{1},\lambda_{2},\ldots,\lambda_{n}\in\mathbb{C}$ are its roots.
Moreover, these roots $\lambda_{1},\lambda_{2},\ldots,\lambda_{n}$ are the
eigenvalues of $A$ (by Theorem \ref{thm.schurtri.ch.fta-cons} \textbf{(b)});
thus, $\left\{  \lambda_{1},\lambda_{2},\ldots,\lambda_{n}\right\}
=\sigma\left(  A\right)  $.

Substituting the matrix $B$ for $t$ on both sides of the equality
(\ref{pf.thm.schurtri.syl.equivalence.pA=prod}), we obtain%
\begin{equation}
p_{A}\left(  B\right)  =\left(  B-\lambda_{1}I_{n}\right)  \left(
B-\lambda_{2}I_{n}\right)  \cdots\left(  B-\lambda_{n}I_{n}\right)
\label{pf.thm.schurtri.syl.equivalence.pAB=prod}%
\end{equation}
(by Lemma \ref{lem.p(A).multiplicative} \textbf{(b)}, applied to $B$ and $n$
and $t-\lambda_{i}$ instead of $A$ and $k$ and $f_{i}$).

Now, let $i\in\left[  n\right]  $. Then, $\lambda_{i}\in\left\{  \lambda
_{1},\lambda_{2},\ldots,\lambda_{n}\right\}  =\sigma\left(  A\right)  $.
Therefore, $\lambda_{i}\notin\sigma\left(  B\right)  $ (since having
$\lambda_{i}\in\sigma\left(  B\right)  $ would yield $\lambda_{i}\in
\sigma\left(  A\right)  \cap\sigma\left(  B\right)  $, which would contradict
$\sigma\left(  A\right)  \cap\sigma\left(  B\right)  =\varnothing$). In other
words, $\lambda_{i}$ is not an eigenvalue of $B$. In other words, $\det\left(
\lambda_{i}I_{n}-B\right)  \neq0$ (by the definition of an eigenvalue). Hence,
the matrix $\lambda_{i}I_{n}-B$ is invertible. In other words, the matrix
$B-\lambda_{i}I_{n}$ is invertible (since $B-\lambda_{i}I_{n}=-\left(
\lambda_{i}I_{n}-B\right)  $).

Forget that we fixed $i$. We thus have shown that the matrix $B-\lambda
_{i}I_{n}$ is invertible for each $i\in\left[  n\right]  $. In other words,
the $n$ matrices $B-\lambda_{1}I_{n},B-\lambda_{2}I_{n},\ldots,B-\lambda
_{n}I_{n}$ are invertible. Hence, their product $\left(  B-\lambda_{1}%
I_{n}\right)  \left(  B-\lambda_{2}I_{n}\right)  \cdots\left(  B-\lambda
_{n}I_{n}\right)  $ is invertible as well. In view of
(\ref{pf.thm.schurtri.syl.equivalence.pAB=prod}), this shows that
$p_{A}\left(  B\right)  $ is invertible. Hence, from $Xp_{A}\left(  B\right)
=0$, we conclude that $X=0$.

Now, forget that we fixed $X$. We thus have shown that $X=0$ for each
$X\in\operatorname*{Ker}L$. In other words, $\operatorname*{Ker}L=0$. Hence,
the linear map $L$ is injective.

However, it is well-known that an injective linear map between two
finite-dimensional vector spaces of the same dimension is necessarily
bijective\footnote{\textit{Proof.} Let $f:U\rightarrow V$ be an injective
linear map between two finite-dimensional vector spaces of the same dimension.
We must show that $f$ is bijective. We have $\operatorname*{Ker}f=0$ (since
$f$ is injective) and thus $\dim\left(  \operatorname*{Ker}f\right)  =0$. The
rank-nullity theorem yields%
\[
\dim U=\underbrace{\dim\left(  \operatorname*{Ker}f\right)  }_{\substack{=0}%
}+\dim\left(  \operatorname{Im}f\right)  =\dim\left(  \operatorname{Im}%
f\right)  ,
\]
so that $\dim\left(  \operatorname{Im}f\right)  =\dim U=\dim V$ (since $U$ and
$V$ have the same dimension), and therefore $\operatorname{Im}f=V$ (because
$\operatorname{Im}f$ is a subspace of $V$). This shows that $f$ is surjective.
Since $f$ is also injective, we thus conclude that $f$ is bijective.}. Hence,
$L$ is bijective (since $L$ is an injective linear map between $\mathbb{C}%
^{n\times m}$ and $\mathbb{C}^{n\times m}$). Therefore, there exists a
\textbf{unique} matrix $X\in\mathbb{C}^{n\times m}$ such that $L\left(
X\right)  =C$. In other words, there is a \textbf{unique} matrix
$X\in\mathbb{C}^{n\times m}$ such that $AX-XB=C$ (since $L\left(  X\right)
=AX-XB$). In other words, statement $\mathcal{U}$ holds. Thus, the implication
$\mathcal{V}\Longrightarrow\mathcal{U}$ is proven.
\end{proof}

\begin{exercise}
\label{exe.schurtri.syl.sigA-B}\fbox{5} Let $A$, $B$ and $C$ be as in Theorem
\ref{thm.schurtri.syl.equivalence}. \medskip

\textbf{(a)} Let the linear map $L$ be as in the above proof of the
$\mathcal{V}\Longrightarrow\mathcal{U}$ part of Theorem
\ref{thm.schurtri.syl.equivalence}. Prove that if $\lambda\in\sigma\left(
A\right)  $ and $\mu\in\sigma\left(  B\right)  $, then $\lambda-\mu$ is an
eigenvalue of $L$ (that is, there exists a nonzero matrix $X\in\mathbb{C}%
^{n\times m}$ satisfying $L\left(  X\right)  =\left(  \lambda-\mu\right)  X$).
\medskip

\textbf{(b)} Prove the implication $\mathcal{U}\Longrightarrow\mathcal{V}$ in
Theorem \ref{thm.schurtri.syl.equivalence} (thus completing the proof of the theorem).
\end{exercise}

We note that more can be said: If $A$, $B$ and $C$ are as in Theorem
\ref{thm.schurtri.syl.equivalence}, and if $L$ is as in the above proof, then
\textbf{all} eigenvalues of $L$ have the form $\lambda-\mu$ for $\lambda
\in\sigma\left(  A\right)  $ and $\mu\in\sigma\left(  B\right)  $. But this
seems harder to prove at this point. \medskip%

\[%
\begin{tabular}
[c]{||l||}\hline\hline
\textbf{Lecture 7 starts here.}\\\hline\hline
\end{tabular}
\]


We shall next prove a somewhat surprising consequence of Theorem
\ref{thm.schurtri.syl.equivalence}: a similarity criterion for certain block matrices:

\begin{corollary}
\label{cor.schurtri.block-sim}Let $A\in\mathbb{C}^{n\times n}$, $B\in
\mathbb{C}^{m\times m}$ and $C\in\mathbb{C}^{n\times m}$ be three matrices
such that $\sigma\left(  A\right)  \cap\sigma\left(  B\right)  =\varnothing$.
Then, the two $\left(  n+m\right)  \times\left(  n+m\right)  $-matrices%
\[
\left(
\begin{array}
[c]{cc}%
A & C\\
0 & B
\end{array}
\right)  \ \ \ \ \ \ \ \ \ \ \text{and}\ \ \ \ \ \ \ \ \ \ \left(
\begin{array}
[c]{cc}%
A & 0\\
0 & B
\end{array}
\right)
\]
(written in block matrix notation) are similar.
\end{corollary}

\begin{example}
Let $A=\left(
\begin{array}
[c]{cc}%
1 & 3\\
0 & 1
\end{array}
\right)  $ and $B=\left(
\begin{array}
[c]{c}%
2
\end{array}
\right)  $ and $C=\left(
\begin{array}
[c]{c}%
7\\
9
\end{array}
\right)  $. Then, Corollary \ref{cor.schurtri.block-sim} says that the
matrices%
\[
\left(
\begin{array}
[c]{ccc}%
1 & 3 & 7\\
0 & 1 & 9\\
0 & 0 & 2
\end{array}
\right)  \ \ \ \ \ \ \ \ \ \ \text{and}\ \ \ \ \ \ \ \ \ \ \left(
\begin{array}
[c]{ccc}%
1 & 3 & 0\\
0 & 1 & 0\\
0 & 0 & 2
\end{array}
\right)
\]
are similar.
\end{example}

\begin{proof}
[Proof of Corollary \ref{cor.schurtri.block-sim}.]Theorem
\ref{thm.schurtri.syl.equivalence} (specifically, its $\mathcal{V}%
\Longrightarrow\mathcal{U}$ direction) shows that there is a unique matrix
$X\in\mathbb{C}^{n\times m}$ such that $AX-XB=C$. Consider this $X$.

Now, let $S=\left(
\begin{array}
[c]{cc}%
I_{n} & X\\
0 & I_{m}%
\end{array}
\right)  $. (This is an $\left(  n+m\right)  \times\left(  n+m\right)
$-matrix written in block matrix notation.) Now, I claim that this matrix $S$
is invertible and that%
\[
\left(
\begin{array}
[c]{cc}%
A & 0\\
0 & B
\end{array}
\right)  =S\left(
\begin{array}
[c]{cc}%
A & C\\
0 & B
\end{array}
\right)  S^{-1}.
\]
Once this claim is proved, the claim of Corollary \ref{cor.schurtri.block-sim}
will follow (by the definition of \textquotedblleft similar\textquotedblright).

To see that $S$ is invertible, we construct an inverse. Namely, we set
$S^{\prime}=\left(
\begin{array}
[c]{cc}%
I_{n} & -X\\
0 & I_{m}%
\end{array}
\right)  $ (again an $\left(  n+m\right)  \times\left(  n+m\right)  $-matrix).
Then, the definitions of $S$ and $S^{\prime}$ yield%
\begin{align*}
SS^{\prime}  &  =\left(
\begin{array}
[c]{cc}%
I_{n} & X\\
0 & I_{m}%
\end{array}
\right)  \left(
\begin{array}
[c]{cc}%
I_{n} & -X\\
0 & I_{m}%
\end{array}
\right) \\
&  =\left(
\begin{array}
[c]{cc}%
I_{n}I_{n}+X\cdot0 & I_{n}\left(  -X\right)  +XI_{m}\\
0I_{n}+I_{m}\cdot0 & 0\left(  -X\right)  +I_{m}I_{m}%
\end{array}
\right)  \ \ \ \ \ \ \ \ \ \ \left(  \text{by Proposition
\ref{prop.blockmatrix.mult-2x2}}\right) \\
&  =\left(
\begin{array}
[c]{cc}%
I_{n} & 0\\
0 & I_{m}%
\end{array}
\right)  \ \ \ \ \ \ \ \ \ \ \left(
\begin{array}
[c]{c}%
\text{since }I_{n}I_{n}+X\cdot0=I_{n}\text{ and }I_{n}\left(  -X\right)
+XI_{m}=-X+X=0\\
\text{and }0I_{n}+I_{m}\cdot0=0\text{ and }0\left(  -X\right)  +I_{m}%
I_{m}=I_{m}%
\end{array}
\right) \\
&  =I_{n+m}%
\end{align*}
and similarly $S^{\prime}S=I_{n+m}$. Thus, the matrices $S$ and $S^{\prime}$
are mutually inverse. Hence, $S$ is invertible.

It remains to check that%
\begin{equation}
\left(
\begin{array}
[c]{cc}%
A & 0\\
0 & B
\end{array}
\right)  =S\left(
\begin{array}
[c]{cc}%
A & C\\
0 & B
\end{array}
\right)  S^{-1}. \label{pf.cor.schurtri.block-sim.4}%
\end{equation}
To do so, it suffices to check the equivalent identity%
\begin{equation}
\left(
\begin{array}
[c]{cc}%
A & 0\\
0 & B
\end{array}
\right)  S=S\left(
\begin{array}
[c]{cc}%
A & C\\
0 & B
\end{array}
\right)  \label{pf.cor.schurtri.block-sim.5}%
\end{equation}
(indeed, these two identities are equivalent, since $S$ is invertible). This
we do by computing both sides and comparing: Namely, the definition of $S$
yields%
\begin{align*}
\left(
\begin{array}
[c]{cc}%
A & 0\\
0 & B
\end{array}
\right)  S  &  =\left(
\begin{array}
[c]{cc}%
A & 0\\
0 & B
\end{array}
\right)  \left(
\begin{array}
[c]{cc}%
I_{n} & X\\
0 & I_{m}%
\end{array}
\right) \\
&  =\left(
\begin{array}
[c]{cc}%
AI_{n}+0\cdot0 & A\cdot X+0\cdot I_{m}\\
0I_{n}+B\cdot0 & 0X+BI_{m}%
\end{array}
\right)  \ \ \ \ \ \ \ \ \ \ \left(  \text{by Proposition
\ref{prop.blockmatrix.mult-2x2}}\right) \\
&  =\left(
\begin{array}
[c]{cc}%
A & AX\\
0 & B
\end{array}
\right)  \ \ \ \ \ \ \ \ \ \ \left(  \text{by the obvious simplifications}%
\right)
\end{align*}
and%
\begin{align*}
S\left(
\begin{array}
[c]{cc}%
A & C\\
0 & B
\end{array}
\right)   &  =\left(
\begin{array}
[c]{cc}%
I_{n} & X\\
0 & I_{m}%
\end{array}
\right)  \left(
\begin{array}
[c]{cc}%
A & C\\
0 & B
\end{array}
\right) \\
&  =\left(
\begin{array}
[c]{cc}%
I_{n}A+X\cdot0 & I_{n}C+XB\\
0A+I_{m}\cdot0 & 0C+I_{m}\cdot B
\end{array}
\right)  \ \ \ \ \ \ \ \ \ \ \left(  \text{by Proposition
\ref{prop.blockmatrix.mult-2x2}}\right) \\
&  =\left(
\begin{array}
[c]{cc}%
A & C+XB\\
0 & B
\end{array}
\right)  \ \ \ \ \ \ \ \ \ \ \left(  \text{by the obvious simplifications}%
\right) \\
&  =\left(
\begin{array}
[c]{cc}%
A & AX\\
0 & B
\end{array}
\right)  \ \ \ \ \ \ \ \ \ \ \ \left(  \text{since }AX-XB=C\text{ entails
}C+XB=AX\right)  .
\end{align*}
Comparing these two equalities yields (\ref{pf.cor.schurtri.block-sim.5}).
Thus, we obtain (\ref{pf.cor.schurtri.block-sim.4}) (by multiplying both sides
of (\ref{pf.cor.schurtri.block-sim.5}) with $S^{-1}$ from the right). But this
shows that the two matrices $\left(
\begin{array}
[c]{cc}%
A & C\\
0 & B
\end{array}
\right)  $ and $\left(
\begin{array}
[c]{cc}%
A & 0\\
0 & B
\end{array}
\right)  $ are similar. This proves Corollary \ref{cor.schurtri.block-sim}.
\end{proof}

\section{The Jordan canonical form (\cite[Chapter 3]{HorJoh13})}

This chapter is devoted to the \emph{Jordan canonical form} (and some of its
variants), which is a normal form for $n\times n$-matrices over $\mathbb{C}$
with respect to similarity. This means that each $n\times n$-matrix is similar
to a more-or-less unique matrix of a certain kind (namely, a block-diagonal
matrix made of a specific type of blocks), called its \textquotedblleft Jordan
canonical form\textquotedblright.

We recall that the notation \textquotedblleft$A\sim B$\textquotedblright%
\ (where $A$ and $B$ are two $n\times n$-matrices) means that the matrices $A$
and $B$ are similar.

\subsection{Jordan cells}

The building blocks for the Jordan canonical form are the so-called Jordan
cells. Let us define them:

\begin{definition}
\label{def.jnf.jcell}Let $\mathbb{F}$ be a field. A \emph{Jordan cell} is an
$m\times m$-matrix of the form%
\[
\left(
\begin{array}
[c]{ccccc}%
\lambda & 1 & 0 & \cdots & 0\\
0 & \lambda & 1 & \cdots & 0\\
0 & 0 & \lambda & \cdots & 0\\
\vdots & \vdots & \vdots & \ddots & \vdots\\
0 & 0 & 0 & \cdots & \lambda
\end{array}
\right)  \ \ \ \ \ \ \ \ \ \ \text{for some }m>0\text{ and some }\lambda
\in\mathbb{F}.
\]
In other words, it is an $m\times m$-matrix

\begin{itemize}
\item whose diagonal entries are $\lambda$,

\item whose entries directly above the diagonal (i.e., just one step upwards
from a diagonal entry) are $1$, and

\item whose all remaining entries are $0$.
\end{itemize}

\noindent In formal terms, it is the $m\times m$-matrix $A$ whose entries are
given by the rule%
\[
A_{i,j}=%
\begin{cases}
\lambda, & \text{if }i=j;\\
1, & \text{if }i=j-1;\\
0, & \text{otherwise}%
\end{cases}
\ \ \ \ \ \ \ \ \ \ \text{for all }i\in\left[  m\right]  \text{ and }%
j\in\left[  m\right]  .
\]


To be specific, this matrix is called the \emph{Jordan cell of size }$m$\emph{
at eigenvalue }$\lambda$. It is denoted by $J_{m}\left(  \lambda\right)  $.
\end{definition}

\begin{example}
\textbf{(a)} The Jordan cell of size $3$ at eigenvalue $-5$ is%
\[
J_{3}\left(  -5\right)  =\left(
\begin{array}
[c]{ccc}%
-5 & 1 & 0\\
0 & -5 & 1\\
0 & 0 & -5
\end{array}
\right)  .
\]


\textbf{(b)} The Jordan cell of size $2$ at eigenvalue $\pi$ is%
\[
J_{2}\left(  \pi\right)  =\left(
\begin{array}
[c]{cc}%
\pi & 1\\
0 & \pi
\end{array}
\right)  .
\]


\textbf{(c)} For any $\lambda\in\mathbb{F}$, the Jordan cell of size $1$ at
eigenvalue $\lambda$ is the $1\times1$-matrix $\left(
\begin{array}
[c]{c}%
\lambda
\end{array}
\right)  $.
\end{example}

\begin{remark}
We will chiefly use Jordan cells as building blocks for the Jordan normal
form. However, they are of some independent interest. In particular, they
serve as matrix representations for several useful linear maps.

For example, fix $m\in\mathbb{N}$, and let $P_{m}$ be the $\mathbb{C}$-vector
space of all polynomials in a single variable $t$ of degree $<m$ (with complex
coefficients). Then, $P_{m}$ has a basis $\left(  t^{0},t^{1},\ldots
,t^{m-1}\right)  $. The derivative operator $\dfrac{d}{dt}:P_{m}\rightarrow
P_{m}$ (which sends each polynomial $f\in P_{m}$ to its derivative $f^{\prime
}$) is a $\mathbb{C}$-linear map that is represented by the matrix%
\[
\left(
\begin{array}
[c]{cccccc}%
0 & 1 & 0 & 0 & \cdots & 0\\
0 & 0 & 2 & 0 & \cdots & 0\\
0 & 0 & 0 & 3 & \cdots & 0\\
0 & 0 & 0 & 0 & \cdots & 0\\
\vdots & \vdots & \vdots & \vdots & \ddots & \vdots\\
0 & 0 & 0 & 0 & \cdots & 0
\end{array}
\right)
\]
with respect to this basis. This matrix has the numbers $1,2,\ldots,m-1$ in
the cells directly above the main diagonal, and $0$s everywhere else. It is
not quite a Jordan cell. However, if we instead use the basis $\left(
\dfrac{t^{0}}{0!},\dfrac{t^{1}}{1!},\ldots,\dfrac{t^{m-1}}{\left(  m-1\right)
!}\right)  $, then the operator $\dfrac{d}{dt}$ is represented by the matrix
\[
\left(
\begin{array}
[c]{cccccc}%
0 & 1 & 0 & 0 & \cdots & 0\\
0 & 0 & 1 & 0 & \cdots & 0\\
0 & 0 & 0 & 1 & \cdots & 0\\
0 & 0 & 0 & 0 & \cdots & 0\\
\vdots & \vdots & \vdots & \vdots & \ddots & \vdots\\
0 & 0 & 0 & 0 & \cdots & 0
\end{array}
\right)
\]
which is precisely the Jordan cell $J_{m}\left(  0\right)  $. (This basis is
just a rescaled version of the basis $\left(  t^{0},t^{1},\ldots
,t^{m-1}\right)  $, where the rescaling factors have been chosen to
\textquotedblleft normalize\textquotedblright\ the $1,2,\ldots,m-1$ entries to
be $1$s.)
\end{remark}

While the Jordan cell $J_{m}\left(  \lambda\right)  $ depends on both $m$ and
$\lambda$, its dependence on $\lambda$ is not very substantial:

\begin{proposition}
\label{prop.jnf.jcell-lambda}Let $\mathbb{F}$ be a field. Let $m$ be a
positive integer, and let $\lambda\in\mathbb{F}$. Then,%
\[
J_{m}\left(  \lambda\right)  =J_{m}\left(  0\right)  +\lambda I_{m}.
\]

\end{proposition}

\begin{proof}
Definition \ref{def.jnf.jcell} yields%
\begin{equation}
J_{m}\left(  \lambda\right)  =\left(
\begin{array}
[c]{ccccc}%
\lambda & 1 & 0 & \cdots & 0\\
0 & \lambda & 1 & \cdots & 0\\
0 & 0 & \lambda & \cdots & 0\\
\vdots & \vdots & \vdots & \ddots & \vdots\\
0 & 0 & 0 & \cdots & \lambda
\end{array}
\right)  \label{pf.prop.jnf.jcell-lambda.1}%
\end{equation}
and%
\begin{equation}
J_{m}\left(  0\right)  =\left(
\begin{array}
[c]{ccccc}%
0 & 1 & 0 & \cdots & 0\\
0 & 0 & 1 & \cdots & 0\\
0 & 0 & 0 & \cdots & 0\\
\vdots & \vdots & \vdots & \ddots & \vdots\\
0 & 0 & 0 & \cdots & 0
\end{array}
\right)  . \label{pf.prop.jnf.jcell-lambda.2}%
\end{equation}
On the other hand,%
\[
\lambda I_{m}=\left(
\begin{array}
[c]{ccccc}%
\lambda & 0 & 0 & \cdots & 0\\
0 & \lambda & 0 & \cdots & 0\\
0 & 0 & \lambda & \cdots & 0\\
\vdots & \vdots & \vdots & \ddots & \vdots\\
0 & 0 & 0 & \cdots & \lambda
\end{array}
\right)  .
\]
Adding this equality to (\ref{pf.prop.jnf.jcell-lambda.2}), we obtain%
\begin{align*}
J_{m}\left(  0\right)  +\lambda I_{m}  &  =\left(
\begin{array}
[c]{ccccc}%
0 & 1 & 0 & \cdots & 0\\
0 & 0 & 1 & \cdots & 0\\
0 & 0 & 0 & \cdots & 0\\
\vdots & \vdots & \vdots & \ddots & \vdots\\
0 & 0 & 0 & \cdots & 0
\end{array}
\right)  +\left(
\begin{array}
[c]{ccccc}%
\lambda & 0 & 0 & \cdots & 0\\
0 & \lambda & 0 & \cdots & 0\\
0 & 0 & \lambda & \cdots & 0\\
\vdots & \vdots & \vdots & \ddots & \vdots\\
0 & 0 & 0 & \cdots & \lambda
\end{array}
\right) \\
&  =\left(
\begin{array}
[c]{ccccc}%
\lambda & 1 & 0 & \cdots & 0\\
0 & \lambda & 1 & \cdots & 0\\
0 & 0 & \lambda & \cdots & 0\\
\vdots & \vdots & \vdots & \ddots & \vdots\\
0 & 0 & 0 & \cdots & \lambda
\end{array}
\right)  =J_{m}\left(  \lambda\right)
\end{align*}
(by (\ref{pf.prop.jnf.jcell-lambda.1})). This proves Proposition
\ref{prop.jnf.jcell-lambda}.
\end{proof}

Thanks to Proposition \ref{prop.jnf.jcell-lambda}, we can reduce many
questions about $J_{m}\left(  \lambda\right)  $ to the corresponding questions
about $J_{m}\left(  0\right)  $. Let us compute the powers of $J_{m}\left(
0\right)  $:

\begin{proposition}
\label{prop.jnf.jcell0-powers}Let $\mathbb{F}$ be a field. Let $m$ be a
positive integer. Let $B=J_{m}\left(  0\right)  $. Let $p\in\mathbb{N}$. Then:
\medskip

\textbf{(a)} The entries of the $m\times m$-matrix $B^{p}$ are given by%
\[
\left(  B^{p}\right)  _{i,j}=%
\begin{cases}
1, & \text{if }i=j-p;\\
0, & \text{otherwise}%
\end{cases}
\ \ \ \ \ \ \ \ \ \ \text{for all }i,j\in\left[  m\right]  .
\]


\textbf{(b)} We have $B^{p}=0$ if $p\geq m$. \medskip

\textbf{(c)} We have $\dim\left(  \operatorname*{Ker}\left(  B^{p}\right)
\right)  =p$ if $p\leq m$. \medskip

\textbf{(d)} We have $\dim\left(  \operatorname*{Ker}\left(  B^{p}\right)
\right)  =m$ if $p\geq m$.
\end{proposition}

\begin{example}
For $m=4$, the matrix $B=J_{4}\left(  0\right)  $ from Proposition
\ref{prop.jnf.jcell0-powers} satisfies%
\begin{align*}
B  &  =\left(
\begin{array}
[c]{cccc}%
0 & 1 & 0 & 0\\
0 & 0 & 1 & 0\\
0 & 0 & 0 & 1\\
0 & 0 & 0 & 0
\end{array}
\right)  ,\ \ \ \ \ \ \ \ \ \ B^{2}=\left(
\begin{array}
[c]{cccc}%
0 & 0 & 1 & 0\\
0 & 0 & 0 & 1\\
0 & 0 & 0 & 0\\
0 & 0 & 0 & 0
\end{array}
\right)  ,\\
B^{3}  &  =\left(
\begin{array}
[c]{cccc}%
0 & 0 & 0 & 1\\
0 & 0 & 0 & 0\\
0 & 0 & 0 & 0\\
0 & 0 & 0 & 0
\end{array}
\right)  ,\ \ \ \ \ \ \ \ \ \ B^{4}=\left(
\begin{array}
[c]{cccc}%
0 & 0 & 0 & 0\\
0 & 0 & 0 & 0\\
0 & 0 & 0 & 0\\
0 & 0 & 0 & 0
\end{array}
\right)  =0.
\end{align*}
Thus, as we go from $B$ to $B^{2}$ to $B^{3}$ to $B^{4}$, the $1$s in the
cells directly above the main diagonal recede further and further upwards,
until they eventually disappear beyond the borders of the matrix. (It is
actually better to start this sequence with $B^{0}$ rather than $B$, so that
the $1$s start on the main diagonal.) Proposition \ref{prop.jnf.jcell0-powers}
\textbf{(a)} is merely a formal way of stating this phenomenon. Parts
\textbf{(b)}, \textbf{(c)} and \textbf{(d)} of Proposition
\ref{prop.jnf.jcell0-powers} follow easily from part \textbf{(a)}.
\end{example}

\begin{proof}
[Proof of Proposition \ref{prop.jnf.jcell0-powers}.]We have%
\begin{equation}
B=J_{m}\left(  0\right)  =\left(
\begin{array}
[c]{ccccc}%
0 & 1 & 0 & \cdots & 0\\
0 & 0 & 1 & \cdots & 0\\
0 & 0 & 0 & \cdots & 0\\
\vdots & \vdots & \vdots & \ddots & \vdots\\
0 & 0 & 0 & \cdots & 0
\end{array}
\right)  \label{pf.prop.jnf.jcell0-powers.B=}%
\end{equation}
(by the definition of $J_{m}\left(  0\right)  $). \medskip

\textbf{(a)} We can prove Proposition \ref{prop.jnf.jcell0-powers}
\textbf{(a)} by induction on $p$, using the definition of matrix
multiplication (and the fact that $B^{q+1}=B^{q}B$ for each $q\in\mathbb{N}$).
However, there is a more elegant proof using the action of $B$ on basis
vectors: \medskip

Forget that we fixed $p$. For each $i\in\left[  m\right]  $, let $e_{i}$ be
the column vector in $\mathbb{F}^{m}$ whose $i$-th entry is $1$ while all its
other entries are $0$. That is,
\[
e_{i}=\left(
\begin{array}
[c]{ccccccccc}%
0 & 0 & \cdots & 0 & 1 & 0 & 0 & \cdots & 0
\end{array}
\right)  ^{T},
\]
where the $1$ is in the $i$-th position. The vectors $e_{1},e_{2},\ldots
,e_{m}$ are the standard basis vectors of $\mathbb{F}^{m}$. It is well-known
that every $m\times m$-matrix $C\in\mathbb{F}^{m\times m}$ satisfies%
\begin{equation}
C_{\bullet,i}=Ce_{i}\ \ \ \ \ \ \ \ \ \ \text{for each }i\in\left[  m\right]
. \label{pf.prop.jnf.jcell0-powers.a.coli}%
\end{equation}
(Recall that $C_{\bullet,i}$ denotes the $i$-th column of $C$.)

We have so far defined the vectors $e_{i}$ only for $i\in\left[  m\right]  $.
Now, for each integer $i\notin\left[  m\right]  $, we define $e_{i}$ to be the
zero vector $0\in\mathbb{F}^{m}$. Thus, we have defined a vector $e_{i}%
\in\mathbb{F}^{m}$ for each $i\in\mathbb{Z}$ (although it is nonzero only when
$i\in\left[  m\right]  $). In particular, $e_{0}=0$ (since $0\notin\left[
m\right]  $). Note that we have%
\begin{equation}
\left(  \text{the }k\text{-th entry of the column vector }e_{i}\right)  =%
\begin{cases}
1, & \text{if }k=i;\\
0, & \text{otherwise}%
\end{cases}
\label{pf.prop.jnf.jcell0-powers.a.ient}%
\end{equation}
for each $i\in\mathbb{Z}$ and each $k\in\left[  m\right]  $%
\ \ \ \ \footnote{\textit{Proof.} If $i\in\left[  m\right]  $, then the
equality (\ref{pf.prop.jnf.jcell0-powers.a.ient}) follows from the definition
of $e_{i}$. On the other hand, if $i\notin\left[  m\right]  $, then $e_{i}=0$
(by definition), so that both sides of the equality
(\ref{pf.prop.jnf.jcell0-powers.a.ient}) are $0$ (since we don't have $k=i$
(because $k\in\left[  m\right]  $ and $i\notin\left[  m\right]  $), and thus
we have $%
\begin{cases}
1, & \text{if }k=i;\\
0, & \text{otherwise}%
\end{cases}
\ \ =0$). Thus, the equality (\ref{pf.prop.jnf.jcell0-powers.a.ient}) holds in
either case.}.

Now, from (\ref{pf.prop.jnf.jcell0-powers.B=}), we see that the columns of the
matrix $B$ are $0,e_{1},e_{2},\ldots,e_{m-1}$ in this order. In other words,
the columns of $B$ are $e_{0},e_{1},e_{2},\ldots,e_{m-1}$ in this order (since
$e_{0}=0$). In other words, we have%
\[
B_{\bullet,i}=e_{i-1}\ \ \ \ \ \ \ \ \ \ \text{for each }i\in\left[  m\right]
\]
(since $B_{\bullet,i}$ denotes the $i$-th column of $B$). However,
(\ref{pf.prop.jnf.jcell0-powers.a.coli}) (applied to $C=B$) shows that we have%
\[
B_{\bullet,i}=Be_{i}\ \ \ \ \ \ \ \ \ \ \text{for each }i\in\left[  m\right]
.
\]
Comparing these two equalities, we obtain%
\begin{equation}
Be_{i}=e_{i-1}\ \ \ \ \ \ \ \ \ \ \text{for each }i\in\left[  m\right]  .
\label{pf.prop.jnf.jcell0-powers.a.Bei}%
\end{equation}
However, this equality also holds for all $i\leq0$ (because if $i\leq0$, then
both $e_{i}$ and $e_{i-1}$ equal the zero vector $0$ (since $i\notin\left[
m\right]  $ and $i-1\notin\left[  m\right]  $), and therefore this equality
boils down to $B\cdot0=0$). Thus,
\begin{equation}
Be_{i}=e_{i-1}\ \ \ \ \ \ \ \ \ \ \text{for each integer }i\leq m.
\label{pf.prop.jnf.jcell0-powers.a.Bei2}%
\end{equation}


Now, we claim that%
\begin{equation}
B^{p}e_{i}=e_{i-p}\ \ \ \ \ \ \ \ \ \ \text{for each }p\in\mathbb{N}\text{ and
}i\in\left[  m\right]  . \label{pf.prop.jnf.jcell0-powers.a.Bpei}%
\end{equation}


[\textit{Proof of (\ref{pf.prop.jnf.jcell0-powers.a.Bpei}):} We induct on $p$:

\textit{Induction base:} We have $B^{0}=I_{m}$ and thus $B^{0}e_{i}=I_{m}%
e_{i}=e_{i}=e_{i-0}$ for each $i\in\left[  m\right]  $. In other words,
(\ref{pf.prop.jnf.jcell0-powers.a.Bpei}) holds for $p=0$.

\textit{Induction step:} Let $q\in\mathbb{N}$. Assume that
(\ref{pf.prop.jnf.jcell0-powers.a.Bpei}) holds for $p=q$. We must prove that
(\ref{pf.prop.jnf.jcell0-powers.a.Bpei}) holds for $p=q+1$.

We have assumed that (\ref{pf.prop.jnf.jcell0-powers.a.Bpei}) holds for $p=q$.
In other words, we have $B^{q}e_{i}=e_{i-q}$ for each $i\in\left[  m\right]
$. Now, let $i\in\left[  m\right]  $ be arbitrary. As we have just seen, we
have $B^{q}e_{i}=e_{i-q}$. However, from $q\geq0$, we obtain $i-q\leq i\leq m$
(since $i\in\left[  m\right]  $). Thus,
(\ref{pf.prop.jnf.jcell0-powers.a.Bei2}) (applied to $i-q$ instead of $i$)
yields $Be_{i-q}=e_{i-q-1}$. Now,%
\[
\underbrace{B^{q+1}}_{=BB^{q}}e_{i}=B\underbrace{B^{q}e_{i}}_{=e_{i-q}%
}=Be_{i-q}=e_{i-q-1}=e_{i-\left(  q+1\right)  }.
\]


Forget that we fixed $i$. We thus have shown that $B^{q+1}e_{i}=e_{i-\left(
q+1\right)  }$ for each $i\in\left[  m\right]  $. In other words,
(\ref{pf.prop.jnf.jcell0-powers.a.Bpei}) holds for $p=q+1$. This completes the
induction step. Thus, (\ref{pf.prop.jnf.jcell0-powers.a.Bpei}) is proved.]
\medskip

Now, let $p\in\mathbb{N}$ and let $i,j\in\left[  m\right]  $. Then,%
\begin{align*}
\left(  B^{p}\right)  _{\bullet,j}  &  =B^{p}e_{j}\ \ \ \ \ \ \ \ \ \ \left(
\text{by (\ref{pf.prop.jnf.jcell0-powers.a.coli}), applied to }B^{p}\text{ and
}j\text{ instead of }C\text{ and }i\right) \\
&  =e_{j-p}\ \ \ \ \ \ \ \ \ \ \left(  \text{by
(\ref{pf.prop.jnf.jcell0-powers.a.Bpei}), applied to }j\text{ instead of
}i\right)  .
\end{align*}
Furthermore,%
\begin{align*}
\left(  B^{p}\right)  _{i,j}  &  =\left(  \text{the }i\text{-th entry of the
column vector }\underbrace{\left(  B^{p}\right)  _{\bullet,j}}_{=e_{j-p}%
}\right) \\
&  =\left(  \text{the }i\text{-th entry of the column vector }e_{j-p}\right)
\\
&  =%
\begin{cases}
1, & \text{if }i=j-p;\\
0, & \text{otherwise}%
\end{cases}
\end{align*}
(by (\ref{pf.prop.jnf.jcell0-powers.a.ient}), applied to $i$ and $j-p$ instead
of $k$ and $i$). This proves Proposition \ref{prop.jnf.jcell0-powers}
\textbf{(a)}. \medskip

\textbf{(b)} Assume that $p\geq m$. Let $i,j\in\left[  m\right]  $ be
arbitrary. Then, $i\geq1$ and $j\leq m$. Hence, $\underbrace{j}_{\leq m}-p\leq
m-p\leq0$ (since $p\geq m$), so that $0\geq j-p$. On the other hand,
$i\geq1>0\geq j-p$. Therefore, $i\neq j-p$. However, Proposition
\ref{prop.jnf.jcell0-powers} \textbf{(a)} yields
\[
\left(  B^{p}\right)  _{i,j}=%
\begin{cases}
1, & \text{if }i=j-p;\\
0, & \text{otherwise}%
\end{cases}
\ \ =0\ \ \ \ \ \ \ \ \ \ \left(  \text{since }i\neq j-p\right)  .
\]


Forget that we fixed $i$ and $j$. We thus have shown that $\left(
B^{p}\right)  _{i,j}=0$ for all $i,j\in\left[  m\right]  $. In other words,
all entries of the matrix $B^{p}$ equal $0$. Thus, $B^{p}=0$. This proves
Proposition \ref{prop.jnf.jcell0-powers} \textbf{(b)}. \medskip

\textbf{(c)} Assume that $p\leq m$. We shall use the column vectors $e_{i}%
\in\mathbb{F}^{m}$ that were defined for all $i\in\mathbb{Z}$ in our above
proof of Proposition \ref{prop.jnf.jcell0-powers} \textbf{(a)}.

Let $v=\left(
\begin{array}
[c]{c}%
v_{1}\\
v_{2}\\
\vdots\\
v_{m}%
\end{array}
\right)  \in\mathbb{F}^{m}$ be a column vector. Thus,
\[
v=\left(
\begin{array}
[c]{c}%
v_{1}\\
v_{2}\\
\vdots\\
v_{m}%
\end{array}
\right)  =v_{1}e_{1}+v_{2}e_{2}+\cdots+v_{m}e_{m}=\sum_{i=1}^{m}v_{i}e_{i}.
\]
Hence,%
\begin{align*}
B^{p}v  &  =B^{p}\cdot\sum_{i=1}^{m}v_{i}e_{i}=\sum_{i=1}^{m}v_{i}%
\underbrace{B^{p}e_{i}}_{\substack{=e_{i-p}\\\text{(by
(\ref{pf.prop.jnf.jcell0-powers.a.Bpei}))}}}=\sum_{i=1}^{m}v_{i}e_{i-p}%
=\sum_{j=1-p}^{m-p}v_{j+p}e_{j}\\
&  \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \left(  \text{here, we have
substituted }j+p\text{ for }i\text{ in the sum}\right) \\
&  =\sum_{j=1-p}^{0}v_{j+p}\underbrace{e_{j}}_{\substack{=0\\\text{(since
}j\notin\left[  m\right]  \\\text{(because }j\leq0\text{))}}}+\sum_{j=1}%
^{m-p}v_{j+p}e_{j}=\underbrace{\sum_{j=1-p}^{0}v_{j+p}0}_{=0}+\sum_{j=1}%
^{m-p}v_{j+p}e_{j}=\sum_{j=1}^{m-p}v_{j+p}e_{j}\\
&  =v_{p+1}e_{1}+v_{p+2}e_{2}+\cdots+v_{m}e_{m-p}=\left(
\begin{array}
[c]{c}%
v_{p+1}\\
v_{p+2}\\
\vdots\\
v_{m}\\
0\\
0\\
\vdots\\
0
\end{array}
\right)  .
\end{align*}
Thus, $B^{p}v=0$ holds if and only if $v_{p+1}=v_{p+2}=\cdots=v_{m}=0$. In
other words, $B^{p}v=0$ holds if and only if $v\in\operatorname*{span}\left(
e_{1},e_{2},\ldots,e_{p}\right)  $ (because $v_{p+1}=v_{p+2}=\cdots=v_{m}=0$
is equivalent to $v\in\operatorname*{span}\left(  e_{1},e_{2},\ldots
,e_{p}\right)  $).

Now, forget that we fixed $v$. We thus have shown that a vector $v\in
\mathbb{F}^{m}$ satisfies $B^{p}v=0$ if and only if $v\in\operatorname*{span}%
\left(  e_{1},e_{2},\ldots,e_{p}\right)  $. Thus,
\[
\operatorname*{Ker}\left(  B^{p}\right)  =\operatorname*{span}\left(
e_{1},e_{2},\ldots,e_{p}\right)
\]
(since $\operatorname*{Ker}\left(  B^{p}\right)  $ is defined as the set of
all vectors $v\in\mathbb{F}^{m}$ satisfying $B^{p}v=0$). Therefore,%
\[
\dim\left(  \operatorname*{Ker}\left(  B^{p}\right)  \right)  =\dim\left(
\operatorname*{span}\left(  e_{1},e_{2},\ldots,e_{p}\right)  \right)  =p
\]
(since $\operatorname*{span}\left(  e_{1},e_{2},\ldots,e_{p}\right)  $ is
clearly a $p$-dimensional subspace of $\mathbb{F}^{m}$). This proves
Proposition \ref{prop.jnf.jcell0-powers} \textbf{(c)}. \medskip

\textbf{(d)} Assume that $p\geq m$. Then, Proposition
\ref{prop.jnf.jcell0-powers} \textbf{(b)} yields $B^{p}=0$. Hence,
$\dim\left(  \operatorname*{Ker}\left(  B^{p}\right)  \right)  =\dim
\underbrace{\left(  \operatorname*{Ker}0\right)  }_{=\mathbb{F}^{m}}%
=\dim\left(  \mathbb{F}^{m}\right)  =0$. Thus, Proposition
\ref{prop.jnf.jcell0-powers} \textbf{(d)} is proven.
\end{proof}

\begin{proposition}
\label{prop.jnf.jcell.multiplicity}Let $m$ be a positive integer, and let
$\lambda\in\mathbb{C}$. The only eigenvalue of the matrix $J_{m}\left(
\lambda\right)  $ is $\lambda$. This eigenvalue has algebraic multiplicity $m$
and geometric multiplicity $1$.
\end{proposition}

\begin{proof}
The definition of $J_{m}\left(  \lambda\right)  $ yields%
\[
J_{m}\left(  \lambda\right)  =\left(
\begin{array}
[c]{ccccc}%
\lambda & 1 & 0 & \cdots & 0\\
0 & \lambda & 1 & \cdots & 0\\
0 & 0 & \lambda & \cdots & 0\\
\vdots & \vdots & \vdots & \ddots & \vdots\\
0 & 0 & 0 & \cdots & \lambda
\end{array}
\right)  .
\]
This matrix is upper-triangular, so its characteristic polynomial is%
\[
p_{J_{m}\left(  \lambda\right)  }=\left(  t-\lambda\right)  \left(
t-\lambda\right)  \cdots\left(  t-\lambda\right)  =\left(  t-\lambda\right)
^{m}.
\]
Thus, the only root of this polynomial is $\lambda$. In other words, the only
eigenvalue of this matrix $J_{m}\left(  \lambda\right)  $ is $\lambda$. Its
algebraic multiplicity is $m$ (since this is its multiplicity as a root of
$p_{J_{m}\left(  \lambda\right)  }$). It remains to show that its geometric
multiplicity is $1$.

Since the geometric multiplicity of $\lambda$ is defined as $\dim\left(
\operatorname*{Ker}\left(  J_{m}\left(  \lambda\right)  -\lambda I_{m}\right)
\right)  $, this means that it remains to show that $\dim\left(
\operatorname*{Ker}\left(  J_{m}\left(  \lambda\right)  -\lambda I_{m}\right)
\right)  =1$.

Let $B=J_{m}\left(  0\right)  $. Proposition \ref{prop.jnf.jcell0-powers}
\textbf{(c)} (applied to $p=1$) yields $\dim\left(  \operatorname*{Ker}\left(
B^{1}\right)  \right)  =1$ (since $1\leq m$). In other words, $\dim\left(
\operatorname*{Ker}B\right)  =1$.

Proposition \ref{prop.jnf.jcell-lambda} yields%
\[
J_{m}\left(  \lambda\right)  =\underbrace{J_{m}\left(  0\right)  }%
_{=B}+\lambda I_{m}=B+\lambda I_{m},
\]
so that $J_{m}\left(  \lambda\right)  -\lambda I_{m}=B$. Hence,
\[
\dim\left(  \operatorname*{Ker}\underbrace{\left(  J_{m}\left(  \lambda
\right)  -\lambda I_{m}\right)  }_{=B}\right)  =\dim\left(
\operatorname*{Ker}B\right)  =1.
\]
This completes our proof of Proposition \ref{prop.jnf.jcell.multiplicity}.
\end{proof}

\subsection{Jordan canonical form: the theorem}

Let us now build larger matrices out of Jordan cells:

\begin{definition}
\label{def.jnf.jmat}Let $\mathbb{F}$ be a field. A \emph{Jordan matrix} means
a block-diagonal matrix whose diagonal blocks are Jordan cells. In other
words, it is a matrix of the form%
\[
\left(
\begin{array}
[c]{cccc}%
J_{n_{1}}\left(  \lambda_{1}\right)  & 0 & \cdots & 0\\
0 & J_{n_{2}}\left(  \lambda_{2}\right)  & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & J_{n_{k}}\left(  \lambda_{k}\right)
\end{array}
\right)  ,
\]
where $n_{1},n_{2},\ldots,n_{k}$ are positive integers and $\lambda
_{1},\lambda_{2},\ldots,\lambda_{k}$ are scalars in $\mathbb{F}$ (not
necessarily distinct, but not necessarily equal either).
\end{definition}

We note that any Jordan matrix is upper-triangular. \medskip

We claim the following:\footnote{Recall that \textquotedblleft$A\sim
B$\textquotedblright\ means that the matrix $A$ is similar to the matrix $B$.}

\begin{theorem}
[Jordan canonical form theorem]\label{thm.jnf.jnf}Let $A\in\mathbb{C}^{n\times
n}$ be an $n\times n$-matrix over $\mathbb{C}$. Then: \medskip

\textbf{(a)} There exists a Jordan matrix $J$ such that $A\sim J$. \medskip

\textbf{(b)} This Jordan matrix $J$ is unique up to the order of the diagonal blocks.
\end{theorem}

This theorem is useful partly (but not only) because it allows to reduce
questions about general square matrices to questions about Jordan matrices.
And the latter can usually further be reduced to questions about Jordan cells,
because a block-diagonal matrix \textquotedblleft behaves like its diagonal
blocks are separate\textquotedblright\ (see, e.g., the discussion before
Proposition \ref{prop.blockmatrix.nullity-diag}).

Note that we cannot hope for the matrix $J$ in Theorem \ref{thm.jnf.jnf} to be
fully unique, unless it has only one diagonal block (i.e., unless $k=1$).
Indeed, Proposition \ref{prop.blockmatrix.mult-uxv} shows that if we permute
the diagonal blocks $J_{n_{1}}\left(  \lambda_{1}\right)  ,J_{n_{2}}\left(
\lambda_{2}\right)  ,\ldots,J_{n_{k}}\left(  \lambda_{k}\right)  $, then the
matrix stays similar to $A$. Thus, the order of these diagonal blocks can be
chosen arbitrary.

\begin{definition}
\label{def.jnf.jnf}Let $A$ be an $n\times n$-matrix over $\mathbb{C}$. Theorem
\ref{thm.jnf.jnf} \textbf{(a)} says that there exists a Jordan matrix $J$ such
that $A\sim J$. Such a matrix $J$ is called a \emph{Jordan canonical form} of
$A$ (or a \emph{Jordan normal form} of $A$).

We often use the definite article (\textquotedblleft the Jordan canonical form
of $A$\textquotedblright), because Theorem \ref{thm.jnf.jnf} \textbf{(b)} says
that $J$ is \textquotedblleft more or less unique\textquotedblright. (Strictly
speaking, of course, it is not entirely appropriate.)

The diagonal blocks $J_{n_{1}}\left(  \lambda_{1}\right)  ,J_{n_{2}}\left(
\lambda_{2}\right)  ,\ldots,J_{n_{k}}\left(  \lambda_{k}\right)  $ of $J$ are
called the \emph{Jordan blocks} (or \emph{Jordan cells}) of $A$.

We often abbreviate \textquotedblleft Jordan canonical form\textquotedblright%
\ as \textquotedblleft JCF\textquotedblright.
\end{definition}

\begin{example}
A Jordan canonical form of $\left(
\begin{array}
[c]{ccc}%
1 & 2 & 3\\
0 & 2 & 5\\
0 & 0 & 1
\end{array}
\right)  $ is $\left(
\begin{array}
[c]{ccc}%
2 & 0 & 0\\
0 & 1 & 1\\
0 & 0 & 1
\end{array}
\right)  $. Indeed, $\left(
\begin{array}
[c]{ccc}%
2 & 0 & 0\\
0 & 1 & 1\\
0 & 0 & 1
\end{array}
\right)  $ is a Jordan matrix (it can be written as $\left(
\begin{array}
[c]{cc}%
J_{1}\left(  2\right)  & 0\\
0 & J_{2}\left(  1\right)
\end{array}
\right)  $) and it can be checked that%
\[
\left(
\begin{array}
[c]{ccc}%
1 & 2 & 3\\
0 & 2 & 5\\
0 & 0 & 1
\end{array}
\right)  \sim\left(
\begin{array}
[c]{ccc}%
2 & 0 & 0\\
0 & 1 & 1\\
0 & 0 & 1
\end{array}
\right)  .
\]
We can swap the two Jordan cells in this Jordan matrix, and obtain another
Jordan canonical form of the same matrix: $\left(
\begin{array}
[c]{ccc}%
1 & 1 & 0\\
0 & 1 & 0\\
0 & 0 & 2
\end{array}
\right)  $.
\end{example}

\begin{example}
If $A=\left(
\begin{array}
[c]{cc}%
0 & -1\\
1 & 0
\end{array}
\right)  $, then a Jordan canonical form of $A$ is $\left(
\begin{array}
[c]{cc}%
-i & 0\\
0 & i
\end{array}
\right)  =\left(
\begin{array}
[c]{cc}%
J_{1}\left(  -i\right)  & 0\\
0 & J_{1}\left(  i\right)
\end{array}
\right)  $, where $i=\sqrt{-1}\in\mathbb{C}$. Note that this Jordan canonical
form has imaginary entries, despite all entries of $A$ being real. This is
unavoidable when $A$ has non-real eigenvalues.
\end{example}

\begin{example}
If $D$ is a diagonal matrix, then $D$ itself is a Jordan canonical form of
$D$. Indeed, each diagonal entry $\lambda$ of $D$ can be viewed as a Jordan
cell of size $1$ (namely, $J_{1}\left(  \lambda\right)  $), so that $D$ is a
Jordan matrix.
\end{example}

\subsection{Jordan canonical form: proof of uniqueness}

We shall approach the proof of Theorem \ref{thm.jnf.jnf} slowly\footnote{See
\cite[Chapter VII, \S 5, section 4]{Bourba03}, \cite[Theorem 31.17]{GalQua20},
\cite[Chapter Five, Section IV, Theorem 2.8]{Heffer20},
\cite[\S 8.10--\S 8.11]{Loehr14}, \cite[\S 4.6]{OmClVi11}, \cite[\S 12.2]%
{Prasolov}, \cite[\S 4.3]{Shapir15}, \cite[\S 2.4]{Taylor20}, \cite[Chapter 9,
Theorem 5.1]{Treil15} and \cite[Theorem 4.4.1]{Woerde16} for other proofs (at
least proofs of Theorem \ref{thm.jnf.jnf} \textbf{(a)}, which is the harder
part of Theorem \ref{thm.jnf.jnf}).}, making sure to record all auxiliary
results obtained on the way (as they are themselves rather useful). We first
try to explore how much of the structure of the Jordan normal form $J$ can be
read off the matrix $A$. This will lead us to the proof of the uniqueness part
(i.e., part \textbf{(b)}) of Theorem \ref{thm.jnf.jnf}.

We start with an example:

\begin{example}
\label{exa.jnf.unique.exa1}Let $A\in\mathbb{C}^{7\times7}$. Suppose that
$A\sim J$ with%
\[
J=\left(
\begin{array}
[c]{ccc}%
J_{2}\left(  8\right)  & 0 & 0\\
0 & J_{3}\left(  8\right)  & 0\\
0 & 0 & J_{2}\left(  9\right)
\end{array}
\right)  =\left(
\begin{array}
[c]{ccccccc}%
8 & 1 & 0 & 0 & 0 & 0 & 0\\
0 & 8 & 0 & 0 & 0 & 0 & 0\\
0 & 0 & 8 & 1 & 0 & 0 & 0\\
0 & 0 & 0 & 8 & 1 & 0 & 0\\
0 & 0 & 0 & 0 & 8 & 0 & 0\\
0 & 0 & 0 & 0 & 0 & 9 & 1\\
0 & 0 & 0 & 0 & 0 & 0 & 9
\end{array}
\right)  .
\]
How are the structural elements of this matrix $J$ (that is, the diagonal
entries $8$ and $9$ and the sizes $2,3,2$ of the diagonal blocks) reflected in
the matrix $A$ ?

First, the matrix $J$ is a Jordan matrix, and thus upper-triangular. Hence,
Proposition \ref{prop.schurtri.similar.T-diag} (applied to $T=J$) shows that
the diagonal entries of $J$ are the eigenvalues of $A$ (with their algebraic
multiplicities). Thus, the eigenvalues of $A$ are $8$ and $9$, with respective
algebraic multiplicities $5$ and $2$.

Now, what about the geometric multiplicities? That is, what are $\dim\left(
\operatorname*{Ker}\left(  A-8I_{7}\right)  \right)  $ and $\dim\left(
\operatorname*{Ker}\left(  A-9I_{7}\right)  \right)  $ ?

From $A\sim J$, we obtain $A-8I_{7}\sim J-8I_{7}$ (by Proposition
\ref{prop.schurtri.similar.same} \textbf{(h)}, applied to $B=J$ and
$\lambda=8$ and $n=7$). Thus, Proposition \ref{prop.schurtri.similar.same}
\textbf{(b)} (applied to $A-8I_{7}$, $J-8I_{7}$ and $7$ instead of $A$, $B$
and $8$) yields that the matrices $A-8I_{7}$ and $J-8I_{7}$ have the same
nullity. In other words,%
\begin{align*}
\dim\left(  \operatorname*{Ker}\left(  A-8I_{7}\right)  \right)   &
=\dim\left(  \operatorname*{Ker}\left(  J-8I_{7}\right)  \right) \\
&  =\dim\left(  \operatorname*{Ker}\left(
\begin{array}
[c]{ccc}%
J_{2}\left(  8\right)  -8I_{2} & 0 & 0\\
0 & J_{3}\left(  8\right)  -8I_{3} & 0\\
0 & 0 & J_{2}\left(  9\right)  -8I_{2}%
\end{array}
\right)  \right)
\end{align*}
(since%
\begin{align*}
J-8I_{7}  &  =\left(
\begin{array}
[c]{ccc}%
J_{2}\left(  8\right)  & 0 & 0\\
0 & J_{3}\left(  8\right)  & 0\\
0 & 0 & J_{2}\left(  9\right)
\end{array}
\right)  -8I_{7}\\
&  =\left(
\begin{array}
[c]{ccc}%
J_{2}\left(  8\right)  -8I_{2} & 0 & 0\\
0 & J_{3}\left(  8\right)  -8I_{3} & 0\\
0 & 0 & J_{2}\left(  9\right)  -8I_{2}%
\end{array}
\right)
\end{align*}
). Thus,%
\begin{align*}
&  \dim\left(  \operatorname*{Ker}\left(  A-8I_{7}\right)  \right) \\
&  =\dim\left(  \operatorname*{Ker}\left(
\begin{array}
[c]{ccc}%
J_{2}\left(  8\right)  -8I_{2} & 0 & 0\\
0 & J_{3}\left(  8\right)  -8I_{3} & 0\\
0 & 0 & J_{2}\left(  9\right)  -8I_{2}%
\end{array}
\right)  \right) \\
&  =\dim\left(  \operatorname*{Ker}\left(  J_{2}\left(  8\right)
-8I_{2}\right)  \right)  +\dim\left(  \operatorname*{Ker}\left(  J_{3}\left(
8\right)  -8I_{3}\right)  \right)  +\dim\left(  \operatorname*{Ker}\left(
J_{2}\left(  9\right)  -8I_{2}\right)  \right)
\end{align*}
(by Proposition \ref{prop.blockmatrix.nullity-diag}). Now, let us find the
three dimensions on the right hand side.

Proposition \ref{prop.jnf.jcell-lambda} yields $J_{2}\left(  8\right)
=J_{2}\left(  0\right)  +2I_{2}$, so that $J_{2}\left(  8\right)
-8I_{2}=J_{2}\left(  0\right)  $. Hence,%
\[
\dim\left(  \operatorname*{Ker}\left(  J_{2}\left(  8\right)  -8I_{2}\right)
\right)  =\dim\left(  \operatorname*{Ker}\left(  J_{2}\left(  0\right)
\right)  \right)  =\dim\left(  \operatorname*{Ker}\left(  \left(  J_{2}\left(
0\right)  \right)  ^{1}\right)  \right)  =1
\]
(by Proposition \ref{prop.jnf.jcell0-powers} \textbf{(c)}, applied to $m=2$
and $p=1$, because the matrix $J_{2}\left(  0\right)  $ is what is called $B$
in this proposition). Similarly, $\dim\left(  \operatorname*{Ker}\left(
J_{3}\left(  8\right)  -8I_{3}\right)  \right)  =1$. On the other hand, the
matrix $J_{2}\left(  9\right)  -8I_{2}=\left(
\begin{array}
[c]{cc}%
1 & 1\\
0 & 1
\end{array}
\right)  $ is an upper-triangular matrix with $1$'s on its main diagonal;
thus, its determinant is $1\cdot1=1\neq0$, so that it is nonsingular. Hence,
$\operatorname*{Ker}\left(  J_{2}\left(  9\right)  -8I_{2}\right)  =0$, so
that $\dim\left(  \operatorname*{Ker}\left(  J_{2}\left(  9\right)
-8I_{2}\right)  \right)  =0$. Thus, our above computation of $\dim\left(
\operatorname*{Ker}\left(  A-8I_{7}\right)  \right)  $ becomes%
\begin{align*}
&  \dim\left(  \operatorname*{Ker}\left(  A-8I_{7}\right)  \right) \\
&  =\underbrace{\dim\left(  \operatorname*{Ker}\left(  J_{2}\left(  8\right)
-8I_{2}\right)  \right)  }_{=1}+\underbrace{\dim\left(  \operatorname*{Ker}%
\left(  J_{3}\left(  8\right)  -8I_{3}\right)  \right)  }_{=1}%
+\underbrace{\dim\left(  \operatorname*{Ker}\left(  J_{2}\left(  9\right)
-8I_{2}\right)  \right)  }_{=0}\\
&  =1+1+0=2.
\end{align*}
Looking back, we see that this comes from the fact that exactly $2$ of the
diagonal blocks in the Jordan canonical form $J$ are Jordan cells at
eigenvalue $8$.
\end{example}

Generalizing this reasoning, we obtain the following:

\begin{proposition}
\label{prop.jnf.unique.mults}Let $A$ be an $n\times n$-matrix, and let
$J=\left(
\begin{array}
[c]{cccc}%
J_{n_{1}}\left(  \lambda_{1}\right)  & 0 & \cdots & 0\\
0 & J_{n_{2}}\left(  \lambda_{2}\right)  & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & J_{n_{k}}\left(  \lambda_{k}\right)
\end{array}
\right)  $ be its Jordan canonical form. Then: \medskip

\textbf{(a)} We have $\sigma\left(  A\right)  =\left\{  \lambda_{1}%
,\lambda_{2},\ldots,\lambda_{k}\right\}  $. \medskip

\textbf{(b)} The geometric multiplicity of a number $\lambda\in\mathbb{C}$ as
an eigenvalue of $A$ is the number of Jordan cells of $A$ at eigenvalue
$\lambda$. In other words, it is the number of $i\in\left[  k\right]  $
satisfying $\lambda_{i}=\lambda$. \medskip

\textbf{(c)} The algebraic multiplicity of a number $\lambda\in\mathbb{C}$ as
an eigenvalue of $A$ is the \textbf{sum} of the sizes of all Jordan cells of
$A$ at eigenvalue $\lambda$. In other words, it is $\sum
\limits_{\substack{i\in\left[  k\right]  ;\\\lambda_{i}=\lambda}}n_{i}$.
\end{proposition}

\begin{proof}
TODO: Scribe?
\end{proof}

With some more effort, we can obtain a more precise result:

\begin{proposition}
\label{prop.jnf.unique.cellsizes}Let $A$ be an $n\times n$-matrix, and let
$J=\left(
\begin{array}
[c]{cccc}%
J_{n_{1}}\left(  \lambda_{1}\right)  & 0 & \cdots & 0\\
0 & J_{n_{2}}\left(  \lambda_{2}\right)  & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & J_{n_{k}}\left(  \lambda_{k}\right)
\end{array}
\right)  $ be its Jordan canonical form. Let $\lambda\in\mathbb{C}$. Let $p$
be a positive integer. Then,%
\begin{align*}
&  \left(  \text{the number of }i\in\left[  k\right]  \text{ such that
}\lambda_{i}=\lambda\text{ and }n_{i}\geq p\right) \\
&  =\dim\left(  \operatorname*{Ker}\left(  \left(  A-\lambda I_{n}\right)
^{p}\right)  \right)  -\dim\left(  \operatorname*{Ker}\left(  \left(
A-\lambda I_{n}\right)  ^{p-1}\right)  \right)  .
\end{align*}

\end{proposition}

\begin{proof}
We have $A\sim J$, so that $A-\lambda I_{n}\sim J-\lambda I_{n}$ (by
Proposition \ref{prop.schurtri.similar.same} \textbf{(h)}, applied to $B=J$),
and therefore $\left(  A-\lambda I_{n}\right)  ^{p}\sim\left(  J-\lambda
I_{n}\right)  ^{p}$ (by Proposition \ref{prop.schurtri.similar.same}
\textbf{(f)}, applied to $A-\lambda I_{n}$ and $B-\lambda I_{n}$ and $p$
instead of $A$, $B$ and $k$). Hence,%
\begin{equation}
\dim\left(  \operatorname*{Ker}\left(  \left(  A-\lambda I_{n}\right)
^{p}\right)  \right)  =\dim\left(  \operatorname*{Ker}\left(  \left(
J-\lambda I_{n}\right)  ^{p}\right)  \right)  .
\label{pf.prop.jnf.unique.cellsizes.dimKer1}%
\end{equation}


For each $i\in\left[  k\right]  $, we set
\begin{equation}
M_{i}:=J_{n_{i}}\left(  \lambda_{i}-\lambda\right)  .
\label{pf.prop.jnf.unique.cellsizes.Mi=}%
\end{equation}


However, for each $i\in\left[  k\right]  $, we have%
\begin{equation}
J_{n_{i}}\left(  \lambda_{i}\right)  -\lambda I_{n_{i}}=J_{n_{i}}\left(
\lambda_{i}-\lambda\right)  \label{pf.prop.jnf.unique.cellsizes.J-lam}%
\end{equation}
(because the two Jordan cells $J_{n_{i}}\left(  \lambda_{i}\right)  $ and
$J_{n_{i}}\left(  \lambda_{i}-\lambda\right)  $ differ only in their diagonal
entries, which are $\lambda_{i}$ in the former matrix and $\lambda_{i}%
-\lambda$ in the latter). Comparing this with
(\ref{pf.prop.jnf.unique.cellsizes.Mi=}), we obtain%
\begin{equation}
J_{n_{i}}\left(  \lambda_{i}\right)  -\lambda I_{n_{i}}=M_{i}
\label{pf.prop.jnf.unique.cellsizes.J-lam2}%
\end{equation}
for each $i\in\left[  k\right]  $.

Now, we have%
\begin{align*}
J  &  =\left(
\begin{array}
[c]{cccc}%
J_{n_{1}}\left(  \lambda_{1}\right)  & 0 & \cdots & 0\\
0 & J_{n_{2}}\left(  \lambda_{2}\right)  & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & J_{n_{k}}\left(  \lambda_{k}\right)
\end{array}
\right)  \ \ \ \ \ \ \ \ \ \ \text{and}\\
\lambda I_{n}  &  =\left(
\begin{array}
[c]{cccc}%
\lambda I_{n_{1}} & 0 & \cdots & 0\\
0 & \lambda I_{n_{2}} & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & \lambda I_{n_{k}}%
\end{array}
\right)  .
\end{align*}
Subtracting these two equalities from one another, we obtain%
\begin{align*}
J-\lambda I_{n}  &  =\left(
\begin{array}
[c]{cccc}%
J_{n_{1}}\left(  \lambda_{1}\right)  -\lambda I_{n_{1}} & 0 & \cdots & 0\\
0 & J_{n_{2}}\left(  \lambda_{2}\right)  -\lambda I_{n_{2}} & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & J_{n_{k}}\left(  \lambda_{k}\right)  -\lambda I_{n_{k}}%
\end{array}
\right) \\
&  =\left(
\begin{array}
[c]{cccc}%
M_{1} & 0 & \cdots & 0\\
0 & M_{2} & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & M_{k}%
\end{array}
\right)
\end{align*}
(by (\ref{pf.prop.jnf.unique.cellsizes.J-lam})). Hence,%
\[
\left(  J-\lambda I_{n}\right)  ^{p}=\left(
\begin{array}
[c]{cccc}%
M_{1} & 0 & \cdots & 0\\
0 & M_{2} & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & M_{k}%
\end{array}
\right)  ^{p}=\left(
\begin{array}
[c]{cccc}%
M_{1}^{p} & 0 & \cdots & 0\\
0 & M_{2}^{p} & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & M_{k}^{p}%
\end{array}
\right)
\]
(by Corollary \ref{cor.blockmatrix.powt-diag}). Thus,%
\begin{align}
&  \dim\left(  \operatorname*{Ker}\left(  \left(  J-\lambda I_{n}\right)
^{p}\right)  \right) \nonumber\\
&  =\dim\left(  \operatorname*{Ker}\left(
\begin{array}
[c]{cccc}%
M_{1}^{p} & 0 & \cdots & 0\\
0 & M_{2}^{p} & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & M_{k}^{p}%
\end{array}
\right)  \right) \nonumber\\
&  =\dim\left(  \operatorname*{Ker}\left(  M_{1}^{p}\right)  \right)
+\dim\left(  \operatorname*{Ker}\left(  M_{2}^{p}\right)  \right)
+\cdots+\dim\left(  \operatorname*{Ker}\left(  M_{k}^{p}\right)  \right)
\nonumber\\
&  \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \left(  \text{by Proposition
\ref{prop.blockmatrix.nullity-diag}}\right) \nonumber\\
&  =\sum_{i=1}^{k}\dim\left(  \operatorname*{Ker}\left(  M_{i}^{p}\right)
\right)  \label{pf.prop.jnf.unique.cellsizes.1}%
\end{align}


Now, fix an $i\in\left[  k\right]  $ satisfying $\lambda_{i}\neq\lambda$.
Thus, $\lambda_{i}-\lambda\neq0$. The matrix $J_{n_{i}}\left(  \lambda
_{i}-\lambda\right)  $ is upper-triangular, and its diagonal entries are all
$\lambda_{i}-\lambda$. Hence, its determinant is $\det\left(  J_{n_{i}}\left(
\lambda_{i}-\lambda\right)  \right)  =\left(  \lambda_{i}-\lambda\right)
^{n_{i}}\neq0$ (since $\lambda_{i}-\lambda\neq0$). Therefore, this matrix
$J_{n_{i}}\left(  \lambda_{i}-\lambda\right)  $ is invertible. In other words,
the matrix $M_{i}$ is invertible (since $M_{i}=J_{n_{i}}\left(  \lambda
_{i}-\lambda\right)  $). Hence, its $p$-th power $M_{i}^{p}$ is also
invertible, and therefore has nullity $0$. In other words, $\dim\left(
\operatorname*{Ker}\left(  M_{i}^{p}\right)  \right)  =0$.

Forget that we fixed $i$. We thus have shown that if $i\in\left[  k\right]  $
satisfies $\lambda_{i}\neq\lambda$, then
\begin{equation}
\dim\left(  \operatorname*{Ker}\left(  M_{i}^{p}\right)  \right)  =0\text{.}
\label{pf.prop.jnf.unique.cellsizes.nullity0}%
\end{equation}
Hence, (\ref{pf.prop.jnf.unique.cellsizes.1}) becomes%
\begin{align}
&  \dim\left(  \operatorname*{Ker}\left(  \left(  J-\lambda I_{n}\right)
^{p}\right)  \right) \nonumber\\
&  =\sum_{i=1}^{k}\dim\left(  \operatorname*{Ker}\left(  M_{i}^{p}\right)
\right) \nonumber\\
&  =\sum_{\substack{i\in\left[  k\right]  ;\\\lambda_{i}\neq\lambda
}}\underbrace{\dim\left(  \operatorname*{Ker}\left(  M_{i}^{p}\right)
\right)  }_{\substack{=0\\\text{(by
(\ref{pf.prop.jnf.unique.cellsizes.nullity0}))}}}+\sum_{\substack{i\in\left[
k\right]  ;\\\lambda_{i}=\lambda}}\dim\left(  \operatorname*{Ker}\left(
M_{i}^{p}\right)  \right) \nonumber\\
&  \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \left(  \text{since each }%
i\in\left[  k\right]  \text{ satisfies either }\lambda_{i}\neq\lambda\text{ or
}\lambda_{i}=\lambda\right) \nonumber\\
&  =\sum_{\substack{i\in\left[  k\right]  ;\\\lambda_{i}=\lambda}}\dim\left(
\operatorname*{Ker}\left(  M_{i}^{p}\right)  \right)  .
\label{pf.prop.jnf.unique.cellsizes.2}%
\end{align}


Now, let us fix some $i\in\left[  k\right]  $ satisfying $\lambda_{i}=\lambda
$. Then, $\lambda_{i}-\lambda=0$. Hence, $J_{n_{i}}\left(  \lambda_{i}%
-\lambda\right)  =J_{n_{i}}\left(  0\right)  =\left(
\begin{array}
[c]{ccccc}%
0 & 1 & 0 & \cdots & 0\\
0 & 0 & 1 & \cdots & 0\\
0 & 0 & 0 & \cdots & 0\\
\vdots & \vdots & \vdots & \ddots & \vdots\\
0 & 0 & 0 & \cdots & 0
\end{array}
\right)  $. Let us denote this matrix $J_{n_{i}}\left(  0\right)  $ by $B$.
Then, Proposition \ref{prop.jnf.jcell0-powers} \textbf{(c)} (applied to
$m=n_{i}$) shows that we have
\[
\dim\left(  \operatorname*{Ker}\left(  B^{p}\right)  \right)
=p\ \ \ \ \ \ \ \ \ \ \text{if }p\leq n_{i}.
\]
On the other hand, Proposition \ref{prop.jnf.jcell0-powers} \textbf{(c)}
(applied to $m=n_{i}$) shows that we have
\[
\dim\left(  \operatorname*{Ker}\left(  B^{p}\right)  \right)  =n_{i}%
\ \ \ \ \ \ \ \ \ \ \text{if }p>n_{i}.
\]
Combining these two equalities, we obtain
\[
\dim\left(  \operatorname*{Ker}\left(  B^{p}\right)  \right)  =%
\begin{cases}
p, & \text{if }p\leq n_{i};\\
n_{i}, & \text{if }p\geq n_{i}.
\end{cases}
\]
\footnote{Note that the two cases $p\leq n_{i}$ and $p\geq n_{i}$ are not
mutually exclusive: They overlap when $p=n_{i}$. (But the answers in this case
are identical.)} In other words,%
\begin{equation}
\dim\left(  \operatorname*{Ker}\left(  M_{i}^{p}\right)  \right)  =%
\begin{cases}
p, & \text{if }p\leq n_{i};\\
n_{i}, & \text{if }p\geq n_{i}%
\end{cases}
\label{pf.prop.jnf.unique.cellsizes.nullity1}%
\end{equation}
(since $B=J_{n_{i}}\left(  0\right)  =J_{n_{i}}\left(  \lambda_{i}%
-\lambda\right)  =M_{i}$ (by (\ref{pf.prop.jnf.unique.cellsizes.Mi=}))).

Now, forget that we fixed $i$. We thus have proved
(\ref{pf.prop.jnf.unique.cellsizes.nullity0}) for each $i\in\left[  k\right]
$ satisfying $\lambda_{i}=\lambda$. Therefore,
(\ref{pf.prop.jnf.unique.cellsizes.2}) becomes%
\[
\dim\left(  \operatorname*{Ker}\left(  \left(  J-\lambda I_{n}\right)
^{p}\right)  \right)  =\sum_{\substack{i\in\left[  k\right]  ;\\\lambda
_{i}=\lambda}}\underbrace{\dim\left(  \operatorname*{Ker}\left(  M_{i}%
^{p}\right)  \right)  }_{\substack{=%
\begin{cases}
p, & \text{if }p\leq n_{i};\\
n_{i}, & \text{if }p\geq n_{i}%
\end{cases}
\\\text{(by (\ref{pf.prop.jnf.unique.cellsizes.nullity1}))}}}=\sum
_{\substack{i\in\left[  k\right]  ;\\\lambda_{i}=\lambda}}%
\begin{cases}
p, & \text{if }p\leq n_{i};\\
n_{i}, & \text{if }p\geq n_{i}.
\end{cases}
\]
Thus, (\ref{pf.prop.jnf.unique.cellsizes.dimKer1}) becomes%
\[
\dim\left(  \operatorname*{Ker}\left(  \left(  A-\lambda I_{n}\right)
^{p}\right)  \right)  =\dim\left(  \operatorname*{Ker}\left(  \left(
J-\lambda I_{n}\right)  ^{p}\right)  \right)  =\sum_{\substack{i\in\left[
k\right]  ;\\\lambda_{i}=\lambda}}%
\begin{cases}
p, & \text{if }p\leq n_{i};\\
n_{i}, & \text{if }p>n_{i}.
\end{cases}
\]
However, we can also apply the same argument to $p-1$ instead of $p$ (since
$p-1\in\mathbb{N}$). Thus, we obtain%
\[
\dim\left(  \operatorname*{Ker}\left(  \left(  A-\lambda I_{n}\right)
^{p-1}\right)  \right)  =\sum_{\substack{i\in\left[  k\right]  ;\\\lambda
_{i}=\lambda}}%
\begin{cases}
p-1, & \text{if }p-1\leq n_{i};\\
n_{i}, & \text{if }p-1\geq n_{i}.
\end{cases}
\]
Subtracting these two equalities, we obtain%
\begin{align*}
&  \dim\left(  \operatorname*{Ker}\left(  \left(  A-\lambda I_{n}\right)
^{p}\right)  \right)  -\dim\left(  \operatorname*{Ker}\left(  \left(
A-\lambda I_{n}\right)  ^{p-1}\right)  \right) \\
&  =\sum_{\substack{i\in\left[  k\right]  ;\\\lambda_{i}=\lambda}}%
\begin{cases}
p, & \text{if }p\leq n_{i};\\
n_{i}, & \text{if }p>n_{i}%
\end{cases}
-\sum_{\substack{i\in\left[  k\right]  ;\\\lambda_{i}=\lambda}}%
\begin{cases}
p-1, & \text{if }p-1\leq n_{i};\\
n_{i}, & \text{if }p-1\geq n_{i}%
\end{cases}
\\
&  =\sum_{\substack{i\in\left[  k\right]  ;\\\lambda_{i}=\lambda
}}\underbrace{\left(
\begin{cases}
p, & \text{if }p\leq n_{i};\\
n_{i}, & \text{if }p>n_{i}%
\end{cases}
-%
\begin{cases}
p-1, & \text{if }p-1\leq n_{i};\\
n_{i}, & \text{if }p-1\geq n_{i}%
\end{cases}
\right)  }_{\substack{=%
\begin{cases}
1, & \text{if }p\leq n_{i};\\
0, & \text{if }p>n_{i}%
\end{cases}
\\\text{(this can be directly checked in each}\\\text{of the two cases }p\leq
n_{i}\text{ and }p>n_{i}\text{)}}}\\
&  =\sum_{\substack{i\in\left[  k\right]  ;\\\lambda_{i}=\lambda}}%
\begin{cases}
1, & \text{if }p\leq n_{i};\\
0, & \text{if }p>n_{i}%
\end{cases}
=\sum_{\substack{i\in\left[  k\right]  ;\\\lambda_{i}=\lambda}}%
\begin{cases}
1, & \text{if }n_{i}\geq p;\\
0, & \text{if }n_{i}<p
\end{cases}
\\
&  \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \left(
\begin{array}
[c]{c}%
\text{since the condition }p\leq n_{i}\text{ is equivalent to }n_{i}\geq
p\text{,}\\
\text{and since the condition }p>n_{i}\text{ is equivalent to }n_{i}<p
\end{array}
\right) \\
&  =\left(  \text{the number of }i\in\left[  k\right]  \text{ such that
}\lambda_{i}=\lambda\text{ and }n_{i}\geq p\right)
\end{align*}
(because the sum has an addend equal to $1$ for each $i\in\left[  k\right]  $
satisfying $\lambda_{i}=\lambda$ and $n_{i}\geq p$, whereas all remaining
addends of this sum are $0$). Thus, Proposition
\ref{prop.jnf.unique.cellsizes} is proved.
\end{proof}

\begin{corollary}
\label{cor.jnf.unique.cellsizes=}Let $A$ be an $n\times n$-matrix, and let
$J=\left(
\begin{array}
[c]{cccc}%
J_{n_{1}}\left(  \lambda_{1}\right)  & 0 & \cdots & 0\\
0 & J_{n_{2}}\left(  \lambda_{2}\right)  & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & J_{n_{k}}\left(  \lambda_{k}\right)
\end{array}
\right)  $ be its Jordan canonical form. Let $\lambda\in\mathbb{C}$. Let $p$
be a positive integer. Then,%
\begin{align*}
&  \left(  \text{the number of }i\in\left[  k\right]  \text{ such that
}\lambda_{i}=\lambda\text{ and }n_{i}=p\right) \\
&  =2\dim\left(  \operatorname*{Ker}\left(  \left(  A-\lambda I_{n}\right)
^{p}\right)  \right) \\
&  \ \ \ \ \ \ \ \ \ \ -\dim\left(  \operatorname*{Ker}\left(  \left(
A-\lambda I_{n}\right)  ^{p-1}\right)  \right)  -\dim\left(
\operatorname*{Ker}\left(  \left(  A-\lambda I_{n}\right)  ^{p+1}\right)
\right)  .
\end{align*}

\end{corollary}

\begin{proof}
An integer $z$ equals $p$ if and only if it satisfies $z\geq p$ but does not
satisfy $z\geq p+1$. Hence, an $i\in\left[  k\right]  $ satisfies $n_{i}=p$ if
and only if it satisfies $n_{i}\geq p$ but does not satisfy $n_{i}\geq p+1$.
Thus,%
\begin{align*}
&  \left(  \text{the number of }i\in\left[  k\right]  \text{ such that
}\lambda_{i}=\lambda\text{ and }n_{i}=p\right) \\
&  =\underbrace{\left(  \text{the number of }i\in\left[  k\right]  \text{ such
that }\lambda_{i}=\lambda\text{ and }n_{i}\geq p\right)  }_{\substack{=\dim
\left(  \operatorname*{Ker}\left(  \left(  A-\lambda I_{n}\right)
^{p}\right)  \right)  -\dim\left(  \operatorname*{Ker}\left(  \left(
A-\lambda I_{n}\right)  ^{p-1}\right)  \right)  \\\text{(by Proposition
\ref{prop.jnf.unique.cellsizes})}}}\\
&  \ \ \ \ \ \ \ \ \ \ -\underbrace{\left(  \text{the number of }i\in\left[
k\right]  \text{ such that }\lambda_{i}=\lambda\text{ and }n_{i}\geq
p+1\right)  }_{\substack{=\dim\left(  \operatorname*{Ker}\left(  \left(
A-\lambda I_{n}\right)  ^{p+1}\right)  \right)  -\dim\left(
\operatorname*{Ker}\left(  \left(  A-\lambda I_{n}\right)  ^{p}\right)
\right)  \\\text{(by Proposition \ref{prop.jnf.unique.cellsizes}%
,}\\\text{applied to }p+1\text{ instead of }p\text{)}}}\\
&  \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \left(  \text{because any }%
i\in\left[  k\right]  \text{ satisfying }n_{i}\geq p+1\text{ must also satisfy
}n_{i}\geq p\right) \\
&  =\left(  \dim\left(  \operatorname*{Ker}\left(  \left(  A-\lambda
I_{n}\right)  ^{p}\right)  \right)  -\dim\left(  \operatorname*{Ker}\left(
\left(  A-\lambda I_{n}\right)  ^{p-1}\right)  \right)  \right) \\
&  \ \ \ \ \ \ \ \ \ \ -\left(  \dim\left(  \operatorname*{Ker}\left(  \left(
A-\lambda I_{n}\right)  ^{p+1}\right)  \right)  -\dim\left(
\operatorname*{Ker}\left(  \left(  A-\lambda I_{n}\right)  ^{p}\right)
\right)  \right) \\
&  =2\dim\left(  \operatorname*{Ker}\left(  \left(  A-\lambda I_{n}\right)
^{p}\right)  \right)  -\dim\left(  \operatorname*{Ker}\left(  \left(
A-\lambda I_{n}\right)  ^{p-1}\right)  \right)  -\dim\left(
\operatorname*{Ker}\left(  \left(  A-\lambda I_{n}\right)  ^{p+1}\right)
\right)  .
\end{align*}
This proves Corollary \ref{cor.jnf.unique.cellsizes=}.
\end{proof}

Now, we can easily prove Theorem \ref{thm.jnf.jnf} \textbf{(b)}:

\begin{proof}
[Proof of Theorem \ref{thm.jnf.jnf} \textbf{(b)}.]Let $A\in\mathbb{C}^{n\times
n}$ be an $n\times n$-matrix. Let $J$ be a Jordan matrix such that $A\sim J$.
Write $J$ as $J=\left(
\begin{array}
[c]{cccc}%
J_{n_{1}}\left(  \lambda_{1}\right)  & 0 & \cdots & 0\\
0 & J_{n_{2}}\left(  \lambda_{2}\right)  & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & J_{n_{k}}\left(  \lambda_{k}\right)
\end{array}
\right)  $ as in Corollary \ref{cor.jnf.unique.cellsizes=}. Then, the Jordan
blocks of $J$ are $J_{n_{1}}\left(  \lambda_{1}\right)  ,J_{n_{2}}\left(
\lambda_{2}\right)  ,\ldots,J_{n_{k}}\left(  \lambda_{k}\right)  $. Hence, for
any $\lambda\in\mathbb{C}$ and any positive integer $p$, we have%
\begin{align*}
&  \left(  \text{the number of Jordan blocks of }J\text{ of size }p\text{ at
eigenvalue }\lambda\right) \\
&  =\left(  \text{the number of }i\in\left[  k\right]  \text{ such that
}\lambda_{i}=\lambda\text{ and }n_{i}=p\right) \\
&  =2\dim\left(  \operatorname*{Ker}\left(  \left(  A-\lambda I_{n}\right)
^{p}\right)  \right)  -\dim\left(  \operatorname*{Ker}\left(  \left(
A-\lambda I_{n}\right)  ^{p-1}\right)  \right)  -\dim\left(
\operatorname*{Ker}\left(  \left(  A-\lambda I_{n}\right)  ^{p+1}\right)
\right)  .
\end{align*}
Therefore, this number is uniquely determined by $A$, $\lambda$ and $p$.
Hence, the whole structure of $J$ is determined uniquely by $A$, up to the
order of the Jordan blocks. This proves Theorem \ref{thm.jnf.jnf} \textbf{(b)}.
\end{proof}

\begin{example}
Let $A$ be an $8\times8$-matrix. Assume that we know that the Jordan canonical
form of $A$ has

\begin{itemize}
\item $1$ Jordan block of size $1$ at eigenvalue $17$;

\item $2$ Jordan blocks of size $1$ at eigenvalue $35$;

\item $1$ Jordan block of size $2$ at eigenvalue $35$;

\item $1$ Jordan block of size $3$ at eigenvalue $59$;

\item no Jordan blocks of other sizes or at other eigenvalues.
\end{itemize}

Then, the Jordan canonical form of $A$ must be the block-diagonal matrix%
\[
\left(
\begin{array}
[c]{ccccc}%
J_{1}\left(  17\right)  & 0 & 0 & 0 & 0\\
0 & J_{2}\left(  35\right)  & 0 & 0 & 0\\
0 & 0 & J_{1}\left(  35\right)  & 0 & 0\\
0 & 0 & 0 & J_{1}\left(  35\right)  & 0\\
0 & 0 & 0 & 0 & J_{3}\left(  59\right)
\end{array}
\right)
\]
or one that is obtained from it by permuting the diagonal blocks.
\end{example}

Let us now approach the existence of the Jordan canonical form (Theorem
\ref{thm.jnf.jnf} \textbf{(a)}).

\subsection{\label{sect.jnf.exist}Jordan canonical form: proof of existence}

We will prove the existence of the Jordan canonical form in several steps,
each of which will bring our matrix $A$ \textquotedblleft
closer\textquotedblright\ to a Jordan matrix. Along the way, we will obtain
several results of independent interest.

\subsubsection{Step 1: Schur triangularization}

Our first step will be an application of Schur triangularization. As we
recall, the \textquotedblleft weak\textquotedblright\ Schur triangularization
theorem (Theorem \ref{thm.schurtri.schurtri}) tells us that if $A\in
\mathbb{C}^{n\times n}$ is an $n\times n$-matrix, then $A$ is unitarily
similar to an upper-triangular matrix $T$. The diagonal entries of the latter
matrix $T$ will be the eigenvalues of $A$ in some order (by Proposition
\ref{prop.schurtri.schurtri.T-diag}). However, let us now be a bit pickier. To
wit, we now want the triangular matrix $T$ to have the property that equal
eigenvalues come in contiguous runs on the main diagonal. In other words, we
want $T$ to have the property that if two diagonal entries of $T$ are equal,
then all the diagonal entries between them are also equal to them. For
instance, if the eigenvalues of $A$ are $1,1,2,2$, we don't want\footnote{In
the following equation, an empty cell of the matrix must be filled with a $0$,
whereas a \textquotedblleft$\ast$\textquotedblright\ in a cell means that any
arbitrary value can go into that cell.}%
\[
T=\left(
\begin{array}
[c]{cccc}%
1 & \ast & \ast & \ast\\
& 2 & \ast & \ast\\
&  & 1 & \ast\\
&  &  & 2
\end{array}
\right)  ,
\]
but instead we want%
\[
T=\left(
\begin{array}
[c]{cccc}%
1 & \ast & \ast & \ast\\
& 1 & \ast & \ast\\
&  & 2 & \ast\\
&  &  & 2
\end{array}
\right)  .
\]
Fortunately, we can achieve this using Theorem \ref{thm.jnf.schurtri-diag}:
Indeed, if we list the eigenvalues of $A$ as $\left(  \lambda_{1},\lambda
_{2},\ldots,\lambda_{n}\right)  $ in such a way that equal eigenvalues come in
contiguous runs in this list, then Theorem \ref{thm.jnf.schurtri-diag} shows
that we can find an upper-triangular matrix $T$ that is unitarily similar to
$A$ and that has diagonal entries $\lambda_{1},\lambda_{2},\ldots,\lambda_{n}$
in this order. This matrix $T$ is what we want.

\subsubsection{Step 2: Separating distinct eigenvalues}

Theorem \ref{thm.jnf.schurtri-diag} brings any $n\times n$-matrix
$A\in\mathbb{C}^{n\times n}$ to a certain simplified form (upper-triangular
with eigenvalues placed contiguously on the diagonal) that is not yet a Jordan
canonical form, but already has some of its aspects. We will now transform it
further to get a bit closer to a Jordan canonical form. To wit, we will get
rid of some of the entries above the diagonal (or, to be more precise, we will
turn them into $0$). Let us demonstrate this on an example:

\begin{example}
\label{exa.jnf.step2}Let $a,b,c,\ldots,p\in\mathbb{C}$ be any numbers. We
shall now show that%
\begin{equation}
\left(
\begin{array}
[c]{cccccc}%
1 & a & b & c & d & e\\
& 1 & f & g & h & i\\
&  & 2 & j & k & \ell\\
&  &  & 2 & m & n\\
&  &  &  & 2 & p\\
&  &  &  &  & 3
\end{array}
\right)  \sim\left(
\begin{array}
[c]{cccccc}%
1 & a &  &  &  & \\
& 1 &  &  &  & \\
&  & 2 & j & k & \\
&  &  & 2 & m & \\
&  &  &  & 2 & \\
&  &  &  &  & 3
\end{array}
\right)  \label{eq.exa.jnf.step2.1}%
\end{equation}
(where the entries in the empty cells are understood to be $0$s).

Indeed, the triangular matrices $\left(
\begin{array}
[c]{cc}%
1 & a\\
& 1
\end{array}
\right)  $ and $\left(
\begin{array}
[c]{cccc}%
2 & j & k & \ell\\
& 2 & m & n\\
&  & 2 & p\\
&  &  & 3
\end{array}
\right)  $ have disjoint spectra (i.e., they have no eigenvalues in common),
because their diagonals have no entries in common. So, by Corollary
\ref{cor.schurtri.block-sim}, we have%
\begin{equation}
\left(
\begin{array}
[c]{cccccc}%
1 & a & b & c & d & e\\
& 1 & f & g & h & i\\
&  & 2 & j & k & \ell\\
&  &  & 2 & m & n\\
&  &  &  & 2 & p\\
&  &  &  &  & 3
\end{array}
\right)  \sim\left(
\begin{array}
[c]{cccccc}%
1 & a &  &  &  & \\
& 1 &  &  &  & \\
&  & 2 & j & k & \ell\\
&  &  & 2 & m & n\\
&  &  &  & 2 & p\\
&  &  &  &  & 3
\end{array}
\right)  . \label{eq.exa.jnf.step2.2}%
\end{equation}
Furthermore, the triangular matrices $\left(
\begin{array}
[c]{ccccc}%
1 & a &  &  & \\
& 1 &  &  & \\
&  & 2 & j & k\\
&  &  & 2 & m\\
&  &  &  & 2
\end{array}
\right)  $ and $\left(
\begin{array}
[c]{c}%
3
\end{array}
\right)  $ have disjoint spectra, so Corollary \ref{cor.schurtri.block-sim}
yields%
\begin{equation}
\left(
\begin{array}
[c]{cccccc}%
1 & a &  &  &  & \\
& 1 &  &  &  & \\
&  & 2 & j & k & \ell\\
&  &  & 2 & m & n\\
&  &  &  & 2 & p\\
&  &  &  &  & 3
\end{array}
\right)  \sim\left(
\begin{array}
[c]{cccccc}%
1 & a &  &  &  & \\
& 1 &  &  &  & \\
&  & 2 & j & k & \\
&  &  & 2 & m & \\
&  &  &  & 2 & \\
&  &  &  &  & 3
\end{array}
\right)  . \label{eq.exa.jnf.step2.3}%
\end{equation}
Since $\sim$ is an equivalence relation, we can combine the two similarities
(\ref{eq.exa.jnf.step2.2}) and (\ref{eq.exa.jnf.step2.3}), and we conclude
that the claim (\ref{eq.exa.jnf.step2.1}) holds.
\end{example}

This example generalizes:

\begin{theorem}
\label{thm.jnf.step2-T}Let $T\in\mathbb{C}^{n\times n}$ be an upper-triangular
matrix. Assume that the diagonal entries of $T$ come in contiguous runs (i.e.,
if $i,j\in\left[  n\right]  $ satisfy $i<j$ and $T_{i,i}=T_{j,j}$, then
$T_{i,i}=T_{i+1,i+1}=T_{i+2,i+2}=\cdots=T_{j,j}$). Let $S$ be the matrix
obtained from $T$ by setting all entries $T_{i,j}$ with $T_{i,i}\neq T_{j,j}$
to $0$. In other words, let $S\in\mathbb{C}^{n\times n}$ be the $n\times
n$-matrix defined by setting%
\[
S_{i,j}=%
\begin{cases}
T_{i,j}, & \text{if }T_{i,i}=T_{j,j};\\
0, & \text{otherwise}%
\end{cases}
\ \ \ \ \ \ \ \ \ \ \text{for all }i,j\in\left[  n\right]  .
\]
Then, $T\sim S$.
\end{theorem}

\begin{proof}
TODO: Scribe!
\end{proof}

Roughly speaking, Theorem \ref{thm.jnf.step2-T} says that whenever we have an
upper-triangular matrix $T$ whose diagonal has no interlaced values (i.e.,
there is never a $\mu$ between two $\lambda$'s on the diagonal when $\mu
\neq\lambda$), we can \textquotedblleft clean out\textquotedblright\ all the
above-diagonal entries that correspond to different diagonal entries (i.e.,
all above-diagonal entries $T_{i,j}$ with $T_{i,i}\neq T_{j,j}$) by a
similarity (i.e., if we set all these entries to $0$, the resulting matrix
will be similar to $T$).

Now, combining this with Theorem \ref{thm.jnf.schurtri-diag}, we obtain the following:

\begin{proposition}
\label{prop.jnf.step2}Let $A\in\mathbb{C}^{n\times n}$ be an $n\times
n$-matrix. Then, $A$ is similar to a block-diagonal matrix of the form%
\[
\left(
\begin{array}
[c]{cccc}%
B_{1} &  &  & \\
& B_{2} &  & \\
&  & \ddots & \\
&  &  & B_{k}%
\end{array}
\right)  ,
\]
where each $B_{i}$ is an upper-triangular matrix such that all entries on the
diagonal of $B_{i}$ are equal. (Here, the cells that we left empty are
understood to be filled with zero matrices.)
\end{proposition}

\begin{proof}
TODO: Scribe!
\end{proof}

Note that we have given up unitary similarity at this point: The word
\textquotedblleft similar\textquotedblright\ in Proposition
\ref{prop.jnf.step2} cannot be replaced by \textquotedblleft unitarily
similar\textquotedblright. (A counterexample is easily obtained from Exercise
\ref{exe.schurtri.unisim.two2x2}.)

\subsubsection{\label{subsect.jnf.exist.step3}Step 3: Strictly
upper-triangular matrices}

The block-diagonal matrix in Proposition \ref{prop.jnf.step2} is not yet a
Jordan canonical form, but it is already somewhat close. At least, we have
separated out all the distinct eigenvalues of $A$ and \textquotedblleft
cleaned out the space between them\textquotedblright. We now can work with the
matrices $B_{1},B_{2},\ldots,B_{k}$ separately; each of these matrices has one
distinct eigenvalue. Our next goal is to show that each of these matrices
$B_{1},B_{2},\ldots,B_{k}$ is similar to a Jordan matrix. (This will easily
yield that the total block-diagonal matrix $\left(
\begin{array}
[c]{cccc}%
B_{1} &  &  & \\
& B_{2} &  & \\
&  & \ddots & \\
&  &  & B_{k}%
\end{array}
\right)  $ is similar to a Jordan matrix, and therefore the same holds for $A$.)

For each $i\in\left[  k\right]  $, the matrix $B_{i}$ has all its diagonal
entries equal. Let us say these diagonal entries all equal $\mu_{i}$. Thus,
$B_{i}-\mu_{i}I$ is a strictly upper-triangular matrix. (Recall: a
\emph{strictly upper-triangular} matrix is an upper-triangular matrix whose
diagonal entries are $0$.) We want to show that $B_{i}$ is similar to a Jordan
matrix. Because of Proposition \ref{prop.schurtri.similar.same} \textbf{(g)},
it will suffice to show that the strictly upper-triangular matrix $B_{i}%
-\mu_{i}I$ is similar to a Jordan matrix (because adding $\mu_{i}I$ to a
Jordan matrix always gives a Jordan matrix again).

Thus, our goal is now to show that every strictly upper-triangular matrix $A$
is similar to a Jordan matrix. Before we approach this goal in general, let us
convince ourselves that it is achievable for $2\times2$-matrices.

\begin{example}
A strictly upper-triangular $2\times2$-matrix $A\in\mathbb{C}^{2\times2}$ must
have the form $\left(
\begin{array}
[c]{cc}%
0 & a\\
0 & 0
\end{array}
\right)  $ for some $a\in\mathbb{C}$.

\begin{itemize}
\item If $a=0$, then $A$ is the Jordan matrix $\left(
\begin{array}
[c]{cc}%
J_{1}\left(  0\right)  & \\
& J_{1}\left(  0\right)
\end{array}
\right)  $.

\item If $a\neq0$, then $A$ is similar to the Jordan matrix $J_{2}\left(
0\right)  =\left(
\begin{array}
[c]{cc}%
0 & 1\\
0 & 0
\end{array}
\right)  $. Indeed,
\[
A=\left(
\begin{array}
[c]{cc}%
a & 0\\
0 & 1
\end{array}
\right)  \left(
\begin{array}
[c]{cc}%
0 & 1\\
0 & 0
\end{array}
\right)  \left(
\begin{array}
[c]{cc}%
a & 0\\
0 & 1
\end{array}
\right)  ^{-1}.
\]

\end{itemize}
\end{example}

Now, we return to the general case. Let $\mathbb{F}$ be any field. Let
$A\in\mathbb{F}^{n\times n}$ be any strictly upper-triangular $n\times
n$-matrix. (We don't need to restrict ourselves to the case $\mathbb{F}%
=\mathbb{C}$ here.) We want to prove that $A$ is similar to a Jordan matrix.

The key to this proof will be to restate the question in terms of certain
bases of $\mathbb{F}^{n}$, and then to construct these bases by an iterative
process. We begin with a few notions:

\begin{convention}
We fix a nonnegative integer $n\in\mathbb{N}$, a field $\mathbb{F}$ and a
strictly upper-triangular matrix $A\in\mathbb{F}^{n\times n}$ for the rest of
Subsection \ref{subsect.jnf.exist.step3}.
\end{convention}

We observe that%
\begin{equation}
A^{n}=0. \label{eq.jnf.exist.step3.An=0}%
\end{equation}
(This is a well-known property of strictly upper-triangular $n\times
n$-matrices. It can be obtained by applying Lemma \ref{lem.uptri.prod=0} to
$T_{i}=A$, because $A$ is an upper-triangular matrix whose all diagonal
entries are $0$. An alternative proof can be found, e.g., in \cite[Corollary
3.78]{lina}\footnote{To be precise, \cite[Corollary 3.78]{lina} proves the
analogous property for strictly \textbf{lower}-triangular matrices. But the
case of strictly upper-triangular matrices is analogous (the roles of rows and
columns are swapped).}.)

\begin{definition}
\label{def.jnf.exist.step3.orbits-etc}\textbf{(a)} An \emph{orbit} shall mean
a tuple of the form $\left(  A^{0}v,\ A^{1}v,\ \ldots,\ A^{k}v\right)  $,
where $v\in\mathbb{F}^{n}$ is a vector and $k\in\mathbb{N}$ is an integer
satisfying $A^{k+1}v=0$. (We can also write this tuple as $\left(
v,\ Av,\ A^{2}v,\ \ldots,\ A^{k}v\right)  $.) \medskip

\textbf{(b)} The \emph{concatenation} of two tuples $\left(  a_{1}%
,a_{2},\ldots,a_{k}\right)  $ and $\left(  b_{1},b_{2},\ldots,b_{\ell}\right)
$ is defined to be the tuple $\left(  a_{1},a_{2},\ldots,a_{k},b_{1}%
,b_{2},\ldots,b_{\ell}\right)  $. Thus, concatenation is a binary operation on
the set of tuples. Since this operation is associative, we thus obtain the
notion of concatenation of several tuples. For example, the concatenation of
three tuples $\left(  a_{1},a_{2},\ldots,a_{k}\right)  $ and $\left(
b_{1},b_{2},\ldots,b_{\ell}\right)  $ and $\left(  c_{1},c_{2},\ldots
,c_{m}\right)  $ is $\left(  a_{1},a_{2},\ldots,a_{k},b_{1},b_{2}%
,\ldots,b_{\ell},c_{1},c_{2},\ldots,c_{m}\right)  $. \medskip

\textbf{(c)} A tuple $\left(  v_{1},v_{2},\ldots,v_{m}\right)  $ of vectors in
$\mathbb{F}^{n}$ will be called \emph{forwarded} if each $i\in\left[
m\right]  $ satisfies $Av_{i}=v_{i+1}$ or $Av_{i}=0$. (Here, $v_{m+1}$ is
understood to be $0$.) \medskip

\textbf{(d)} A tuple $\left(  v_{1},v_{2},\ldots,v_{m}\right)  $ of vectors in
$\mathbb{F}^{n}$ will be called \emph{backwarded} if each $i\in\left[
m\right]  $ satisfies $Av_{i}=v_{i-1}$ or $Av_{i}=0$. (Here, $v_{0}$ is
understood to be $0$.) \medskip

Note that the notions of \textquotedblleft orbit\textquotedblright,
\textquotedblleft forwarded\textquotedblright\ and \textquotedblleft
backwarded\textquotedblright\ depend on $A$, but we do not mention $A$ since
$A$ is fixed.
\end{definition}

\begin{example}
Let $p,q,r$ be three vectors in $\mathbb{F}^{n}$ satisfying $A^{3}p=0$ and
$A^{2}q=0$ and $A^{4}r=0$. Then, the $9$-tuple
\[
\left(  p,\ Ap,\ A^{2}p,\ q,\ Aq,\ r,\ Ar,\ A^{2}r,\ A^{3}r\right)
\]
is forwarded. Indeed, if we rename this tuple as $\left(  v_{1},v_{2}%
,\ldots,v_{9}\right)  $, then each $i\in\left\{  1,2,4,6,7,8\right\}  $
satisfies $Av_{i}=v_{i+1}$, whereas each $i\in\left\{  3,5\right\}  $
satisfies $Av_{i}=0$. This $9$-tuple is furthermore the concatenation of the
orbits $\left(  p,\ Ap,\ A^{2}p\right)  $, $\left(  q,\ Aq\right)  $ and
$\left(  r,\ Ar,\ A^{2}r,\ A^{3}r\right)  $. Reversing this $9$-tuple yields a
new $9$-tuple%
\[
\left(  A^{3}r,\ A^{2}r,\ Ar,\ r,\ Aq,\ q,\ A^{2}p,\ Ap,\ p\right)  ,
\]
which is backwarded.
\end{example}

What we have seen in this example can be generalized:

\begin{proposition}
\label{prop.jnf.exist.step3.bwd}\textbf{(a)} A tuple $\left(  v_{1}%
,v_{2},\ldots,v_{m}\right)  $ of vectors in $\mathbb{F}^{n}$ is forwarded if
and only if it is a concatenation of finitely many orbits. \medskip

\textbf{(b)} A tuple $\left(  v_{1},v_{2},\ldots,v_{m}\right)  $ of vectors in
$\mathbb{F}^{n}$ is backwarded if and only if the tuple $\left(  v_{m}%
,v_{m-1},\ldots,v_{1}\right)  $ is forwarded.
\end{proposition}

\begin{proof}
TODO: Scribe?
\end{proof}

More importantly, backwarded tuples are closely related to Jordan forms. To wit:

\begin{proposition}
\label{prop.jnf.exist.step3.basis}Let $\left(  s_{1},s_{2},\ldots
,s_{n}\right)  $ be a basis of $\mathbb{F}^{n}$. Let $S\in\mathbb{F}^{n\times
n}$ be the $n\times n$-matrix with columns $s_{1},s_{2},\ldots,s_{n}$. Then,
$S^{-1}AS$ is a Jordan matrix if and only if the $n$-tuple $\left(
s_{1},s_{2},\ldots,s_{n}\right)  $ is backwarded.
\end{proposition}

\begin{proof}
We shall only prove the \textquotedblleft if\textquotedblright\ part, since
this is the only part that we will use; however, the proof of the
\textquotedblleft only if\textquotedblright\ part can essentially be obtained
from the proof of the \textquotedblleft if\textquotedblright\ part by reading
it in reverse.

So let us prove the \textquotedblleft if\textquotedblright\ part. Thus, we
assume that the $n$-tuple $\left(  s_{1},s_{2},\ldots,s_{n}\right)  $ is
backwarded. In other words, each $i\in\left[  n\right]  $ satisfies
\begin{equation}
As_{i}=s_{i-1}\text{ or }As_{i}=0 \label{pf.prop.jnf.exist.step3.basis.fwd}%
\end{equation}
(where $s_{0}$ means $0$). Our goal is to show that $S^{-1}AS$ is a Jordan matrix.

The matrix $S$ is invertible, since its columns $s_{1},s_{2},\ldots,s_{n}$
form a basis of $\mathbb{F}^{n}$.

We call an $i\in\left[  n\right]  $

\begin{itemize}
\item \emph{red} if it satisfies $As_{i}=s_{i-1}\neq0$, and

\item \emph{blue} if it satisfies $As_{i}=0$.
\end{itemize}

Thus, (\ref{pf.prop.jnf.exist.step3.basis.fwd}) shows that each $i\in\left[
n\right]  $ is either red or blue. Note that $1$ is always blue, since
$s_{1-1}=s_{0}=0$.

Let $J$ be the $n\times n$-matrix whose $\left(  i,j\right)  $-th entry is%
\[%
\begin{cases}
1, & \text{if }j=i+1\text{ and }j\text{ is red};\\
0, & \text{otherwise.}%
\end{cases}
\]
For instance, if $n=8$ and if $2,3,5,7,8$ are red whereas $1,4,6$ are blue,
then%
\[
J=\left(
\begin{array}
[c]{cccccccc}%
0 & 1 & 0 & 0 & 0 & 0 & 0 & 0\\
0 & 0 & 1 & 0 & 0 & 0 & 0 & 0\\
0 & 0 & 0 & 0 & 0 & 0 & 0 & 0\\
0 & 0 & 0 & 0 & 1 & 0 & 0 & 0\\
0 & 0 & 0 & 0 & 0 & 0 & 0 & 0\\
0 & 0 & 0 & 0 & 0 & 0 & 1 & 0\\
0 & 0 & 0 & 0 & 0 & 0 & 0 & 1\\
0 & 0 & 0 & 0 & 0 & 0 & 0 & 0
\end{array}
\right)
\]
Thus, all entries of $J$ are $0$ except for some $1$s placed in cells directly
above the main diagonal. This shows that $J$ is a Jordan matrix, with each
Jordan block covering the rows and columns between one blue $i\in\left[
n\right]  $ and the next. Explicitly, if $i_{1},i_{2},\ldots,i_{k}$ are the
blue $i$'s listed from smallest to largest (i.e., with $i_{1}<i_{2}%
<\cdots<i_{k}$), then $J$ is the block-diagonal matrix $\left(
\begin{array}
[c]{cccc}%
J_{i_{2}-i_{1}}\left(  0\right)  & 0 & \cdots & 0\\
0 & J_{i_{3}-i_{2}}\left(  0\right)  & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & J_{i_{k+1}-i_{k}}\left(  0\right)
\end{array}
\right)  $, where we set $i_{k+1}:=n+1$.

We shall now show that $AS=SJ$. This will entail that $S^{-1}AS=J$, which as
we know is a Jordan matrix.

For each $i\in\left[  n\right]  $, we have%
\begin{align*}
&  \left(  \text{the }i\text{-th column of the matrix }AS\right) \\
&  =A\cdot\underbrace{\left(  \text{the }i\text{-th column of the matrix
}S\right)  }_{\substack{=s_{i}\\\text{(by the definition of }S\text{)}%
}}\ \ \ \ \ \ \ \ \ \ \left(  \text{by the rules for multiplying
matrices}\right) \\
&  =As_{i}=%
\begin{cases}
s_{i-1}, & \text{if }i\text{ is red};\\
0, & \text{if }i\text{ is blue}%
\end{cases}
\ \ \ \ \ \ \ \ \ \ \left(  \text{by the definition of \textquotedblleft
red\textquotedblright\ and \textquotedblleft blue\textquotedblright}\right)  .
\end{align*}
On the other hand, for each $i\in\left[  n\right]  $, we have%
\begin{align*}
&  \left(  \text{the }i\text{-th column of the matrix }SJ\right) \\
&  =S\cdot\underbrace{\left(  \text{the }i\text{-th column of the matrix
}J\right)  }_{\substack{=%
\begin{cases}
e_{i-1}, & \text{if }i\text{ is red};\\
0, & \text{if }i\text{ is blue}%
\end{cases}
\\\text{(by the definition of }J\text{)}}}\ \ \ \ \ \ \ \ \ \ \left(  \text{by
the rules for multiplying matrices}\right) \\
&  =S\cdot%
\begin{cases}
e_{i-1}, & \text{if }i\text{ is red};\\
0, & \text{if }i\text{ is blue}%
\end{cases}
\ \ \ =%
\begin{cases}
Se_{i-1}, & \text{if }i\text{ is red};\\
0, & \text{if }i\text{ is blue}%
\end{cases}
\\
&  =%
\begin{cases}
s_{i-1}, & \text{if }i\text{ is red};\\
0, & \text{if }i\text{ is blue.}%
\end{cases}
\end{align*}
(since $Se_{i-1}=\left(  \text{the }\left(  i-1\right)  \text{-th column of
the matrix }S\right)  =s_{i-1}$). Comparing these two equalities, we obtain%
\[
\left(  \text{the }i\text{-th column of the matrix }AS\right)  =\left(
\text{the }i\text{-th column of the matrix }SJ\right)
\]
for each $i\in\left[  n\right]  $. Hence, $AS=SJ$. Therefore, $S^{-1}AS=J$.
Hence, $S^{-1}AS$ is a Jordan matrix (since we know that $J$ is a Jordan
matrix). This proves the \textquotedblleft if\textquotedblright\ direction of
Proposition \ref{prop.jnf.exist.step3.basis}.
\end{proof}

Recall that our goal is to show that $A$ is similar to a Jordan matrix.
Proposition \ref{prop.jnf.exist.step3.basis} shows us a way to this goal: We
just need to find a basis for $\mathbb{F}^{n}$ that is forwarded. In view of
Proposition \ref{prop.jnf.exist.step3.bwd} \textbf{(b)}, this is tantamount to
finding a basis for $\mathbb{F}^{n}$ that is backwarded. Let us first see how
to do so on examples:

[...]

TODO: Polish from here!

TODO: Empty cells = $0$ entries.

\begin{example}
Let $n=4$ and $A=$... (where the cells we leave empty are understood to
contain zeroes). Then, ... find some interesting orbits and bases

TODO: Scribe?
\end{example}

We begin by finding forwarded bases in some examples:

\begin{example}
Let $n=2$. Then, $A=\left(
\begin{array}
[c]{cc}%
0 & a\\
0 & 0
\end{array}
\right)  $ for some $a\in\mathbb{F}$.

We are looking for an invertible matrix $S\in\mathbb{F}^{2\times2}$ such that
$S^{-1}AS$ is a Jordan matrix.

If $a=0$, then this is obvious (just take $S=I_{2}$), since $A=\left(
\begin{array}
[c]{cc}%
J_{1}\left(  0\right)  & \\
& J_{1}\left(  0\right)
\end{array}
\right)  $ is already a Jordan matrix.

Now assume $a\neq0$.

Consider our unknown invertible matrix $S$. Let $s_{1}$ and $s_{2}$ be its
columns. Then, $s_{1}$ and $s_{2}$ are linearly independent (since $S$ is
invertible). Moreover, we want $S^{-1}AS=\left(
\begin{array}
[c]{cc}%
0 & 1\\
0 & 0
\end{array}
\right)  $. In other words, we want $AS=S\left(
\begin{array}
[c]{cc}%
0 & 1\\
0 & 0
\end{array}
\right)  $. However, $S=\left(
\begin{array}
[c]{cc}%
s_{1} & s_{2}%
\end{array}
\right)  $ (in block-matrix notation), so $S\left(
\begin{array}
[c]{cc}%
0 & 1\\
0 & 0
\end{array}
\right)  =\left(
\begin{array}
[c]{cc}%
0 & s_{1}%
\end{array}
\right)  $. Thus our equation $AS=S\left(
\begin{array}
[c]{cc}%
0 & 1\\
0 & 0
\end{array}
\right)  $ is equivalent to%
\[
\left(
\begin{array}
[c]{cc}%
As_{1} & As_{2}%
\end{array}
\right)  =\left(
\begin{array}
[c]{cc}%
0 & s_{1}%
\end{array}
\right)  .
\]
In other words, $As_{1}=0$ and $As_{2}=s_{1}$.

So we are looking for two linearly independent vectors $s_{1},s_{2}%
\in\mathbb{F}^{2}$ such that $As_{1}=0$ and $As_{2}=s_{1}$.

One way to do so is to pick some nonzero vector $s_{1}\in\operatorname*{Ker}%
A$, and then define $s_{2}$ to be some preimage of $s_{1}$ under $A$. (It can
be shown that such preimage exists.) This way, however, does not generalize to
higher $n$.

Another (better) way is to start by picking $s_{2}\in\mathbb{F}^{2}%
\setminus\operatorname*{Ker}A$ and then setting $s_{1}=As_{2}$. We claim that
$s_{1}$ and $s_{2}$ are linearly independent, and that $As_{1}=0$.

To show that $As_{1}=0$, we just observe that $As_{1}=\underbrace{AA}%
_{=A^{2}=0}s_{2}=0$.

To show that $s_{1}$ and $s_{2}$ are linearly independent, we argue as
follows: Let $\lambda_{1},\lambda_{2}\in\mathbb{F}$ be such that $\lambda
_{1}s_{1}+\lambda_{2}s_{2}=0$. Applying $A$ to this, we obtain $A\cdot\left(
\lambda_{1}s_{1}+\lambda_{2}s_{2}\right)  =A\cdot0=0$. However,%
\[
A\cdot\left(  \lambda_{1}s_{1}+\lambda_{2}s_{2}\right)  =\lambda
_{1}\underbrace{As_{1}}_{=0}+\lambda_{2}\underbrace{As_{2}}_{=s_{1}}%
=\lambda_{2}s_{1},
\]
so this becomes $\lambda_{2}s_{1}=0$. However, $s_{1}\neq0$ (because
$s_{1}=As_{2}$ but $s_{2}\notin\operatorname*{Ker}A$). Hence, $\lambda_{2}=0$.
Now, $\lambda_{1}s_{1}+\lambda_{2}s_{2}=0$ becomes $\lambda_{1}s_{1}=0$. Since
$s_{1}\neq0$, this yields $\lambda_{1}=0$. Now both $\lambda_{i}$s are $0$, qed.
\end{example}

\begin{example}
Let $n=3$ and $A=\left(
\begin{array}
[c]{ccc}
& 1 & 1\\
&  & 0\\
&  &
\end{array}
\right)  $.

Our first method above doesn't work, because most vectors in
$\operatorname*{Ker}A$ do not have preimages under $A$.

However, our second method can be made to work:

We pick a vector $s_{3}\notin\operatorname*{Ker}A$. To wit, we pick
$s_{3}=e_{3}=\left(
\begin{array}
[c]{c}%
0\\
0\\
1
\end{array}
\right)  $. Then, $As_{3}=e_{1}$. Set $s_{2}=As_{3}=e_{1}$. Note that
$s_{2}\in\operatorname*{Ker}A$. Let $s_{1}$ be another nonzero vector in
$\operatorname*{Ker}A$, namely $e_{2}-e_{3}$.{} These three vectors
$s_{1},s_{2},s_{3}$ are linearly independent and satisfy $As_{1}=0$ and
$As_{2}=0$ and $As_{3}=s_{2}$.

So $S=\left(
\begin{array}
[c]{ccc}%
0 & 1 & 0\\
1 & 0 & 0\\
-1 & 0 & 1
\end{array}
\right)  $. And indeed, $S^{-1}AS=\allowbreak\left(
\begin{array}
[c]{ccc}%
0 & 0 & 0\\
0 & 0 & 1\\
0 & 0 & 0
\end{array}
\right)  $ is a Jordan matrix.
\end{example}

So what is the general algorithm here? Can we always find $n$ linearly
independent vectors $s_{1},s_{2},\ldots,s_{n}$ such that each $As_{i}$ is
either $0$ or $s_{i-1}$ ?

\bigskip

Now, we return to the general case: How do we find a backwarded basis $\left(
s_{1},s_{2},\ldots,s_{n}\right)  $ of $\mathbb{F}^{n}$ ?

(The following proof is due to Terence Tao \cite{Tao07}.)

We recall that an \emph{orbit} was defined to be a tuple of the form $\left(
v,Av,A^{2}v,\ldots,A^{k}v\right)  $, where $v\in\mathbb{F}^{n}$ satisfies
$A^{k+1}v=0$. Note that for each $v\in\mathbb{F}^{n}$, there is an orbit that
starts with $v$, since $A^{n}=0$.

Now, we claim the existence of a forwarded basis:

\begin{lemma}
[orbit basis lemma]\label{lem.jnf.exist.tao}There exists a basis of
$\mathbb{F}^{n}$ that is a concatenation of orbits.
\end{lemma}

Once Lemma \ref{lem.jnf.exist.tao} is proved, we will be done, because such a
basis will be a forwarded basis (by Proposition \ref{prop.jnf.exist.step3.bwd}
\textbf{(a)}), and therefore reading it backwards will gives us a backwarded
basis (by Proposition \ref{prop.jnf.exist.step3.bwd} \textbf{(b)}), which is
precisely what we wish. For example, if the basis that Lemma
\ref{lem.jnf.exist.tao} gives us is%
\[
\left(  u,Au,A^{2}u,\ \ v,Av,A^{2}v,A^{3}v,\ \ w,Aw\right)
\]
(with $A^{3}u=0$ and $A^{4}v=0$ and $A^{2}w=0$), then reading it backwards
gives
\[
\left(  Aw,w,\ \ A^{3}v,A^{2}v,Av,v,\ \ A^{2}u,Au,u\right)  ,
\]
which is a backwarded basis of $\mathbb{F}^{n}$.

\begin{proof}
[Proof of Lemma \ref{lem.jnf.exist.tao}.]It is easy to find a finite
\textbf{spanning set} of $\mathbb{F}^{n}$ that is a concatenation of orbits.
Indeed, we can start with the standard basis $\left(  e_{1},e_{2},\ldots
,e_{n}\right)  $, and extend it to the list%
\begin{align*}
&  (e_{1},Ae_{1},A^{2}e_{1},\ldots,A^{n-1}e_{1},\\
&  \ e_{2},Ae_{2},A^{2}e_{2},\ldots,A^{n-1}e_{2},\\
&  \ \ldots,\\
&  \ e_{n},Ae_{n},A^{2}e_{n},\ldots,A^{n-1}e_{n}).
\end{align*}
This is clearly a spanning set of $\mathbb{F}^{n}$ (since $e_{1},e_{2}%
,\ldots,e_{n}$ already span $\mathbb{F}^{n}$), and also a concatenation of
orbits (since $A^{n}=0$).

Now, we will gradually shorten this spanning set (i.e., replace it by smaller
ones) until we get a basis. We have to do this in such a way that it remains a
spanning set throughout the process, and that it remains a concatenation of
orbits throughout the process.

For the sake of concreteness, let us assume that our spanning set is%
\[
\left(  x,Ax,\ \ y,Ay,A^{2}y,\ \ z,Az,A^{2}z,A^{3}z,\ \ w\right)  ,
\]
with $A^{2}x=0$ and $A^{3}y=0$ and $A^{4}z=0$ and $Aw=0$. If this spanning set
is linearly independent, then it is already a basis, and we are done. So
assume that it is not. Thus, there exists some linear dependence relation --
say,%
\[
3x+4Ax+5Ay+6A^{2}y+7A^{2}z+8w=0.
\]
Applying $A$ to this relation, we obtain%
\begin{align*}
3Ax+4A^{2}x+5A^{2}y+6A^{3}y+7A^{3}z+8Aw  &  =0,\ \ \ \ \ \ \ \ \ \ \text{i.e.}%
\\
3Ax+5A^{2}y+7A^{3}z  &  =0
\end{align*}
(since $A^{2}x=0$ and $A^{3}y=0$ and $Aw=0$). Applying $A$ to this relation
again, we obtain%
\begin{align*}
3A^{2}x+5A^{3}y+7A^{4}z  &  =0,\ \ \ \ \ \ \ \ \ \ \text{i.e.}\\
0  &  =0.
\end{align*}
We have gone too far, so let us revert to the previous equation:%
\[
3Ax+5A^{2}y+7A^{3}z=0.
\]
So this is a linear dependence relation between the \textbf{final} vectors of
the orbits in our spanning set. (\textquotedblleft\emph{Final}%
\textquotedblright\ means the last vector in the orbit.) Factoring out an $A$
in this relation, we obtain%
\[
A\left(  3x+5Ay+7A^{2}z\right)  =0.
\]
Thus, the $1$-tuple $\left(  3x+5Ay+7A^{2}z\right)  $ is an orbit.

Now, let us replace the orbit $\left(  x,Ax\right)  $ in our spanning set
$\left(  x,Ax,\ \ y,Ay,A^{2}y,\ \ z,Az,A^{2}z,A^{3}z,\ \ w\right)  $ by the
orbit $\left(  3x+5Ay+7A^{2}z\right)  $. We get
\[
\left(  3x+5Ay+7A^{2}z,\ \ y,Ay,A^{2}y,\ \ z,Az,A^{2}z,A^{3}z,\ \ w\right)  .
\]
This is still a concatenation of orbits, since the $1$-tuple $\left(
3x+5Ay+7A^{2}z\right)  $ is an orbit. Furthermore, this is still a spanning
set of $\mathbb{F}^{n}$; why? Because we removed the vector $Ax$, which was
unnecessary for spanning $\mathbb{F}^{n}$ (because the equality $3Ax+5A^{2}%
y+7A^{3}z=0$ reveals that it is a linear combination of the other vectors in
our spanning set), and we replaced $x$ by $3x+5Ay+7A^{2}z$ (which does not
change the span, because $Ay$ and $A^{2}z$ are still in the spanning set).

This example generalizes. In the general case, you have a spanning set
$\mathbf{s}$ that is a concatenation of orbits:%
\[
\mathbf{s}=\left(  v_{1},Av_{1},\ldots,A^{m_{1}}v_{1},\ v_{2},Av_{2}%
,\ldots,A^{m_{2}}v_{2},\ \ldots,\ v_{k},Av_{k},\ldots,A^{m_{k}}v_{k}\right)
,
\]
where $A^{m_{1}+1}v_{1}=0$ and $A^{m_{2}+1}v_{2}=0$ and $\ldots$ and
$A^{m_{k}+1}v_{k}=0$. If this spanning set $\mathbf{s}$ is a basis, you are
done. If not, you pick a linear dependence relation:%
\[
\sum_{i,j}\lambda_{i,j}A^{j}v_{i}=0.
\]
By multiplying this by $A$ an appropriate amount of times (namely, you keep
multiplying until it becomes $0=0$, and then you take a step back), you obtain
a linear dependence relation that involves only the \textbf{final} vectors of
the orbits (i.e., the vectors $A^{m_{1}}v_{1},\ A^{m_{2}}v_{2},\ \ldots
,\ A^{m_{k}}v_{k}$). Thus, it will look like this:%
\[
\mu_{1}A^{m_{1}}v_{1}+\mu_{2}A^{m_{2}}v_{2}+\cdots+\mu_{k}A^{m_{k}}v_{k}=0
\]
(with at least one of $\mu_{1},\mu_{2},\ldots,\mu_{k}$ being nonzero). Assume
WLOG that the first $p$ of the scalars $\mu_{1},\mu_{2},\ldots,\mu_{k}$ are
nonzero, while the remaining $k-p$ are $0$ (this can always be achieved by
permuting the orbits, which of course does not change anything about the
spanning set being a spanning set). So the relation becomes%
\[
\mu_{1}A^{m_{1}}v_{1}+\mu_{2}A^{m_{2}}v_{2}+\cdots+\mu_{p}A^{m_{p}}v_{p}=0,
\]
with $\mu_{1},\mu_{2},\ldots,\mu_{p}$ being nonzero. Note that $p>0$ (since at
least one of $\mu_{1},\mu_{2},\ldots,\mu_{k}$ is nonzero), so that $\mu
_{1}\neq0$. Assume WLOG that $m_{1}=\min\left\{  m_{1},m_{2},\ldots
,m_{p}\right\}  $, and factor out $A^{m_{1}}$ from this relation. This yields%
\[
A^{m_{1}}\left(  \mu_{1}v_{1}+\mu_{2}A^{m_{2}-m_{1}}v_{2}+\cdots+\mu
_{p}A^{m_{p}-m_{1}}v_{p}\right)  =0.
\]
Now, set $w_{1}=\mu_{1}v_{1}+\mu_{2}A^{m_{2}-m_{1}}v_{2}+\cdots+\mu
_{p}A^{m_{p}-m_{1}}v_{p}$. Thus, $A^{m_{1}}w_{1}=0$. Hence, $\left(
w_{1},Aw_{1},A^{2}w_{1},\ldots,A^{m_{1}-1}w_{1}\right)  $ is an orbit of
length $m_{1}$. Now, replace the orbit $\left(  v_{1},Av_{1},\ldots,A^{m_{1}%
}v_{1}\right)  $ in the spanning set $\mathbf{s}$ by the shorter orbit
$\left(  w_{1},Aw_{1},A^{2}w_{1},\ldots,A^{m_{1}-1}w_{1}\right)  $. The
resulting list%
\[
\left(  w_{1},Aw_{1},A^{2}w_{1},\ldots,A^{m_{1}-1}w_{1},\ v_{2},Av_{2}%
,\ldots,A^{m_{2}}v_{2},\ \ldots,\ v_{k},Av_{k},\ldots,A^{m_{k}}v_{k}\right)
\]
is still a concatenation of orbits (since $A^{m_{1}}w_{1}=0$). Also, it still
spans $\mathbb{F}^{n}$, because the $m_{1}+1$ vectors $v_{1},Av_{1}%
,\ldots,A^{m_{1}}v_{1}$ that we have removed from $\mathbf{s}$ can be
recovered as linear combinations of the vectors in our new list as follows:%
\begin{align*}
v_{1}  &  =\dfrac{1}{\mu_{1}}\left(  w_{1}-\left(  \mu_{2}A^{m_{2}-m_{1}}%
v_{2}+\cdots+\mu_{p}A^{m_{p}-m_{1}}v_{p}\right)  \right) \\
&  \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \left(  \text{since }w_{1}=\mu
_{1}v_{1}+\mu_{2}A^{m_{2}-m_{1}}v_{2}+\cdots+\mu_{p}A^{m_{p}-m_{1}}%
v_{p}\right)  ;\\
Av_{1}  &  =A\cdot\dfrac{1}{\mu_{1}}\left(  w_{1}-\left(  \mu_{2}%
A^{m_{2}-m_{1}}v_{2}+\cdots+\mu_{p}A^{m_{p}-m_{1}}v_{p}\right)  \right) \\
&  =\dfrac{1}{\mu_{1}}\left(  Aw_{1}-\left(  \mu_{2}A^{m_{2}-m_{1}+1}%
v_{2}+\cdots+\mu_{p}A^{m_{p}-m_{1}+1}v_{p}\right)  \right)  ;\\
A^{2}v_{1}  &  =A^{2}\cdot\dfrac{1}{\mu_{1}}\left(  w_{1}-\left(  \mu
_{2}A^{m_{2}-m_{1}}v_{2}+\cdots+\mu_{p}A^{m_{p}-m_{1}}v_{p}\right)  \right) \\
&  =\dfrac{1}{\mu_{1}}\left(  A^{2}w_{1}-\left(  \mu_{2}A^{m_{2}-m_{1}+2}%
v_{2}+\cdots+\mu_{p}A^{m_{p}-m_{1}+2}v_{p}\right)  \right)  ;\\
\ldots;  & \\
A^{m_{1}}v_{1}  &  =\dfrac{1}{\mu_{1}}\left(  \underbrace{A^{m_{1}}w_{1}}%
_{=0}-\left(  \mu_{2}A^{m_{2}}v_{2}+\cdots+\mu_{p}A^{m_{p}}v_{p}\right)
\right) \\
&  =\dfrac{1}{\mu_{1}}\left(  -\left(  \mu_{2}A^{m_{2}}v_{2}+\cdots+\mu
_{p}A^{m_{p}}v_{p}\right)  \right)  .
\end{align*}


So we have found a new spanning set of $\mathbb{F}^{n}$ that is still a
concatenation of orbits, but is shorter than $\mathbf{s}$ (namely, it has one
less vector than $\mathbf{s}$). In other words, we have found a way to replace
a spanning set of $\mathbb{F}^{n}$ that is a concatenation of orbits by a
smaller such set as long as it is linearly independent. Performing this
process repeatedly, we will eventually obtain a basis (since we cannot keep
making a finite list shorter and shorter indefinitely). This proves Lemma
\ref{lem.jnf.exist.tao}.
\end{proof}

As we said, Lemma \ref{lem.jnf.exist.tao} gives us a basis of $\mathbb{F}^{n}$
that is a concatenation of orbits. In other words, it gives us a forwarded
basis (by Proposition \ref{prop.jnf.exist.step3.bwd} \textbf{(a)}), and
therefore reading it backwards will gives us a backwarded basis (by
Proposition \ref{prop.jnf.exist.step3.bwd} \textbf{(b)}). In view of
Proposition \ref{prop.jnf.exist.step3.basis}, this lets us find an invertible
matrix $S\in\mathbb{F}^{n\times n}$ such that $S^{-1}AS$ is a Jordan matrix.
This completes the proof of Theorem \ref{thm.jnf.jnf} \textbf{(a)} (the
existence part of the Jordan canonical form).

\begin{example}
Let $\mathbb{F}=\mathbb{C}$ and
\[
A=\left(
\begin{array}
[c]{cccccc}%
0 & 1 & 0 & -1 & 1 & -1\\
0 & 1 & 1 & -2 & 2 & -2\\
0 & 1 & 0 & -1 & 2 & -2\\
0 & 1 & 0 & -1 & 2 & -2\\
0 & 1 & 0 & -1 & 1 & -1\\
0 & 1 & 0 & -1 & 1 & -1
\end{array}
\right)  .
\]
This matrix $A$ is not strictly upper-triangular, but it is nilpotent, with
$A^{3}=0$, so the above argument goes equally well with this $A$.

[TODO: Replace this by a better example, with an actual strictly
upper-triangular $A$.]

Let us try to find a basis of $\mathbb{F}^{6}$ that is a concatenation of orbits.

We begin with the spanning set%
\[
\left(  e_{1},Ae_{1},A^{2}e_{1},\ \ e_{2},Ae_{2},A^{2}e_{2},\ \ \ldots
,\ \ e_{6},Ae_{6},A^{2}e_{6}\right)  .
\]
It has lots of linear dependencies. For one, $Ae_{1}=0$. Multiplying it by $A$
gives $A^{2}e_{1}=0$, so we can replace $\left(  e_{1},Ae_{1},A^{2}%
e_{1}\right)  $ by $\left(  e_{1},Ae_{1}\right)  $. So our spanning set
becomes%
\[
\left(  e_{1},Ae_{1},\ \ e_{2},Ae_{2},A^{2}e_{2},\ \ \ldots,\ \ e_{6}%
,Ae_{6},A^{2}e_{6}\right)  .
\]
One more step of the same form gives%
\[
\left(  e_{1},\ \ e_{2},Ae_{2},A^{2}e_{2},\ \ \ldots,\ \ e_{6},Ae_{6}%
,A^{2}e_{6}\right)  .
\]


Now, observe that $Ae_{3}=e_{2}$. That is, $e_{2}-Ae_{3}=0$. Multiplying it by
$A^{2}$, we obtain $A^{2}e_{2}=0$ (since $A^{2}\cdot Ae_{3}=A^{3}e_{3}=0$). So
we replace the orbit $\left(  e_{2},Ae_{2},A^{2}e_{2}\right)  $ by $\left(
e_{2},Ae_{2}\right)  $. So we get the spanning set
\[
\left(  e_{1},\ \ e_{2},Ae_{2},\ \ e_{3},Ae_{3},A^{2}e_{3},\ \ e_{4}%
,Ae_{4},A^{2}e_{4},\ \ e_{5},Ae_{5},A^{2}e_{5},\ \ e_{6},Ae_{6},A^{2}%
e_{6}\right)  .
\]


We observe that%
\[
Ae_{2}=e_{1}+e_{2}+e_{3}+e_{4}+e_{5}+e_{6}.
\]
In other words,%
\[
Ae_{2}-e_{1}-e_{2}-e_{3}-e_{4}-e_{5}-e_{6}=0.
\]
Multiplying this by $A^{2}$, we obtain%
\[
-A^{2}e_{3}-A^{2}e_{4}-A^{2}e_{5}-A^{2}e_{6}=0.
\]
In other words,%
\[
A^{2}\left(  -e_{3}-e_{4}-e_{5}-e_{6}\right)  =0.
\]
Thus, we set $w_{1}:=-e_{3}-e_{4}-e_{5}-e_{6}$, and we replace $\left(
e_{3},Ae_{3},A^{2}e_{3}\right)  $ by $\left(  w_{1},Aw_{1}\right)  $. So we
get the spanning set
\[
\left(  e_{1},\ \ e_{2},Ae_{2},\ \ w_{1},Aw_{1},\ \ e_{4},Ae_{4},A^{2}%
e_{4},\ \ e_{5},Ae_{5},A^{2}e_{5},\ \ e_{6},Ae_{6},A^{2}e_{6}\right)  .
\]
Keep making these steps. Eventually, there will be no more linear
dependencies, so we will have a basis.
\end{example}

\begin{exercise}
\fbox{3} Compute the Jordan canonical form of the matrix $\left(
\begin{array}
[c]{ccc}%
1 & 0 & 2\\
0 & 1 & 1\\
0 & 0 & 1
\end{array}
\right)  $.
\end{exercise}

\begin{exercise}
\label{exe.jnf.step3.An=0}\fbox{4} Let $A\in\mathbb{C}^{n\times n}$ be a
matrix. Prove that the following three statements are equivalent:

\begin{itemize}
\item $\mathcal{A}$\textit{:} The matrix $A$ is nilpotent.

\item $\mathcal{B}$\textit{:} We have $A^{n}=0$.

\item $\mathcal{C}$\textit{:} The only eigenvalue of $A$ is $0$ (that is, we
have $\sigma\left(  A\right)  =\left\{  0\right\}  $).
\end{itemize}
\end{exercise}

\begin{exercise}
\fbox{4} Let $A\in\mathbb{C}^{n\times n}$ be a matrix. Let $\lambda
\in\mathbb{C}$ be nonzero. Prove the following: \medskip

\textbf{(a)} If $A$ is nilpotent, then $A\sim\lambda A$. \medskip

\textbf{(b)} If $A\sim\lambda A$ and if all $n$ numbers $\lambda^{1}%
,\lambda^{2},\ldots,\lambda^{n}$ are distinct from $1$, then $A$ is nilpotent.
\end{exercise}

\subsection{Powers and the Jordan canonical form}

Let $n\in\mathbb{N}$ and $A\in\mathbb{F}^{n\times n}$ for some field
$\mathbb{F}$. Assume that we know the JCF $J$ of $A$ (this always exists when
$\mathbb{F}=\mathbb{C}$, but sometimes exists for other fields as well) and an
invertible matrix $S\in\mathbb{F}^{n\times n}$ such that
\[
A=SJS^{-1}.
\]
Then, it is fairly easy to compute all powers $A^{m}$ of $A$. Indeed, recall that

\begin{itemize}
\item $\left(  SJS^{-1}\right)  ^{m}=SJ^{m}S^{-1}$ for any $m\in\mathbb{N}$.

\item $\left(
\begin{array}
[c]{cccc}%
A_{1} &  &  & \\
& A_{2} &  & \\
&  & \ddots & \\
&  &  & A_{k}%
\end{array}
\right)  ^{m}=\left(
\begin{array}
[c]{cccc}%
A_{1}^{m} &  &  & \\
& A_{2}^{m} &  & \\
&  & \ddots & \\
&  &  & A_{k}^{m}%
\end{array}
\right)  $ for any $m\in\mathbb{N}$.
\end{itemize}

Thus, it suffices to compute the $m$-th power of any Jordan cell $J_{k}\left(
\lambda\right)  $.

So let us consider a Jordan cell
\[
C:=J_{5}\left(  \lambda\right)  =\left(
\begin{array}
[c]{ccccc}%
\lambda & 1 &  &  & \\
& \lambda & 1 &  & \\
&  & \lambda & 1 & \\
&  &  & \lambda & 1\\
&  &  &  & \lambda
\end{array}
\right)  .
\]
Then,%
\begin{align*}
C^{2}  &  =\left(
\begin{array}
[c]{ccccc}%
\lambda^{2} & 2\lambda & 1 &  & \\
& \lambda^{2} & 2\lambda & 1 & \\
&  & \lambda^{2} & 2\lambda & 1\\
&  &  & \lambda^{2} & 2\lambda\\
&  &  &  & \lambda^{2}%
\end{array}
\right)  ;\ \ \ \ \ \ \ \ \ \ C^{3}=\left(
\begin{array}
[c]{ccccc}%
\lambda^{3} & 3\lambda^{2} & 3\lambda & 1 & \\
& \lambda^{3} & 3\lambda^{2} & 3\lambda & 1\\
&  & \lambda^{3} & 3\lambda^{2} & 3\lambda\\
&  &  & \lambda^{3} & 3\lambda^{2}\\
&  &  &  & \lambda^{3}%
\end{array}
\right)  ;\\
C^{4}  &  =\left(
\begin{array}
[c]{ccccc}%
\lambda^{4} & 4\lambda^{3} & 6\lambda^{2} & 4\lambda & 1\\
& \lambda^{4} & 4\lambda^{3} & 6\lambda^{2} & 4\lambda\\
&  & \lambda^{4} & 4\lambda^{3} & 6\lambda^{2}\\
&  &  & \lambda^{4} & 4\lambda^{3}\\
&  &  &  & \lambda^{4}%
\end{array}
\right)  ;\ \ \ \ \ \ \ \ \ \ C^{5}=\left(
\begin{array}
[c]{ccccc}%
\lambda^{5} & 5\lambda^{4} & 10\lambda^{3} & 10\lambda^{2} & 5\lambda\\
& \lambda^{5} & 5\lambda^{4} & 10\lambda^{3} & 10\lambda^{2}\\
&  & \lambda^{5} & 5\lambda^{4} & 10\lambda^{3}\\
&  &  & \lambda^{5} & 5\lambda^{4}\\
&  &  &  & \lambda^{5}%
\end{array}
\right)  .
\end{align*}
In general, we have the following:

\begin{theorem}
\label{thm.jnf.powers.Cm}Let $\mathbb{F}$ be a field. Let $k>0$ and
$\lambda\in\mathbb{F}$. Let $C=J_{k}\left(  \lambda\right)  $. Let
$m\in\mathbb{N}$. Then, $C^{m}$ is the upper-triangular $k\times k$-matrix
whose $\left(  i,j\right)  $-th entry is $\dbinom{m}{j-i}\lambda^{m-j+i}$ for
all $i,j\in\left[  k\right]  $. (Here, we follow the convention that
$\dbinom{m}{\ell}\lambda^{m-\ell}:=0$ when $\ell\notin\mathbb{N}$. Also,
recall that $\dbinom{n}{\ell}=0$ when $n\in\mathbb{N}$ and $\ell>n$.)
\end{theorem}

\begin{proof}
[First proof of Theorem \ref{thm.jnf.powers.Cm}.]Induct on $m$ and use
$C^{m}=CC^{m-1}$ as well as Pascal's recursion%
\[
\dbinom{n}{\ell}=\dbinom{n-1}{\ell}+\dbinom{n-1}{\ell-1}.
\]

\end{proof}

\begin{proof}
[Second proof of Theorem \ref{thm.jnf.powers.Cm}.]Set $B:=J_{k}\left(
0\right)  =\left(
\begin{array}
[c]{ccccc}
& 1 &  &  & \\
&  & 1 &  & \\
&  &  & \ddots & \\
&  &  &  & 1\\
&  &  &  &
\end{array}
\right)  $. Proposition \ref{prop.jnf.jcell0-powers} \textbf{(a)} tells us
what the powers of $B$ are: Namely, $B^{i}$ has $1$s $i$ steps above the main
diagonal, and $0$s everywhere else.

However, $C=B+\lambda I_{k}$. The matrices $\lambda I_{k}$ and $B$ commute
(i.e., we have $B\cdot\lambda I_{k}=\lambda I_{k}\cdot B$). It is a general
fact that if $X$ and $Y$ are two commuting $n\times n$-matrices, then the
binomial formula%
\[
\left(  X+Y\right)  ^{m}=\sum_{i=0}^{m}\dbinom{m}{i}X^{i}Y^{m-i}%
\ \ \ \ \ \ \ \ \ \ \text{holds.}%
\]
(This can be proved in the same way as for numbers, because the commutativity
of $X$ and $Y$ lets you move any $X$es past any $Y$s.) Applying this formula
to $X=B$ and $Y=\lambda I_{k}$, we obtain%
\begin{align*}
\left(  B+\lambda I_{k}\right)  ^{m}  &  =\sum_{i=0}^{m}\dbinom{m}{i}%
B^{i}\underbrace{\left(  \lambda I_{k}\right)  ^{m-i}}_{=\lambda^{m-i}I_{k}%
}=\sum_{i=0}^{m}\dbinom{m}{i}\lambda^{m-i}B^{i}\\
&  =\left(
\begin{array}
[c]{ccccccc}%
\lambda^{m} & \dbinom{m}{1}\lambda^{m-1} & \dbinom{m}{2}\lambda^{m-2} & \cdots
& \cdots & \cdots & \cdots\\
& \lambda^{m} & \dbinom{m}{1}\lambda^{m-1} & \dbinom{m}{2}\lambda^{m-2} &
\cdots & \cdots & \cdots\\
&  & \lambda^{m} & \dbinom{m}{1}\lambda^{m-1} & \dbinom{m}{2}\lambda^{m-2} &
\cdots & \cdots\\
&  &  & \lambda^{m} & \dbinom{m}{1}\lambda^{m-1} & \cdots & \cdots\\
&  &  &  & \lambda^{m} & \cdots & \vdots\\
&  &  &  &  & \ddots & \vdots\\
&  &  &  &  &  & \lambda^{m}%
\end{array}
\right)  ,
\end{align*}
which is precisely the matrix claimed in the theorem.
\end{proof}

Now we know how to take powers of Jordan cells, and therefore how to take
powers of any matrix that we know how to bring to a Jordan canonical form.

\begin{corollary}
\label{cor.jnf.powers.to0}Let $A\in\mathbb{C}^{n\times n}$. Then,
$\lim\limits_{m\rightarrow\infty}A^{m}=0$ if and only if all eigenvalues of
$A$ have absolute value $<1$.
\end{corollary}

\begin{proof}
$\Longrightarrow:$ Suppose that $\lim\limits_{m\rightarrow\infty}A^{m}=0$, and
let $\lambda$ be an eigenvalue of $A$. We must show that $\left\vert
\lambda\right\vert <1$.

Consider a nonzero eigenvector $x$ for eigenvalue $\lambda$. Thus, $Ax=\lambda
x$. Then, $A^{2}x=\lambda^{2}x$ (since $A^{2}x=A\underbrace{Ax}_{=\lambda
x}=\lambda\underbrace{Ax}_{=\lambda x}=\lambda\lambda x=\lambda^{2}x$) and
similarly $A^{3}x=\lambda^{3}x$ and $A^{4}x=\lambda^{4}x$ and so on. Thus,
\[
A^{m}x=\lambda^{m}x\ \ \ \ \ \ \ \ \ \ \text{for each }m\in\mathbb{N}.
\]


Now, as $m\rightarrow\infty$, the vector $A^{m}x$ goes to $0$ (since
$A^{m}\rightarrow0$). Thus, the vector $\lambda^{m}x$ goes to $0$ as well
(since $A^{m}x=\lambda^{m}x$ for each $m\in\mathbb{N}$). Since $x\neq0$, this
entails that the scalar $\lambda^{m}$ goes to $0$ as well. Hence, $\left\vert
\lambda\right\vert <1$ (because if $\left\vert \lambda\right\vert $ was
$\geq1$, then $\lambda^{m}$ would either oscillate along the unit circle, or
move further and further away from the origin). \medskip

$\Longleftarrow:$ Suppose that all eigenvalues of $A$ have absolute value $<1$.

Let $A=SJS^{-1}$ be the Jordan canonical form of $A$. Write $J$ as $\left(
\begin{array}
[c]{cccc}%
J_{1} &  &  & \\
& J_{2} &  & \\
&  & \ddots & \\
&  &  & J_{p}%
\end{array}
\right)  $, where $J_{1},J_{2},\ldots,J_{p}$ are Jordan cells.

It suffices to show that $\lim\limits_{m\rightarrow\infty}J_{h}^{m}=0$ for
each $h\in\left[  p\right]  $ (because this will yield $\lim
\limits_{m\rightarrow\infty}J^{m}=0$, and therefore
\[
\lim\limits_{m\rightarrow\infty}A^{m}=\lim\limits_{m\rightarrow\infty
}\underbrace{\left(  SJS^{-1}\right)  ^{m}}_{=SJ^{m}S^{-1}}=\lim
\limits_{m\rightarrow\infty}SJ^{m}S^{-1}=S\underbrace{\left(  \lim
\limits_{m\rightarrow\infty}J^{m}\right)  }_{=0}S^{-1}=0,
\]
which is what we want to show).

Fix some $h\in\left[  p\right]  $. Write $J_{h}$ as $J_{k}\left(
\lambda\right)  $, with $\left\vert \lambda\right\vert <1$. Theorem
\ref{thm.jnf.powers.Cm} thus yields that the powers $J_{h}^{m}$ of this matrix
$J_{h}$ have a very specific form; in particular, each $J_{h}^{m}$ is an
upper-triangular $k\times k$-matrix whose $\left(  i,j\right)  $-th entry is
$\dbinom{m}{j-i}\lambda^{m-j+i}$. Thus, we need to show that for each
$i,j\in\left[  k\right]  $, we have%
\[
\lim\limits_{m\rightarrow\infty}\dbinom{m}{j-i}\lambda^{m-j+i}=0.
\]
However, this follows from a standard asymptotics argument:%
\[
\underbrace{\dbinom{m}{j-i}}_{\substack{=\dfrac{m\left(  m-1\right)  \left(
m-2\right)  \cdots\left(  m-j+i\right)  }{\left(  j-i\right)  !}\\\text{(for
}i\leq j\text{; otherwise the claim is trivial)}}}\underbrace{\lambda^{m-j+i}%
}_{\substack{\text{exponential in }m\text{,}\\\text{with quotient }%
\lambda\text{ having absolute value }\left\vert \lambda\right\vert
<1}}\rightarrow0
\]
because exponential functions with a quotient of absolute value $<1$ converge
to $0$ faster than polynomials can go to $\infty$.
\end{proof}

\begin{exercise}
\fbox{6} Let $\lambda\in\mathbb{C}$. Let $n$ and $k$ be two positive integers.
Prove the following: \medskip

\textbf{(a)} If a $k\times k$-matrix $C$ has eigenvalue $\lambda$ with
algebraic multiplicity $k$ and geometric multiplicity $1$, then $C\sim
J_{k}\left(  \lambda\right)  $. \medskip

\textbf{(b)} We have $\left(  J_{k}\left(  \lambda\right)  \right)  ^{n}\sim
J_{k}\left(  \lambda^{n}\right)  $ if $\lambda\neq0$. \medskip

\textbf{(c)} If $A\in\mathbb{C}^{k\times k}$ is an invertible matrix such that
$A^{n}$ is diagonalizable, then $A$ is diagonalizable.
\end{exercise}

\begin{exercise}
\fbox{4} Let $\mathbb{F}$ be a field. Compute $\left(  J_{k}\left(
\lambda\right)  \right)  ^{-1}$ for any nonzero $\lambda\in\mathbb{F}$ and any
$k>0$.
\end{exercise}

\subsection{The minimal polynomial}

\textbf{Recall:} A polynomial $p\left(  t\right)  \in\mathbb{F}\left[
t\right]  $ (where $\mathbb{F}$ is any field, and $t$ is an indeterminate) is
said to be \emph{monic} if its leading coefficient is $1$ -- that is, if it
can be written in the form%
\begin{align*}
p\left(  t\right)   &  =t^{m}+p_{m-1}t^{m-1}+p_{m-2}t^{m-2}+\cdots+p_{0}%
t^{0}\\
&  \ \ \ \ \ \ \ \ \ \ \text{for some }m\in\mathbb{N}\text{ and }p_{0}%
,p_{1},\ldots,p_{m-1}\in\mathbb{F}.
\end{align*}


\begin{definition}
\label{def.jnf.mipo.annih}Given a matrix $A\in\mathbb{F}^{n\times n}$ and a
polynomial $p\left(  t\right)  \in\mathbb{F}\left[  t\right]  $, we say that
$p\left(  t\right)  $ \emph{annihilates} $A$ if $p\left(  A\right)  =0$.
\end{definition}

The Cayley--Hamilton theorem says that the characteristic polynomial $p_{A}$
of a square matrix $A$ always annihilates $A$. However, often there are
matrices that are annihilated by other -- sometimes simpler -- polynomials.

\begin{example}
The identity matrix $I_{n}$ is annihilated by the polynomial $p\left(
t\right)  :=t-1$, because%
\[
p\left(  I_{n}\right)  =I_{n}-I_{n}=0.
\]

\end{example}

\begin{example}
The matrix $\left(
\begin{array}
[c]{ccc}%
0 & 1 & 0\\
0 & 0 & 0\\
0 & 0 & 0
\end{array}
\right)  $ is annihilated by the polynomial $p\left(  t\right)  :=t^{2}$,
since its square is $0$.
\end{example}

\begin{example}
The diagonal matrix $\left(
\begin{array}
[c]{ccc}%
2 &  & \\
& 2 & \\
&  & 3
\end{array}
\right)  $ (where empty cells are understood to be filled with zeroes) is
annihilated by the polynomial $p\left(  t\right)  :=\left(  t-2\right)
\left(  t-3\right)  $, since%
\begin{align*}
p\left(  \left(
\begin{array}
[c]{ccc}%
2 &  & \\
& 2 & \\
&  & 3
\end{array}
\right)  \right)   &  =\left(  \left(
\begin{array}
[c]{ccc}%
2 &  & \\
& 2 & \\
&  & 3
\end{array}
\right)  -2I_{3}\right)  \left(  \left(
\begin{array}
[c]{ccc}%
2 &  & \\
& 2 & \\
&  & 3
\end{array}
\right)  -3I_{3}\right) \\
&  =\left(
\begin{array}
[c]{ccc}%
0 &  & \\
& 0 & \\
&  & 1
\end{array}
\right)  \left(
\begin{array}
[c]{ccc}%
-1 &  & \\
& -1 & \\
&  & 0
\end{array}
\right)  =0.
\end{align*}

\end{example}

\begin{theorem}
\label{thm.jnf.mipo.unique}Let $\mathbb{F}$ be a field. Let $A\in
\mathbb{F}^{n\times n}$ be an $n\times n$-matrix. Then, there is a
\textbf{unique} monic polynomial $q_{A}\left(  t\right)  $ of minimum degree
that annihilates $A$.
\end{theorem}

\begin{proof}
The Cayley--Hamilton theorem (Theorem \ref{thm.schurtri.ch.ch}) shows that
$p_{A}$ annihilates $A$. Since $p_{A}$ is monic, we thus conclude that there
exists \textbf{some} monic polynomial that annihilates $A$. Hence, there
exists such a polynomial of minimum degree.

It remains to show that it is unique. To do so, we let $q_{A}$ and
$\widetilde{q}_{A}$ be two monic polynomials of minimum degree that annihilate
$A$. Our goal then is to show that $q_{A}=\widetilde{q}_{A}$.

Assume the contrary. Thus, $q_{A}-\widetilde{q}_{A}\neq0$. However, the two
polynomials $q_{A}$ and $\widetilde{q}_{A}$ have the same degree (since both
have minimum degree) and the same leading coefficients (because they are both
monic). Thus, their difference $q_{A}-\widetilde{q}_{A}$ is a polynomial of
smaller degree than $q_{A}$ and $\widetilde{q}_{A}$; furthermore, this
difference $q_{a}-\widetilde{q}_{A}$ annihilates $A$ (because $\left(
q_{A}-\widetilde{q}_{A}\right)  \left(  A\right)  =q_{A}\left(  A\right)
-\widetilde{q}_{A}\left(  A\right)  =0-0=0$). Thus, by scaling this difference
by an appropriate scalar in $\mathbb{F}$, we can make it monic (since it is
nonzero), and of course it will still annihilate $A$. Therefore, we obtain a
monic polynomial of smaller degree than $q_{A}$ that annihilates $A$. This
contradicts the minimality of $q_{A}$'s degree. This concludes the proof of
Theorem \ref{thm.jnf.mipo.unique}.
\end{proof}

\begin{definition}
\label{def.jnf.mipo.mipo}Let $A\in\mathbb{F}^{n\times n}$ be an $n\times
n$-matrix. Theorem \ref{thm.jnf.mipo.unique} shows that there is a
\textbf{unique} monic polynomial $q_{A}\left(  t\right)  $ of minimum degree
that annihilates $A$. This unique polynomial will be denoted $q_{A}\left(
t\right)  $ and will be called the \emph{minimal polynomial} of $A$.
\end{definition}

\begin{example}
Let $\mathbb{F}=\mathbb{C}$. Let $A$ be the diagonal matrix $\left(
\begin{array}
[c]{ccc}%
2 &  & \\
& 2 & \\
&  & 3
\end{array}
\right)  $ (where empty cells are supposed to contain $0$ entries). Then,%
\[
q_{A}\left(  t\right)  =\left(  t-2\right)  \left(  t-3\right)  .
\]
Indeed, we already know that the monic polynomial $\left(  t-2\right)  \left(
t-3\right)  $ annihilates $A$. If there was any monic polynomial of smaller
degree that would annihilate $A$, then it would have the form $t-\lambda$ for
some $\lambda\in\mathbb{F}$, but $\lambda$ cannot be $2$ and $3$ at the same time.

For comparison: The characteristic polynomial of $A$ is $p_{A}\left(
t\right)  =\left(  t-2\right)  ^{2}\left(  t-3\right)  $.
\end{example}

\begin{exercise}
\fbox{2} Find the minimal polynomial of a diagonal matrix whose
\textbf{distinct} diagonal entries are $\lambda_{1},\lambda_{2},\ldots
,\lambda_{k}$. (Each of these $\lambda_{1},\lambda_{2},\ldots,\lambda_{k}$ can
appear on the diagonal any positive number of times.)
\end{exercise}

\begin{exercise}
\fbox{3} Let $\mathbb{F}$ be a field. Let $n\geq2$ be an integer. Let
$x_{1},x_{2},\ldots,x_{n}\in\mathbb{F}$ and $y_{1},y_{2},\ldots,y_{n}%
\in\mathbb{F}$. Let $A$ be the $n\times n$-matrix $\left(
\begin{array}
[c]{cccc}%
x_{1}y_{1} & x_{1}y_{2} & \cdots & x_{1}y_{n}\\
x_{2}y_{1} & x_{2}y_{2} & \cdots & x_{2}y_{n}\\
\vdots & \vdots & \ddots & \vdots\\
x_{n}y_{1} & x_{n}y_{2} & \cdots & x_{n}y_{n}%
\end{array}
\right)  \in\mathbb{F}^{n\times n}$. \medskip

\textbf{(a)} Find the minimal polynomial of $A$ under the assumption that
$x_{1},x_{2},\ldots,x_{n}$ and $y_{1},y_{2},\ldots,y_{n}$ are nonzero.
\medskip

\textbf{(b)} What changes if we drop this assumption? \medskip

[\textbf{Hint:} Compute $A^{2}$.]
\end{exercise}

\begin{theorem}
\label{thm.jnf.mipo.annih-iff-mult}Let $A\in\mathbb{F}^{n\times n}$ be an
$n\times n$-matrix. Let $f\left(  t\right)  \in\mathbb{F}\left[  t\right]  $
be any polynomial. Then, $f$ annihilates $A$ if and only if $f$ is a multiple
of $q_{A}$ (that is, $f\left(  t\right)  =q_{A}\left(  t\right)  \cdot
g\left(  t\right)  $ for some polynomial $g\left(  t\right)  \in
\mathbb{F}\left[  t\right]  $).
\end{theorem}

\begin{proof}
$\Longrightarrow:$ Assume that $f$ annihilates $A$. Thus, $f\left(  A\right)
=0$. WLOG, assume that $f\neq0$. Thus, we can make $f$ monic by scaling it.
Thus, $\deg f\geq\deg q_{A}$ (since $q_{A}$ had minimum degree). Hence, we can
divide $f$ by $q_{A}$ with remainder, obtaining%
\begin{equation}
f\left(  t\right)  =q_{A}\left(  t\right)  \cdot g\left(  t\right)  +r\left(
t\right)  , \label{pf.thm.jnf.mipo.annih-iff-mult.1}%
\end{equation}
where $g\left(  t\right)  $ and $r\left(  t\right)  $ are two polynomials with
$\deg r<\deg q_{A}$. (Note that $r\left(  t\right)  $ is allowed to be the
zero polynomial.) Consider these $g\left(  t\right)  $ and $r\left(  t\right)
$.

Substituting $A$ for $t$ in the equality
(\ref{pf.thm.jnf.mipo.annih-iff-mult.1}) (and using Lemma
\ref{lem.p(A).multiplicative} \textbf{(a)}), we obtain%
\[
f\left(  A\right)  =\underbrace{q_{A}\left(  A\right)  }%
_{\substack{=0\\\text{(since }q_{A}\text{ annihilates }A\text{)}}}\cdot
g\left(  A\right)  +r\left(  A\right)  =r\left(  A\right)  ,
\]
so that $r\left(  A\right)  =f\left(  A\right)  =0$. In other words, $r$
annihilates $A$. Since $\deg r<\deg q_{A}$, this entails that $r=0$ (since
otherwise, we could scale the polynomial $r\left(  t\right)  $ to make it
monic, and then we would obtain a monic polynomial of degree $\deg r<\deg
q_{A}$ that annihilates $A$; but this would contradict the minimality of $\deg
q_{A}$). Thus,%
\[
f\left(  t\right)  =q_{A}\left(  t\right)  \cdot g\left(  t\right)
+\underbrace{r\left(  t\right)  }_{=0}=q_{A}\left(  t\right)  \cdot g\left(
t\right)  .
\]
Thus, $f$ is a multiple of $q_{A}$. \medskip

$\Longleftarrow:$ Easy and LTTR.
\end{proof}

\begin{corollary}
\label{cor.jnf.mipo.div-charpol}Let $\mathbb{F}$ be a field. Let
$A\in\mathbb{F}^{n\times n}$ be a matrix. Then, $q_{A}\left(  t\right)  \mid
p_{A}\left(  t\right)  $.
\end{corollary}

\begin{proof}
Apply the previous theorem to $f=p_{A}$, recalling that $p_{A}$ annihilates
$A$.
\end{proof}

The corollary yields that any root of $q_{A}$ must be a root of $p_{A}$, that
is, an eigenvalue of $A$. Conversely, we can show that any eigenvalue of $A$
is a root of $q_{A}$ (but we don't know with which multiplicity):

\begin{proposition}
\label{prop.jnf.mipo.qlam=0}Let $A\in\mathbb{C}^{n\times n}$ be an $n\times
n$-matrix. If $\lambda\in\sigma\left(  A\right)  $, then $q_{A}\left(
\lambda\right)  =0$.
\end{proposition}

\begin{proof}
Let $\lambda\in\sigma\left(  A\right)  $. Thus, there exists a nonzero
eigenvector $x$ for $\lambda$.

Then, $Ax=\lambda x$. As we have seen above, this entails $A^{m}x=\lambda
^{m}x$ for each $m\in\mathbb{N}$. Therefore, $f\left(  A\right)  x=f\left(
\lambda\right)  x$ for each polynomial $f\left(  t\right)  \in\mathbb{C}%
\left[  t\right]  $ (because you can write $f\left(  t\right)  $ as
$f_{0}t^{0}+f_{1}t^{1}+\cdots+f_{p}t^{p}$, and then apply $A^{m}x=\lambda
^{m}x$ to each of $m=0,1,\ldots,p$). Hence, $q_{A}\left(  A\right)
x=q_{A}\left(  \lambda\right)  x$, so that%
\[
q_{A}\left(  \lambda\right)  x=\underbrace{q_{A}\left(  A\right)
}_{\substack{=0\\\text{(since }q_{A}\text{ annihilates }A\text{)}}}x=0.
\]
Since $x\neq0$, this entails $q_{A}\left(  \lambda\right)  =0$, qed.
\end{proof}

Combining Corollary \ref{cor.jnf.mipo.div-charpol} with Proposition
\ref{prop.jnf.mipo.qlam=0}, we see that the roots of $q_{A}\left(  t\right)  $
are precisely the eigenvalues of $A$ (when $A\in\mathbb{C}^{n\times n}$); we
just don't know yet with which multiplicities they appear as roots. In other
words, we have
\[
q_{A}\left(  t\right)  =\left(  t-\lambda_{1}\right)  ^{k_{1}}\left(
t-\lambda_{2}\right)  ^{k_{2}}\cdots\left(  t-\lambda_{p}\right)  ^{k_{p}},
\]
where $\lambda_{1},\lambda_{2},\ldots,\lambda_{p}$ are the distinct
eigenvalues of $A$, and the $k_{1},k_{2},\ldots,k_{p}$ are positive integers;
but we don't know these $k_{1},k_{2},\ldots,k_{p}$ yet. So let us find them.
We will use some lemmas for this.

\begin{lemma}
\label{lem.jnf.mipo.similar}Let $\mathbb{F}$ be a field. Let $A$ and $B$ be
two similar $n\times n$-matrices in $\mathbb{F}^{n\times n}$. Then,
$q_{A}\left(  t\right)  =q_{B}\left(  t\right)  $.
\end{lemma}

\begin{proof}
This is obvious from the viewpoint of endomorphisms (see Remark
\ref{rmk.schurtri.similar.endo-vp}). For a pedestrian proof, you can just
argue that a polynomial $f$ annihilates $A$ if and only if it annihilates $B$.
But this is easy: We have $A=SBS^{-1}$ for some invertible $S$ (since $A$ and
$B$ are similar), and therefore every polynomial $f$ satisfies
\[
f\left(  A\right)  =f\left(  SBS^{-1}\right)  =Sf\left(  B\right)  S^{-1}%
\]
and therefore $f\left(  A\right)  =0$ holds if and only if $f\left(  B\right)
=0$.
\end{proof}

We recall the notion of the lcm (= least common multiple) of several
polynomials. It is defined as one would expect: If $p_{1},p_{2},\ldots,p_{m}$
are $m$ nonzero polynomials (in a single indeterminate $t$), then
$\operatorname{lcm}\left(  p_{1},p_{2},\ldots,p_{m}\right)  $ is the monic
polynomial of smallest degree that is a common multiple of $p_{1},p_{2}%
,\ldots,p_{m}$. For example,%
\begin{align*}
\operatorname{lcm}\left(  t^{2}-1,\ t^{3}-1\right)   &  =\operatorname{lcm}%
\left(  \left(  t-1\right)  \left(  t+1\right)  ,\ \left(  t-1\right)  \left(
t^{2}+t+1\right)  \right) \\
&  =\left(  t-1\right)  \left(  t+1\right)  \left(  t+t^{2}+1\right)
=t^{4}+t^{3}-t-1.
\end{align*}
(Again, the lcm of several polynomials is unique. This can be shown in the
same way that we used to prove uniqueness of the minimal polynomial.)

Our next lemma tells us what the minimal polynomial of a block-diagonal matrix is:

\begin{lemma}
\label{lem.jnf.mipo.block-diag}Let $A_{1},A_{2},\ldots,A_{m}$ be $m$ square
matrices. Let%
\[
A=\left(
\begin{array}
[c]{cccc}%
A_{1} &  &  & \\
& A_{2} &  & \\
&  & \ddots & \\
&  &  & A_{m}%
\end{array}
\right)  .
\]
Then,%
\[
q_{A}=\operatorname{lcm}\left(  q_{A_{1}},\ q_{A_{2}},\ \ldots,\ q_{A_{m}%
}\right)  .
\]

\end{lemma}

\begin{proof}
For any polynomial $f\in\mathbb{F}\left[  t\right]  $, we have%
\[
f\left(  A\right)  =f\left(
\begin{array}
[c]{cccc}%
A_{1} &  &  & \\
& A_{2} &  & \\
&  & \ddots & \\
&  &  & A_{m}%
\end{array}
\right)  =\left(
\begin{array}
[c]{cccc}%
f\left(  A_{1}\right)  &  &  & \\
& f\left(  A_{2}\right)  &  & \\
&  & \ddots & \\
&  &  & f\left(  A_{m}\right)
\end{array}
\right)
\]
(indeed, the last equality follows from%
\[
\left(
\begin{array}
[c]{cccc}%
A_{1} &  &  & \\
& A_{2} &  & \\
&  & \ddots & \\
&  &  & A_{m}%
\end{array}
\right)  ^{k}=\left(
\begin{array}
[c]{cccc}%
A_{1}^{k} &  &  & \\
& A_{2}^{k} &  & \\
&  & \ddots & \\
&  &  & A_{m}^{k}%
\end{array}
\right)
\]
and from the fact that a polynomial $f$ is just a $\mathbb{F}$-linear
combination of $t^{k}$s). Thus, $f\left(  A\right)  =0$ holds if and only if%
\[
f\left(  A_{1}\right)  =0\text{ and }f\left(  A_{2}\right)  =0\text{ and
}\cdots\text{ and }f\left(  A_{m}\right)  =0.
\]
However, $f\left(  A\right)  =0$ holds if and only if $f$ is a multiple of
$q_{A}$, whereas $f\left(  A_{i}\right)  =0$ holds if and only if $f$ is a
multiple of $q_{A_{i}}$. Thus, the previous sentence says that $f$ is a
multiple of $q_{A}$ if and only if $f$ is a multiple of all of the $q_{A_{i}}%
$s. In other words, the multiples of $q_{A}$ are precisely the common
multiples of all the $q_{A_{i}}$s. Therefore, $q_{A}$ is the lcm of the
$q_{A_{i}}$s (because the universal property of an lcm characterizes the lcm
of the $q_{A_{i}}$s as the unique monic polynomial whose multiples are the
common multiples of all the $q_{A_{i}}$s).
\end{proof}

\begin{lemma}
\label{lem.jnf.mipo.Jklam}Let $\mathbb{F}$ be a field. Let $k>0$ and
$\lambda\in\mathbb{F}$. Let $A=J_{k}\left(  \lambda\right)  $. Then,%
\[
q_{A}=\left(  t-\lambda\right)  ^{k}.
\]

\end{lemma}

\begin{proof}
It is easy to see that $q_{A}=q_{A-\lambda I_{k}}\left(  t-\lambda\right)  $,
because for a polynomial $f\in\mathbb{F}\left[  t\right]  $ to annihilate
$A-\lambda I_{k}$ is the same as for the polynomial $f\left(  t-\lambda
\right)  $ to annihilate $A$. So we need to find $q_{A-\lambda I_{k}}$. Recall
(from Proposition \ref{prop.jnf.jcell-lambda}) that
\[
A-\lambda I_{k}=J_{k}\left(  0\right)  =\left(
\begin{array}
[c]{ccccc}
& 1 &  &  & \\
&  & 1 &  & \\
&  &  & \ddots & \\
&  &  &  & 1\\
&  &  &  &
\end{array}
\right)  .
\]
Therefore, for any polynomial $f=f_{0}t^{0}+f_{1}t^{1}+f_{2}t^{2}+\cdots$, we
have%
\[
f\left(  A-\lambda I_{k}\right)  =\left(
\begin{array}
[c]{ccccc}%
f_{0} & f_{1} & f_{2} & \cdots & f_{k-1}\\
& f_{0} & f_{1} & \cdots & f_{k-2}\\
&  & f_{0} & \cdots & f_{k-3}\\
&  &  & \ddots & \vdots\\
&  &  &  & f_{0}%
\end{array}
\right)  .
\]
So $f\left(  A-\lambda I_{k}\right)  =0$ if and only if $f_{0}=f_{1}%
=\cdots=f_{k-1}=0$, i.e., if and only if the first $k$ coefficients of $f$ are
$0$. Now, the monic polynomial of smallest degree whose first $k$ coefficients
are $0$ is the polynomial $t^{k}$. So the monic polynomial $f$ of smallest
degree that satisfies $f\left(  A-\lambda I_{k}\right)  =0$ is $t^{k}$. In
other words, $q_{A-\lambda I_{k}}=t^{k}$.

Now, recall that $q_{A}=q_{A-\lambda I_{k}}\left(  t-\lambda\right)  =\left(
t-\lambda\right)  ^{k}$ (since $q_{A-\lambda I_{k}}=t^{k}$). This proves Lemma
\ref{lem.jnf.mipo.Jklam}.
\end{proof}

\begin{theorem}
\label{thm.jnf.mipo.jnf}Let $A\in\mathbb{C}^{n\times n}$ be an $n\times
n$-matrix. Let $J$ be the Jordan canonical form of $A$. Let $\lambda
_{1},\lambda_{2},\ldots,\lambda_{p}$ be the distinct eigenvalues of $A$. Then,%
\[
q_{A}=\left(  t-\lambda_{1}\right)  ^{k_{1}}\left(  t-\lambda_{2}\right)
^{k_{2}}\cdots\left(  t-\lambda_{p}\right)  ^{k_{p}},
\]
where $k_{i}$ is the size of the largest Jordan cell at eigenvalue
$\lambda_{i}$ in $J$.
\end{theorem}

\begin{example}
Let $A$ have Jordan canonical form%
\[
J=\left(
\begin{array}
[c]{cccccccc}%
5 & 1 &  &  &  &  &  & \\
& 5 & 1 &  &  &  &  & \\
&  & 5 &  &  &  &  & \\
&  &  & 5 & 1 &  &  & \\
&  &  &  & 5 &  &  & \\
&  &  &  &  & 8 &  & \\
&  &  &  &  &  & 8 & 1\\
&  &  &  &  &  &  & 8
\end{array}
\right)  .
\]
Then,%
\[
q_{A}=\left(  t-5\right)  ^{3}\left(  t-8\right)  ^{2}.
\]

\end{example}

\begin{proof}
[Proof of Theorem \ref{thm.jnf.mipo.jnf}.]We have $A\sim J$, so that
$q_{A}=q_{J}$ (by Lemma \ref{lem.jnf.mipo.similar}).

Recall that $J$ is a Jordan matrix, i.e., a block-diagonal matrix whose
diagonal blocks are Jordan cells $J_{1},J_{2},\ldots,J_{m}$. Thus, by Lemma
\ref{lem.jnf.mipo.block-diag}, we have%
\begin{align*}
q_{J}  &  =\operatorname{lcm}\left(  q_{J_{1}},\ q_{J_{2}},\ \ldots
,\ q_{J_{m}}\right) \\
&  =\operatorname{lcm}\left(  \left(  t-\lambda_{J_{1}}\right)  ^{k_{J_{1}}%
},\ \left(  t-\lambda_{J_{2}}\right)  ^{k_{J_{2}}},\ \ldots,\ \left(
t-\lambda_{J_{m}}\right)  ^{k_{J_{m}}}\right)  ,
\end{align*}
where each $J_{i}$ has eigenvalue $\lambda_{J_{i}}$ and size $k_{J_{i}}$ (by
Lemma \ref{lem.jnf.mipo.Jklam}). This lcm must be divisible by each
$t-\lambda$ at least as often as each of the $\left(  t-\lambda_{J_{i}%
}\right)  ^{k_{J_{i}}}$s is; i.e., it must be divisible by $\left(
t-\lambda\right)  ^{k}$, where $k$ is the largest size of a Jordan cell of $J$
at eigenvalue $\lambda$. So the lcm is the product of these $\left(
t-\lambda\right)  ^{k}$s. But this is precisely our claim.
\end{proof}

\begin{exercise}
\fbox{1} Let $\mathbb{F}$ be a field. Let $A\in\mathbb{F}^{n\times n}$ be any
$n\times n$-matrix. \medskip

\textbf{(a)} Prove that $q_{A^{T}}=q_{A}$, where $A^{T}$ denotes the transpose
of the matrix $A$. \medskip

\textbf{(b)} Assume that $\mathbb{F}=\mathbb{C}$. Prove that $q_{A^{\ast}%
}=\overline{q_{A}}$, where $\overline{q_{A}}$ denotes the result of replacing
all coefficients of the polynomial $q_{A}$ by their complex conjugates.
\end{exercise}

\begin{exercise}
\fbox{5} \textbf{(a)} A matrix $A\in\mathbb{C}^{3\times3}$ has characteristic
polynomial $t\left(  t-1\right)  \left(  t-2\right)  $. What can its JCF be?
\medskip

\textbf{(b)} A matrix $A\in\mathbb{C}^{3\times3}$ has characteristic
polynomial $t^{2}\left(  t-2\right)  $. What can its JCF be? \medskip

\textbf{(c)} A matrix $A\in\mathbb{C}^{3\times3}$ has minimal polynomial
$t^{2}\left(  t-2\right)  $. What can its JCF be? \medskip

\textbf{(d)} A matrix $A\in\mathbb{C}^{3\times3}$ has minimal polynomial
$t\left(  t-2\right)  $. What can its JCF be?
\end{exercise}

\begin{exercise}
\fbox{5} Let $A\in\mathbb{C}^{n\times n}$ be a matrix. Prove that $A^{T}\sim
A$, where $A^{T}$ denotes the transpose of the matrix $A$. \medskip

[\textbf{Hint:} Reduce to the case of a Jordan cell.]
\end{exercise}

\subsection{Application of functions to matrices}

Consider a square matrix $A\in\mathbb{C}^{n\times n}$. We have already defined
what it means to apply a polynomial $f$ to $A$: We just write $f$ as $\sum
_{i}f_{i}t^{i}$, and substitute $A$ for $t$.

Can we do the same with non-polynomial functions $f$ ? For example, can we
define $\exp A$ or $\sin A$ ?

One option to do so is to follow the same rule as for polynomials, but using
the Taylor series for $f$. For example, since $\exp$ has Taylor series $\exp
t=\sum_{i\in\mathbb{N}}\dfrac{t^{i}}{i!}$, we can set%
\[
\exp A=\sum_{i\in\mathbb{N}}\dfrac{A^{i}}{i!}.
\]
This indeed works for $\exp$ and for $\sin$, as the sums you get always
converge. But it doesn't generally work, e.g., for $f=\tan$, since its Taylor
series only converges in a certain neighborhood of $0$. Is this the best we
can do?

There is a different approach that gives a more general definition. We begin
with a lemma about Jordan cells:

\begin{lemma}
\label{lem.jnf.applying-funcs.Jklam}Let $k>0$ and $\lambda\in\mathbb{C}$. Let
$A=J_{k}\left(  \lambda\right)  $. Then, for any polynomial $f\in
\mathbb{C}\left[  t\right]  $, we have%
\[
f\left(  A\right)  =\left(
\begin{array}
[c]{ccccc}%
\dfrac{f\left(  \lambda\right)  }{0!} & \dfrac{f^{\prime}\left(
\lambda\right)  }{1!} & \dfrac{f^{\prime\prime}\left(  \lambda\right)  }{2!} &
\cdots & \dfrac{f^{\left(  k-1\right)  }\left(  \lambda\right)  }{\left(
k-1\right)  !}\\
& \dfrac{f\left(  \lambda\right)  }{0!} & \dfrac{f^{\prime}\left(
\lambda\right)  }{1!} & \cdots & \dfrac{f^{\left(  k-2\right)  }\left(
\lambda\right)  }{\left(  k-2\right)  !}\\
&  & \dfrac{f\left(  \lambda\right)  }{0!} & \cdots & \dfrac{f^{\left(
k-3\right)  }\left(  \lambda\right)  }{\left(  k-3\right)  !}\\
&  &  & \ddots & \vdots\\
&  &  &  & \dfrac{f\left(  \lambda\right)  }{0!}%
\end{array}
\right)
\]

\end{lemma}

\begin{exercise}
\fbox{2} Prove Lemma \ref{lem.jnf.applying-funcs.Jklam}.
\end{exercise}

Now, we aim to define $f\left(  A\right)  $ by the above formula, at least
when $A$ is a Jordan cell. This only requires $f$ to be $\left(  k-1\right)
$-times differentiable at $\lambda$.

\begin{definition}
Let $A\in\mathbb{C}^{n\times n}$ be an $n\times n$-matrix that has minimal
polynomial%
\[
q_{A}\left(  t\right)  =\left(  t-\lambda_{1}\right)  ^{k_{1}}\left(
t-\lambda_{2}\right)  ^{k_{2}}\cdots\left(  t-\lambda_{p}\right)  ^{k_{p}},
\]
where the $\lambda_{1},\lambda_{2},\ldots,\lambda_{p}$ are the distinct
eigenvalues of $A$.

Let $f$ be a function from $\mathbb{C}$ to $\mathbb{C}$ that is defined at
each of the numbers $\lambda_{1},\lambda_{2},\ldots,\lambda_{p}$ and is
holomorphic at each of them, or at least $\left(  k_{i}-1\right)  $-times
differentiable at each $\lambda_{i}$ if $\lambda_{i}$ is real. Then, we can
define an $n\times n$-matrix $f\left(  A\right)  \in\mathbb{C}^{n\times n}$ as
follows: Write $A=SJS^{-1}$, where $J$ is a Jordan matrix and $S$ is
invertible. Write $J$ as $\left(
\begin{array}
[c]{cccc}%
J_{1} &  &  & \\
& J_{2} &  & \\
&  & \ddots & \\
&  &  & J_{m}%
\end{array}
\right)  $, where the $J_{1},J_{2},\ldots,J_{m}$ are Jordan cells. Then, we
set%
\begin{align*}
f\left(  A\right)   &  :=Sf\left(  J\right)  S^{-1}%
,\ \ \ \ \ \ \ \ \ \ \text{where}\\
f\left(  J\right)   &  :=\left(
\begin{array}
[c]{cccc}%
f\left(  J_{1}\right)  &  &  & \\
& f\left(  J_{2}\right)  &  & \\
&  & \ddots & \\
&  &  & f\left(  J_{m}\right)
\end{array}
\right)  ,\ \ \ \ \ \ \ \ \ \ \text{where}\\
f\left(  J_{k}\left(  \lambda\right)  \right)   &  :=\left(
\begin{array}
[c]{ccccc}%
\dfrac{f\left(  \lambda\right)  }{0!} & \dfrac{f^{\prime}\left(
\lambda\right)  }{1!} & \dfrac{f^{\prime\prime}\left(  \lambda\right)  }{2!} &
\cdots & \dfrac{f^{\left(  k-1\right)  }\left(  \lambda\right)  }{\left(
k-1\right)  !}\\
& \dfrac{f\left(  \lambda\right)  }{0!} & \dfrac{f^{\prime}\left(
\lambda\right)  }{1!} & \cdots & \dfrac{f^{\left(  k-2\right)  }\left(
\lambda\right)  }{\left(  k-2\right)  !}\\
&  & \dfrac{f\left(  \lambda\right)  }{0!} & \cdots & \dfrac{f^{\left(
k-3\right)  }\left(  \lambda\right)  }{\left(  k-3\right)  !}\\
&  &  & \ddots & \vdots\\
&  &  &  & \dfrac{f\left(  \lambda\right)  }{0!}%
\end{array}
\right)  .
\end{align*}

\end{definition}

\begin{theorem}
This definition is actually well-defined. That is, the value $f\left(
A\right)  $ does not depend on the choice of $S$ and $J$.
\end{theorem}

\begin{exercise}
\fbox{5} Prove this.

[\textbf{Hint:} Use Hermite interpolation to find a polynomial $g\in
\mathbb{C}\left[  t\right]  $ such that $g^{\left(  m\right)  }\left(
\lambda\right)  =f^{\left(  m\right)  }\left(  \lambda\right)  $ for each
$\lambda\in\sigma\left(  A\right)  $ and each $m\in\left\{  0,1,\ldots
,m_{\lambda}-1\right\}  $, where $m_{\lambda}$ is the algebraic multiplicity
of $\lambda$ as an eigenvalue of $A$.]
\end{exercise}

\subsection{The companion matrix}

For each $n\times n$-matrix $A$, we have defined its characteristic polynomial
$p_{A}$ and its minimal polynomial $q_{A}$. What variety of polynomials do we
get this way? Do all characteristic polynomials share some property, or can
any monic polynomial be a characteristic polynomial?

The latter turns out to be true (and moreover, the same holds for the minimal
polynomial). We shall prove this by explicitly constructing a matrix with a
given polynomial as its characteristic polynomial.

\begin{definition}
\label{def.jnf.companion.Cf}Let $\mathbb{F}$ be a field, and let
$n\in\mathbb{N}$.

Let $f\left(  t\right)  =t^{n}+f_{n-1}t^{n-1}+f_{n-2}t^{n-2}+\cdots+f_{1}%
t^{1}+f_{0}t^{0}$ be a monic polynomial of degree $n$ with coefficients in
$\mathbb{F}$. Then, the \emph{companion matrix} of $f\left(  t\right)  $ is
defined to be the matrix%
\[
C_{f}:=\left(
\begin{array}
[c]{cccccc}%
0 &  &  &  &  & -f_{0}\\
1 & 0 &  &  &  & -f_{1}\\
& 1 & 0 &  &  & -f_{2}\\
&  & 1 & \ddots &  & \vdots\\
&  &  & \ddots & 0 & -f_{n-2}\\
&  &  &  & 1 & -f_{n-1}%
\end{array}
\right)  \in\mathbb{F}^{n\times n}%
\]
(where each cell that is left empty is supposed to be filled with a $0$). This
is the $n\times n$-matrix whose first $n-1$ columns are the standard basis
vectors $e_{2},e_{3},\ldots,e_{n}$, and whose last column is $\left(
-f_{0},-f_{1},\ldots,-f_{n-1}\right)  ^{T}$.
\end{definition}

\begin{proposition}
\label{prop.jnf.companion.pCf=qCf=f}For any monic polynomial $f\left(
t\right)  $, we have%
\[
p_{C_{f}}\left(  t\right)  =q_{C_{f}}\left(  t\right)  =f\left(  t\right)  .
\]

\end{proposition}

\begin{proof}
Let us first show that $p_{C_{f}}\left(  t\right)  =f\left(  t\right)  $.

To do so, we induct on $n$. The \textit{base case} (that is, the case $n=0$)
is obvious (since the determinant of the $0\times0$-matrix is $1$ by
definition). Let us thus proceed to the \textit{induction step}: Let $n$ be a
positive integer. Let $f\left(  t\right)  =t^{n}+f_{n-1}t^{n-1}+f_{n-2}%
t^{n-2}+\cdots+f_{1}t^{1}+f_{0}t^{0}$ be a monic polynomial of degree $n$ with
coefficients in $\mathbb{F}$. We must show that $p_{C_{f}}\left(  t\right)
=f\left(  t\right)  $. We assume (as the induction hypothesis) that the same
holds for all monic polynomials of degree $n-1$.

Let $g\left(  t\right)  $ be the polynomial $t^{n-1}+f_{n-1}t^{n-2}%
+\cdots+f_{2}t^{1}+f_{1}t^{0}$. This is a monic polynomial of degree $n-1$;
thus, we can apply the induction hypothesis to it. Thus, we conclude that
$p_{C_{g}}\left(  t\right)  =g\left(  t\right)  $.

The definition of the characteristic polynomial yields%
\[
p_{C_{f}}\left(  t\right)  =\det\left(  tI_{n}-C_{f}\right)  =\det\left(
\begin{array}
[c]{cccccc}%
t &  &  &  &  & f_{0}\\
-1 & t &  &  &  & f_{1}\\
& -1 & t &  &  & f_{2}\\
&  & -1 & \ddots &  & \vdots\\
&  &  & \ddots & t & f_{n-2}\\
&  &  &  & -1 & t+f_{n-1}%
\end{array}
\right)  .
\]
We compute this determinant by Laplace expansion along the first row
(exploiting the fact that only two entries of this first row are nonzero):%
\begin{align*}
&  \det\left(
\begin{array}
[c]{cccccc}%
t &  &  &  &  & f_{0}\\
-1 & t &  &  &  & f_{1}\\
& -1 & t &  &  & f_{2}\\
&  & -1 & \ddots &  & \vdots\\
&  &  & \ddots & t & f_{n-2}\\
&  &  &  & -1 & t+f_{n-1}%
\end{array}
\right) \\
&  =t\ \det\underbrace{\left(
\begin{array}
[c]{ccccc}%
t &  &  &  & f_{1}\\
-1 & t &  &  & f_{2}\\
& -1 & \ddots &  & \vdots\\
&  & \ddots & t & f_{n-2}\\
&  &  & -1 & t+f_{n-1}%
\end{array}
\right)  }_{\substack{=tI_{n-1}-C_{g}\\\text{(by the definition of }%
C_{g}\text{)}}}+\left(  -1\right)  ^{n+1}f_{0}\ \underbrace{\det\left(
\begin{array}
[c]{ccccc}%
-1 & t &  &  & \\
& -1 & t &  & \\
&  & -1 & \ddots & \\
&  &  & \ddots & t\\
&  &  &  & -1
\end{array}
\right)  }_{\substack{=\left(  -1\right)  ^{n-1}\\\text{(since this matrix is
upper-triangular of}\\\text{size }n-1\text{, and its diagonal entries are
}-1\text{'s)}}}\\
&  =t\ \underbrace{\det\left(  tI_{n-1}-C_{g}\right)  }_{\substack{=p_{C_{g}%
}\left(  t\right)  \\\text{(by the definition of the characteristic
polynomial)}}}+\left(  -1\right)  ^{n+1}\underbrace{f_{0}\left(  -1\right)
^{n-1}}_{=\left(  -1\right)  ^{n-1}f_{0}}\\
&  =t\underbrace{p_{C_{g}}\left(  t\right)  }_{\substack{=g\left(  t\right)
\\=t^{n-1}+f_{n-1}t^{n-2}+\cdots+f_{2}t^{1}+f_{1}t^{0}}}+\underbrace{\left(
-1\right)  ^{n+1}\left(  -1\right)  ^{n-1}}_{=1}f_{0}\\
&  =t\left(  t^{n-1}+f_{n-1}t^{n-2}+\cdots+f_{2}t^{1}+f_{1}t^{0}\right)
+f_{0}\\
&  =t^{n}+f_{n-1}t^{n-1}+f_{2}t^{2}+f_{1}t^{1}+f_{0}=f\left(  t\right)  .
\end{align*}
Thus, $p_{C_{f}}\left(  t\right)  =f\left(  t\right)  $ is proved. This
completes the induction step. Hence, we have proved the $p_{C_{f}}\left(
t\right)  =f\left(  t\right)  $ part of Proposition
\ref{prop.jnf.companion.pCf=qCf=f}.

Now, let us show that $q_{C_{f}}\left(  t\right)  =f\left(  t\right)  $.
Indeed, both $q_{C_{f}}\left(  t\right)  $ and $f\left(  t\right)  $ are monic
polynomials, and we know from Corollary \ref{cor.jnf.mipo.div-charpol} that
$q_{C_{f}}\left(  t\right)  \mid p_{C_{f}}\left(  t\right)  =f\left(
t\right)  $. Hence, if $q_{C_{f}}\left(  t\right)  \neq f\left(  t\right)  $,
then $q_{C_{f}}\left(  t\right)  $ is a proper divisor of $f\left(  t\right)
$, thus has degree $<n$ (since $f\left(  t\right)  $ has degree $n$). So we
just need to rule out the possibility that $q_{C_{f}}\left(  t\right)  $ has
degree $<n$.

Indeed, assume (for the sake of contradiction) that $q_{C_{f}}\left(
t\right)  $ has degree $<n$. Thus, $q_{C_{f}}\left(  t\right)  =a_{k}%
t^{k}+a_{k-1}t^{k-1}+\cdots+a_{0}t^{0}$ with $k<n$ and $a_{k}=1$ (since
$q_{C_{f}}$ is monic of degree $<n$). However, the definition of $q_{C_{f}}$
yields $q_{C_{f}}\left(  C_{f}\right)  =0$. In other words,%
\[
a_{k}C_{f}^{k}+a_{k-1}C_{f}^{k-1}+\cdots+a_{0}C_{f}^{0}=0.
\]


However, let us look at what $C_{f}$ does to the standard basis vector
$e_{1}=\left(  1,0,0,0,\ldots,0\right)  ^{T}$. We have%
\begin{align*}
C_{f}^{0}e_{1}  &  =e_{1};\\
C_{f}^{1}e_{1}  &  =C_{f}e_{1}=e_{2};\\
C_{f}^{2}e_{1}  &  =C_{f}e_{2}=e_{3};\\
&  \ldots;\\
C_{f}^{n-1}e_{1}  &  =e_{n}.
\end{align*}
Thus, applying our equality%
\[
a_{k}C_{f}^{k}+a_{k-1}C_{f}^{k-1}+\cdots+a_{0}C_{f}^{0}=0
\]
to $e_{1}$, we obtain%
\[
a_{k}e_{k+1}+a_{k-1}e_{k}+\cdots+a_{0}e_{1}=0\ \ \ \ \ \ \ \ \ \ \left(
\text{since }k<n\right)  .
\]
But this is absurd, since $e_{1},e_{2},\ldots,e_{n}$ are linearly independent.
So we found a contradiction, and thus we conclude that $q_{C_{f}}\left(
t\right)  $ has degree $\geq n$. So, by the above, we obtain $q_{C_{f}}\left(
t\right)  =f\left(  t\right)  $.
\end{proof}

\begin{remark}
For algebraists: The companion matrix $C_{f}$ has a natural meaning. To wit,
consider the quotient ring $\mathbb{F}\left[  t\right]  /\left(  f\left(
t\right)  \right)  $ as an $n$-dimensional $\mathbb{F}$-vector space with
basis $\left(  \overline{t^{0}},\overline{t^{1}},\ldots,\overline{t^{n-1}%
}\right)  $. Then, the companion matrix $C_{f}$ represents the endomorphism
\textquotedblleft multiply by $t$\textquotedblright\ (that is, the
endomorphism that sends each residue class $\overline{g\left(  t\right)  }$ to
$\overline{t\cdot g\left(  t\right)  }$) in this basis.
\end{remark}

\begin{exercise}
\fbox{3} Let $A\in\mathbb{C}^{n\times n}$ be an $n\times n$-matrix that has
$n$ \textbf{distinct} eigenvalues. Prove that $A\sim C_{p_{A}}$.
\end{exercise}

\begin{exercise}
\fbox{4} Let $\mathbb{F}$ be a field. Let $A\in\mathbb{F}^{n\times n}$ be an
$n\times n$-matrix. Prove that $A\sim C_{p_{A}}$ if and only if there exists a
vector $v\in\mathbb{F}^{n}$ such that
\[
\left(  v,\ Av,\ A^{2}v,\ \ldots,\ A^{n-1}v\right)  =\left(  A^{0}%
v,\ A^{1}v,\ \ldots,\ A^{n-1}v\right)
\]
is a basis of $\mathbb{F}^{n}$.
\end{exercise}

\subsection{The Jordan--Chevalley decomposition}

Recall that:

\begin{itemize}
\item A matrix $A\in\mathbb{C}^{n\times n}$ is said to be
\emph{diagonalizable} if it is similar to a diagonal matrix.

\item A matrix $A\in\mathbb{C}^{n\times n}$ is said to be \emph{nilpotent} if
some power of it is the zero matrix (i.e., if $A^{k}=0$ for some
$k\in\mathbb{N}$). As we know from Exercise \ref{exe.jnf.step3.An=0}, for an
$n\times n$-matrix $A$ to be nilpotent, it is necessary and sufficient that
$A^{n}=0$.
\end{itemize}

\begin{theorem}
[Jordan--Chevalley decomposition]\label{thm.jnf.jcdec.main}Let $A\in
\mathbb{C}^{n\times n}$ be an $n\times n$-matrix. \medskip

\textbf{(a)} Then, there exists a unique pair $\left(  D,N\right)  $
consisting of

\begin{itemize}
\item a diagonalizable matrix $D\in\mathbb{C}^{n\times n}$ and

\item a nilpotent matrix $N\in\mathbb{C}^{n\times n}$
\end{itemize}

\noindent such that $DN=ND$ and $A=D+N$. \medskip

\textbf{(b)} Both matrices $D$ and $N$ in this pair can be written as
polynomials in $A$. In other words, there exist two polynomials $f,g\in
\mathbb{C}\left[  t\right]  $ such that $D=f\left(  A\right)  $ and
$N=g\left(  A\right)  $.
\end{theorem}

The pair $\left(  D,N\right)  $ in this theorem is known as the
\emph{Jordan--Chevalley decomposition} (or the \emph{Dunford decomposition})
of $A$.

For a complete proof of Theorem \ref{thm.jnf.jcdec.main}, see \cite[Chapter
VII, \S 5, section 9, Theorem 1]{Bourba03}. An outline can also be found on
\href{https://en.wikipedia.org/wiki/Jordan-Chevalley_decomposition}{the
Wikipedia page for \textquotedblleft Jordan--Chevalley
decomposition\textquotedblright}.

\begin{proof}
[Partial proof of Theorem \ref{thm.jnf.jcdec.main}.]We will only show the
following claim:

\begin{statement}
\textit{Claim 1:} There exists a Jordan--Chevalley decomposition of $A$.
\end{statement}

To prove Claim 1, we can WLOG assume that $A$ is a Jordan matrix. Indeed, if
$A=SJS^{-1}$ for some invertible $S\in\mathbb{C}^{n\times n}$ and some Jordan
matrix $J\in\mathbb{C}^{n\times n}$, and if $\left(  D^{\prime},N^{\prime
}\right)  $ is a Jordan--Chevalley decomposition of $J$, then $\left(
SD^{\prime}S^{-1},SN^{\prime}S^{-1}\right)  $ is a Jordan--Chevalley
decomposition of $A$.

\begin{noncompile}
\textit{Claim 2:} The $D$ and $N$ in this particular pair can be written as
polynomials in $A$.
\end{noncompile}

\begin{noncompile}
To prove both Claims 1 and 2, we can WLOG assume that $A$ is a Jordan matrix.
Indeed, if $A=SJS^{-1}$ for some invertible $S\in\mathbb{C}^{n\times n}$ and
some Jordan matrix $J\in\mathbb{C}^{n\times n}$, and if $\left(  D^{\prime
},N^{\prime}\right)  $ is a Jordan--Chevalley decomposition of $J$, then
$\left(  SD^{\prime}S^{-1},SN^{\prime}S^{-1}\right)  $ is a Jordan--Chevalley
decomposition of $A$. Moreover, if $D^{\prime}$ and $N^{\prime}$ can be
written as polynomials in $J$, then $SD^{\prime}S^{-1}$ and $SN^{\prime}%
S^{-1}$ can be written as (the same!) polynomials in $A$ (since $f\left(
SPS^{-1}\right)  =Sf\left(  P\right)  S^{-1}$ for any matrix $P\in
\mathbb{C}^{n\times n}$ and any polynomial $f\in\mathbb{C}\left[  t\right]
$). Therefore, we only need to prove Claims 1 and 2 for $J$ instead of $A$; in
other words, we can WLOG assume that $A$ itself is a Jordan matrix.
\end{noncompile}

So we WLOG assume that $A$ is a Jordan matrix. Thus,%
\[
A=\left(
\begin{array}
[c]{cccc}%
J_{k_{1}}\left(  \lambda_{1}\right)  &  &  & \\
& J_{k_{2}}\left(  \lambda_{2}\right)  &  & \\
&  & \ddots & \\
&  &  & J_{k_{p}}\left(  \lambda_{p}\right)
\end{array}
\right)
\]
for some $\lambda_{1},\lambda_{2},\ldots,\lambda_{p}$ and some $k_{1}%
,k_{2},\ldots,k_{p}$. (Here, empty cells are understood to be filled with zero matrices.)

We want to find a Jordan--Chevalley decomposition of $A$. In other words, we
want to find a pair $\left(  D,N\right)  $, where $D\in\mathbb{C}^{n\times n}$
is a diagonalizable matrix and $N\in\mathbb{C}^{n\times n}$ is a nilpotent
matrix satisfying $DN=ND$ and $A=D+N$. We do this by setting%
\[
D:=\left(
\begin{array}
[c]{cccc}%
\lambda_{1}I_{k_{1}} &  &  & \\
& \lambda_{2}I_{k_{2}} &  & \\
&  & \ddots & \\
&  &  & \lambda_{p}I_{k_{p}}%
\end{array}
\right)  \ \ \ \ \ \ \ \ \ \ \text{and}\ \ \ \ \ \ \ \ \ \ N:=\left(
\begin{array}
[c]{cccc}%
J_{k_{1}}\left(  0\right)  &  &  & \\
& J_{k_{2}}\left(  0\right)  &  & \\
&  & \ddots & \\
&  &  & J_{k_{p}}\left(  0\right)
\end{array}
\right)  .
\]
It is easy to check that $A=D+N$ (since $J_{k}\left(  \lambda\right)  =\lambda
I_{k}+J_{k}\left(  0\right)  $ for each $k>0$ and $\lambda\in\mathbb{C}$) and
$DN=ND$ (since block-diagonal matrices can be multiplied block by block, and
since matrices of the form $\lambda I_{k}$ for $k>0$ and $\lambda\in
\mathbb{C}$ commute with every $k\times k$-matrix). Clearly, the matrix $D$ is
diagonalizable (since $D$ is diagonal) and the matrix $N$ is nilpotent (since
$N$ is strictly upper-triangular). Thus, Claim 1 is proved.

\begin{noncompile}
Now, we need to prove Claim 2 -- i.e., we need to prove that $D$ and $N$ can
be written as polynomials in $A$. We shall first piece together a polynomial
$f\in\mathbb{C}\left[  t\right]  $ that satisfies $f\left(  D\right)  =A$. The
construction will be somewhat similar to Lagrange interpolation: We will look
at distinct eigenvalues one at a time, finding polynomials that have a desired
effect for their corresponding Jordan blocks; then we will obtain our $f$ as a
sum of these polynomials.

Our above construction of $D$ shows that $D$ is simply the matrix $A$ with its
non-diagonal entries removed. Let $\mu_{1},\mu_{2},\ldots,\mu_{m}$ be the
\textbf{distinct} eigenvalues (i.e., diagonal entries) of $A$. Thus, $\left\{
\mu_{1},\mu_{2},\ldots,\mu_{m}\right\}  =\left\{  \lambda_{1},\lambda
_{2},\ldots,\lambda_{p}\right\}  $. That is, each $\lambda_{i}$ equals
$\mu_{j}$ for some $j\in\left[  m\right]  $, and conversely, each $\mu_{j}$
equals $\lambda_{i}$ for some $i\in\left[  p\right]  $, but there is no 1-to-1
correspondence in general because some of $\lambda_{1},\lambda_{2}%
,\ldots,\lambda_{p}$ can be equal. However, for each $u\in\left[  p\right]  $,
there is a \textbf{unique} $i\in\left[  m\right]  $ satisfying $\lambda
_{u}=\mu_{i}$ (because $\lambda_{u}\in\left\{  \lambda_{1},\lambda_{2}%
,\ldots,\lambda_{p}\right\}  =\left\{  \mu_{1},\mu_{2},\ldots,\mu_{m}\right\}
$, and because $\mu_{1},\mu_{2},\ldots,\mu_{m}$ are distinct). We denote this
$i$ by $\operatorname*{index}\left(  u\right)  $. Thus,
\begin{equation}
\lambda_{u}=\mu_{\operatorname*{index}\left(  u\right)  }%
\ \ \ \ \ \ \ \ \ \ \text{for each }u\in\left[  p\right]  .
\label{pf.thm.jnf.jcdec.main.c2.index}%
\end{equation}


For each $i\in\left[  m\right]  $, let $\ell_{i}$ be the size of the largest
Jordan block of $J$ at eigenvalue $\mu_{i}$. (Thus, the minimal polynomial of
$A$ is $\prod\limits_{i=1}^{m}\left(  t-\mu_{i}\right)  ^{\ell_{i}}$,
according to Theorem \ref{thm.jnf.mipo.jnf}.)

For each $j\in\left[  m\right]  $, we define a polynomial%
\begin{equation}
g_{j}\left(  t\right)  :=\prod\limits_{j\neq i}\left(  t-\mu_{j}\right)
^{\ell_{j}}\in\mathbb{C}\left[  t\right]  ,
\label{pf.thm.jnf.jcdec.main.c2.g-def}%
\end{equation}
where the product sign $\prod\limits_{j\neq i}$ means a product over all
$j\in\left[  m\right]  $ except for $j=i$.

It is easy to see that each $u\in\left[  p\right]  $ and each $i\in\left[
m\right]  $ satisfy%
\begin{equation}
g_{i}\left(  J_{k_{u}}\left(  \lambda_{u}\right)  \right)
=0\ \ \ \ \ \ \ \ \ \ \text{if }i\neq\operatorname*{index}\left(  u\right)  .
\label{pf.thm.jnf.jcdec.main.c2.g=0}%
\end{equation}


[\textit{Proof of (\ref{pf.thm.jnf.jcdec.main.c2.g=0}):} Let $u\in\left[
p\right]  $ and $i\in\left[  m\right]  $ satisfy $i\neq\operatorname*{index}%
\left(  u\right)  $. Let $Z:=J_{k_{u}}\left(  \lambda_{u}\right)  $ and
$W:=J_{k_{u}}\left(  0\right)  $. Proposition \ref{prop.jnf.jcell-lambda}
yields $Z=W+\lambda_{u}I_{k_{u}}$, so that $Z-\lambda_{u}I_{k_{u}}=W$.
However, $W$ is a strictly upper-triangular $k_{u}\times k_{u}$-matrix. Hence,
$W^{k_{u}}=0$ (by (\ref{eq.jnf.exist.step3.An=0}), applied to $k_{u}$ and $W$
instead of $n$ and $A$).

From (\ref{pf.thm.jnf.jcdec.main.c2.index}), we obtain $\lambda_{u}%
=\mu_{\operatorname*{index}\left(  u\right)  }$.

Recall that $\ell_{\operatorname*{index}\left(  u\right)  }$ was defined as
the size of the largest Jordan block of $J$ at eigenvalue $\mu
_{\operatorname*{index}\left(  u\right)  }$. In other words, $\ell
_{\operatorname*{index}\left(  u\right)  }$ is the size of the largest Jordan
block of $J$ at eigenvalue $\lambda_{u}$ (since $\lambda_{u}=\mu
_{\operatorname*{index}\left(  u\right)  }$). Since $J_{k_{u}}\left(
\lambda_{u}\right)  $ is such a Jordan block, we thus conclude that
$\ell_{\operatorname*{index}\left(  u\right)  }\geq k_{u}$ (since $J_{k_{u}%
}\left(  \lambda_{u}\right)  $ has size $k_{u}$). Hence, from $W^{k_{u}}=0$,
we obtain $W^{\ell_{\operatorname*{index}\left(  u\right)  }}=0$.

Substituting $Z$ for $t$ in the equality (\ref{pf.thm.jnf.jcdec.main.c2.g-def}%
), we obtain%
\begin{equation}
g_{i}\left(  Z\right)  =\prod\limits_{j\neq i}\left(  Z-\mu_{j}I_{k_{u}%
}\right)  ^{\ell_{j}}. \label{pf.thm.jnf.jcdec.main.c2.g=0.pf.1}%
\end{equation}
From $i\neq\operatorname*{index}\left(  u\right)  $, we see that the product
$\prod\limits_{j\neq i}\left(  Z-\mu_{j}I_{k_{u}}\right)  ^{\ell_{j}}$ has a
factor for $j=\operatorname*{index}\left(  u\right)  $. This factor is
\begin{align*}
\left(  Z-\mu_{\operatorname*{index}\left(  u\right)  }I_{k_{u}}\right)
^{\ell_{\operatorname*{index}\left(  u\right)  }}  &  =\left(  Z-\lambda
_{u}I_{k_{u}}\right)  ^{\ell_{\operatorname*{index}\left(  u\right)  }%
}\ \ \ \ \ \ \ \ \ \ \left(  \text{since }\mu_{\operatorname*{index}\left(
u\right)  }=\lambda_{u}\right) \\
&  =W^{\ell_{\operatorname*{index}\left(  u\right)  }}%
\ \ \ \ \ \ \ \ \ \ \left(  \text{since }Z-\lambda_{u}I_{k_{u}}=W\right) \\
&  =0.
\end{align*}
Thus, the product $\prod\limits_{j\neq i}\left(  Z-\mu_{j}I_{k_{u}}\right)
^{\ell_{j}}$ has a factor that equals $0$. Therefore, this entire product is
$0$. Thus, (\ref{pf.thm.jnf.jcdec.main.c2.g=0.pf.1}) simplifies to
$g_{i}\left(  Z\right)  =0$. In light of $Z=J_{k_{u}}\left(  \lambda
_{u}\right)  $, this rewrites as $g_{i}\left(  J_{k_{u}}\left(  \lambda
_{u}\right)  \right)  =0$. Thus, (\ref{pf.thm.jnf.jcdec.main.c2.g=0}) is
proven.] \medskip

On the other hand, for any $u\in\left[  p\right]  $, we define the
$k_{u}\times k_{u}$-matrix%
\begin{equation}
G_{u}:=g_{\operatorname*{index}\left(  u\right)  }\left(  J_{k_{u}}\left(
\lambda_{u}\right)  \right)  . \label{pf.thm.jnf.jcdec.main.c2.Gu}%
\end{equation}
We claim that for any $u\in\left[  p\right]  $,
\begin{equation}
\text{the matrix }G_{u}\text{ is invertible.}
\label{pf.thm.jnf.jcdec.main.c2.gindu}%
\end{equation}


[\textit{Proof of (\ref{pf.thm.jnf.jcdec.main.c2.gindu}):} Let $u\in\left[
p\right]  $. Let $Z:=J_{k_{u}}\left(  \lambda_{u}\right)  $. This is an
upper-triangular matrix whose diagonal entries are $\lambda_{u},\lambda
_{u},\ldots,\lambda_{u}$.

Set $i:=\operatorname*{index}\left(  u\right)  $. Then,
(\ref{pf.thm.jnf.jcdec.main.c2.index}) yields $\lambda_{u}=\mu_{i}$.

We have $g_{i}\left(  Z\right)  =g_{\operatorname*{index}\left(  u\right)
}\left(  J_{k_{u}}\left(  \lambda_{u}\right)  \right)  $ (since
$i=\operatorname*{index}\left(  u\right)  $ and $Z=J_{k_{u}}\left(
\lambda_{u}\right)  $). In other words, $g_{i}\left(  Z\right)  =G_{u}$ (since
$G_{u}=g_{\operatorname*{index}\left(  u\right)  }\left(  J_{k_{u}}\left(
\lambda_{u}\right)  \right)  $). Thus,%
\[
G_{u}=g_{i}\left(  Z\right)  =\prod\limits_{j\neq i}\left(  Z-\mu_{j}I_{k_{u}%
}\right)  ^{\ell_{j}}%
\]
(by substituting $Z$ for $t$ in the equality
(\ref{pf.thm.jnf.jcdec.main.c2.g-def})). Hence, it suffices to show that the
matrix $Z-\mu_{j}I_{k_{u}}$ is invertible for each $j\neq i$ (because a
product of invertible matrices is always invertible).

So let $j\in\left[  m\right]  $ be such that $j\neq i$. We aim to show that
the matrix $Z-\mu_{j}I_{k_{u}}$ is invertible.

We have $j\neq i$ and thus $\mu_{j}\neq\mu_{i}$ (since $\mu_{1},\mu_{2}%
,\ldots,\mu_{m}$ are distinct). Hence, $\lambda_{u}=\mu_{i}\neq\mu_{j}$, so
that $\lambda_{u}-\mu_{j}\neq0$. However, we know that $Z$ is an
upper-triangular matrix whose diagonal entries are $\lambda_{u},\lambda
_{u},\ldots,\lambda_{u}$. Thus, $Z-\mu_{j}I_{k_{u}}$ is an upper-triangular
matrix whose diagonal entries are $\lambda_{u}-\mu_{j},\lambda_{u}-\mu
_{j},\ldots,\lambda_{u}-\mu_{j}$. Therefore, its determinant is $\det\left(
Z-\mu_{j}I_{k_{u}}\right)  =\left(  \lambda_{u}-\mu_{j}\right)  ^{k_{u}}\neq0$
(since $\lambda_{u}-\mu_{j}\neq0$). Consequently, the matrix $Z-\mu
_{j}I_{k_{u}}$ is invertible. As explained above, this completes our proof of
(\ref{pf.thm.jnf.jcdec.main.c2.gindu}).] \medskip

Now, let $u\in\left[  p\right]  $. From (\ref{pf.thm.jnf.jcdec.main.c2.gindu}%
), we know that the $k_{u}\times k_{u}$-matrix $G_{u}$ is invertible. Hence,
Exercise \ref{exe.schurtri.ch.inverse-poly} (applied to $k_{u}$ and $G_{u}$
instead of $n$ and $A$) shows that there exists a polynomial $h_{u}$ of degree
$n-1$ in the single indeterminate $t$ over $\mathbb{F}$ such that $\left(
G_{u}\right)  ^{-1}=h_{u}\left(  G_{u}\right)  $. Consider this $h_{u}$.

Thus, for each $u\in\left[  p\right]  $, we have defined a polynomial
$h_{u}\in\mathbb{C}\left[  t\right]  $ satisfying%
\begin{equation}
\left(  G_{u}\right)  ^{-1}=h_{u}\left(  G_{u}\right)  .
\label{pf.thm.jnf.jcdec.main.c2.hu}%
\end{equation}


Now, define the polynomial%
\begin{equation}
f\left(  t\right)  :=\sum_{i=1}^{m}\mu_{i}g_{i}\left(  t\right)  \cdot
h_{i}\left(  t\right)  \in\mathbb{C}\left[  t\right]  .
\label{pf.thm.jnf.jcdec.main.c2.f-def}%
\end{equation}
We claim that $f\left(  A\right)  =D$. In other words, we claim that applying
$f$ to $A$ has the effect of cleaning out all off-diagonal entries (while the
diagonal entries remain as they are).

To prove this claim, we recall that%
\[
A=\left(
\begin{array}
[c]{cccc}%
J_{k_{1}}\left(  \lambda_{1}\right)  &  &  & \\
& J_{k_{2}}\left(  \lambda_{2}\right)  &  & \\
&  & \ddots & \\
&  &  & J_{k_{p}}\left(  \lambda_{p}\right)
\end{array}
\right)  .
\]
Hence,%
\[
f\left(  A\right)  =\left(
\begin{array}
[c]{cccc}%
f\left(  J_{k_{1}}\left(  \lambda_{1}\right)  \right)  &  &  & \\
& f\left(  J_{k_{2}}\left(  \lambda_{2}\right)  \right)  &  & \\
&  & \ddots & \\
&  &  & f\left(  J_{k_{p}}\left(  \lambda_{p}\right)  \right)
\end{array}
\right)  .
\]
So we need to show that%
\[
f\left(  J_{k_{u}}\left(  \lambda_{u}\right)  \right)  =\lambda_{u}I_{k_{u}%
}\ \ \ \ \ \ \ \ \ \ \text{for each }u\in\left[  p\right]  .
\]
But this can be proved by a straightforward computation: For each $u\in\left[
p\right]  $, we have%
\begin{align*}
f\left(  J_{k_{u}}\left(  \lambda_{u}\right)  \right)   &  =\sum_{i=1}^{m}%
\mu_{i}\underbrace{g_{i}\left(  J_{k_{u}}\left(  \lambda_{u}\right)  \right)
}_{\substack{=0\text{ if }i\neq\operatorname*{index}\left(  u\right)
\\\text{(by (\ref{pf.thm.jnf.jcdec.main.c2.g=0}))}}}\cdot h_{i}\left(
J_{k_{u}}\left(  \lambda_{u}\right)  \right) \\
&  \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \left(  \text{by substituting
}J_{k_{u}}\left(  \lambda_{u}\right)  \text{ for }t\text{ in
(\ref{pf.thm.jnf.jcdec.main.c2.f-def})}\right) \\
&  =\underbrace{\mu_{\operatorname*{index}\left(  u\right)  }}%
_{\substack{=\lambda_{u}\\\text{(by (\ref{pf.thm.jnf.jcdec.main.c2.index}))}%
}}\underbrace{g_{\operatorname*{index}\left(  u\right)  }\left(  J_{k_{u}%
}\left(  \lambda_{u}\right)  \right)  }_{\substack{=G_{u}\\\text{(by
(\ref{pf.thm.jnf.jcdec.main.c2.Gu}))}}}\cdot h_{\operatorname*{index}\left(
u\right)  }\left(  J_{k_{u}}\left(  \lambda_{u}\right)  \right) \\
&  \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \left(
\begin{array}
[c]{c}%
\text{here, we have removed all the addends with }i\neq\operatorname*{index}%
\left(  u\right) \\
\text{from the sum, because all these addends are }0\text{ and thus}\\
\text{do not affect the sum}%
\end{array}
\right) \\
&  =\lambda_{u}G_{u}\cdot h_{\operatorname*{index}\left(  u\right)  }\left(
J_{k_{u}}\left(  \lambda_{u}\right)  \right)
\end{align*}


TODO: This is not what we need. Something needs to be fixed!

[Alternatively, use the Chinese Remainder Theorem for polynomials to find a
polynomial $f$ that satisfies $f\left(  J_{k_{u}}\left(  \lambda_{u}\right)
\right)  =\lambda_{u}I_{k_{u}}$. This polynomial $f$ should satisfy
$f\equiv\left(  t-\lambda_{u}\right)  ^{k_{u}}+\lambda_{u}$ for each $u$.]

Thus, we have constructed a polynomial $f\in\mathbb{C}\left[  t\right]  $
satisfying $f\left(  A\right)  =D$. Therefore, $D$ can be written as a
polynomial in $A$. Moreover, if we define a polynomial $g\in\mathbb{C}\left[
t\right]  $ by $g\left(  t\right)  =t-f\left(  t\right)  $, then we have
$g\left(  A\right)  =A-\underbrace{f\left(  A\right)  }_{=D}=A-D=N$ (since
$A=D+N$), so that $N$ can be written as a polynomial in $A$ as well. This
completes the proof of Claim 2.
\end{noncompile}

The rest of the proof of Theorem \ref{thm.jnf.jcdec.main} is omitted for now.
\end{proof}

\subsection{The real Jordan canonical form}

Given a matrix $A\in\mathbb{R}^{n\times n}$ with real entries, its Jordan
canonical form doesn't necessarily have real entries. Indeed, the eigenvalues
of $A$ don't have to be real. Sometimes, we want to find a \textquotedblleft
simple\textquotedblright\ form for $A$ that does have real entries. What
follows is a way to tweak the Jordan canonical form to this use case.

We observe the following:

\begin{lemma}
\label{lem.jnf.real-jord.same-nums}Let $A\in\mathbb{R}^{n\times n}$ and
$\lambda\in\mathbb{C}$. Then, the \textquotedblleft Jordan structure of $A$ at
$\lambda$\textquotedblright\ (meaning the multiset of the sizes of the Jordan
blocks of $A$ at $\lambda$) equals the Jordan structure of $A$ at
$\overline{\lambda}$. In other words, for each $p>0$, we have%
\begin{align*}
&  \left(  \text{the number of Jordan blocks of }A\text{ at }\lambda\text{
having size }p\right) \\
&  =\left(  \text{the number of Jordan blocks of }A\text{ at }\overline
{\lambda}\text{ having size }p\right)  .
\end{align*}


In other words, Jordan blocks at $\lambda$ and Jordan blocks at $\overline
{\lambda}$ come in pairs of equal sizes (when $\lambda\neq\overline{\lambda}$).
\end{lemma}

\begin{exercise}
\fbox{2} Prove Lemma \ref{lem.jnf.real-jord.same-nums}.
\end{exercise}

So we can try to combine each Jordan block at $\lambda$ with an equally sized
Jordan block at $\overline{\lambda}$ (when $\lambda\notin\mathbb{R}$) and hope
that something real comes out somehow, in the same way as multiplying the
complex polynomials $t-\lambda$ and $t-\overline{\lambda}$ yields the real
polynomial $\left(  t-\lambda\right)  \left(  t-\overline{\lambda}\right)
=t^{2}-2\left(  \operatorname*{Re}\lambda\right)  t+\left\vert \lambda
\right\vert ^{2}\in\mathbb{R}\left[  t\right]  $.

How to do this? For Jordan blocks of size $1$, this is easy:

\begin{lemma}
\label{lem.jnf.real-jord.2x2}Let $\lambda\in\mathbb{C}$. Let $L$ be the
$2\times2$-matrix $\left(
\begin{array}
[c]{cc}%
\lambda & 0\\
0 & \overline{\lambda}%
\end{array}
\right)  $. Let $a=\operatorname*{Re}\lambda$ and $b=\operatorname*{Im}%
\lambda$ (so that $\lambda=a+bi$). Then,%
\[
L\sim\left(
\begin{array}
[c]{cc}%
a & b\\
-b & a
\end{array}
\right)  .
\]

\end{lemma}

\begin{exercise}
\fbox{2} Prove Lemma \ref{lem.jnf.real-jord.2x2}.
\end{exercise}

Now, let us see how to combine a Jordan block at $\lambda$ with an equally
sized Jordan block at $\overline{\lambda}$ when the size is arbitrary. We can
WLOG assume that these two Jordan blocks are adjacent (since we can permute
the Jordan blocks at will). Thus, they form the following matrix
together:\footnote{Again, empty cells in matrices signify $0$s (or zero
matrices).}%
\[
\left(
\begin{array}
[c]{cc}%
J_{p}\left(  \lambda\right)  & \\
& J_{p}\left(  \overline{\lambda}\right)
\end{array}
\right)  =\left(
\begin{array}
[c]{cccccccc}%
\lambda & 1 &  &  &  &  &  & \\
& \lambda & \ddots &  &  &  &  & \\
&  & \ddots & 1 &  &  &  & \\
&  &  & \lambda &  &  &  & \\
&  &  &  & \overline{\lambda} & 1 &  & \\
&  &  &  &  & \overline{\lambda} & \ddots & \\
&  &  &  &  &  & \ddots & 1\\
&  &  &  &  &  &  & \overline{\lambda}%
\end{array}
\right)  .
\]
This matrix is similar to the $2p\times2p$-matrix
\[
L_{p}:=\left(
\begin{array}
[c]{cccccccc}%
\lambda &  & 1 &  &  &  &  & \\
& \overline{\lambda} &  & 1 &  &  &  & \\
&  & \lambda &  & 1 &  &  & \\
&  &  & \overline{\lambda} &  & 1 &  & \\
&  &  &  & \ddots &  & \ddots & \\
&  &  &  &  & \ddots &  & \ddots\\
&  &  &  &  &  & \lambda & \\
&  &  &  &  &  &  & \overline{\lambda}%
\end{array}
\right)  =\left(
\begin{array}
[c]{ccccc}%
L & I_{2} &  &  & \\
& L & I_{2} &  & \\
&  & \ddots & \ddots & \\
&  &  & L & I_{2}\\
&  &  &  & L
\end{array}
\right)  ,
\]
where $L$ is the $2\times2$-matrix $\left(
\begin{array}
[c]{cc}%
\lambda & 0\\
0 & \overline{\lambda}%
\end{array}
\right)  $. (In fact, we can easily see that $\left(
\begin{array}
[c]{cc}%
J_{p}\left(  \lambda\right)  & \\
& J_{p}\left(  \overline{\lambda}\right)
\end{array}
\right)  =P_{\sigma}^{-1}L_{p}P_{\sigma}$, where $P_{\sigma}$ is the
permutation matrix of the permutation $\sigma$ of $\left[  2p\right]  $ that
sends $1,2,3,\ldots,p,p+1,p+2,p+3,\ldots,2p$ to $1,3,5,\ldots
,2p-1,2,4,6,\ldots,2p$.) However, Lemma \ref{lem.jnf.real-jord.2x2} yields%
\[
L\sim\left(
\begin{array}
[c]{cc}%
a & b\\
-b & a
\end{array}
\right)  ,
\]
where $a=\operatorname*{Re}\lambda$ and $b=\operatorname*{Im}\lambda$ (so that
$\lambda=a+bi$). So our matrix $L_{p}$ is similar to%
\[
\left(
\begin{array}
[c]{cccccccc}%
a & b & 1 &  &  &  &  & \\
-b & a &  & 1 &  &  &  & \\
&  & a & b & 1 &  &  & \\
&  & -b & a &  & 1 &  & \\
&  &  &  & \ddots & \ddots & \ddots & \\
&  &  &  &  & \ddots & \ddots & \ddots\\
&  &  &  &  &  & a & b\\
&  &  &  &  &  & -b & a
\end{array}
\right)
\]
(why?). Hence, altogether, we obtain%
\[
\left(
\begin{array}
[c]{cc}%
J_{p}\left(  \lambda\right)  & \\
& J_{p}\left(  \overline{\lambda}\right)
\end{array}
\right)  \sim\left(
\begin{array}
[c]{cccccccc}%
a & b & 1 &  &  &  &  & \\
-b & a &  & 1 &  &  &  & \\
&  & a & b & 1 &  &  & \\
&  & -b & a &  & 1 &  & \\
&  &  &  & \ddots & \ddots & \ddots & \\
&  &  &  &  & \ddots & \ddots & \ddots\\
&  &  &  &  &  & a & b\\
&  &  &  &  &  & -b & a
\end{array}
\right)  .
\]
The matrix on the right is a real matrix. Thus, we can replace our two Jordan
blocks (at $\lambda$ and $\overline{\lambda}$, of equal sizes) by a real
$2p\times2p$-matrix, obtaining a similar matrix. Performing the same procedure
with all Jordan blocks at non-real eigenvalues, we thus obtain a
\textquotedblleft normal form\textquotedblright\ that has real entries. See
\cite[\S 3.4.1]{HorJoh13} for details and for further results in this direction.

\subsection{The centralizer of a matrix}

Here is a fairly natural question: Which matrices commute with a given square
matrix $A$ ?

\begin{proposition}
Let $\mathbb{F}$ be a field. Let $A\in\mathbb{F}^{n\times n}$ be an $n\times
n$-matrix. Let $f$ and $g$ be two polynomials in a single variable $t$ over
$\mathbb{F}$. Then, $f\left(  A\right)  $ commutes with $g\left(  A\right)  $.
\end{proposition}

\begin{proof}
Write $f\left(  t\right)  $ as $f\left(  t\right)  =\sum_{i=0}^{n}f_{i}t^{i}$,
and write $g\left(  t\right)  $ as $g\left(  t\right)  =\sum_{j=0}^{m}%
g_{j}t^{j}$. Then,%
\[
f\left(  A\right)  =\sum_{i=0}^{n}f_{i}A^{i}\ \ \ \ \ \ \ \ \ \ \text{and}%
\ \ \ \ \ \ \ \ \ \ g\left(  A\right)  =\sum_{j=0}^{m}g_{j}A^{j}.
\]
Thus,%
\[
f\left(  A\right)  \cdot g\left(  A\right)  =\left(  \sum_{i=0}^{n}f_{i}%
A^{i}\right)  \cdot\left(  \sum_{j=0}^{m}g_{j}A^{j}\right)  =\sum_{i=0}%
^{n}\ \ \sum_{j=0}^{m}f_{i}g_{j}\underbrace{A^{i}A^{j}}_{=A^{i+j}}=\sum
_{i=0}^{n}\ \ \sum_{j=0}^{m}f_{i}g_{j}A^{i+j}.
\]
A similar computation shows that%
\[
g\left(  A\right)  \cdot f\left(  A\right)  =\sum_{i=0}^{n}\ \ \sum_{j=0}%
^{m}f_{i}g_{j}A^{i+j}.
\]
Comparing these two, we obtain $f\left(  A\right)  \cdot g\left(  A\right)
=g\left(  A\right)  \cdot f\left(  A\right)  $, qed.
\end{proof}

Thus, in particular, $f\left(  A\right)  $ commutes with $A$ for any
polynomial $f$ (because $A=g\left(  A\right)  $ for $g\left(  t\right)  =t$).

But are there other matrices that commute with $A$ ?

There certainly can be. For instance, if $A=\lambda I_{n}$ for some
$\lambda\in\mathbb{F}$, then \textbf{every} $n\times n$-matrix commutes with
$A$ (but very few matrices are of the form $f\left(  A\right)  $ for some
polynomial $f$). This is, in a sense, the \textquotedblleft best case
scenario\textquotedblright. Only for $A=\lambda I_{n}$ is it true that every
$n\times n$-matrix commutes with $A$.

Let us study the general case now.

\begin{definition}
Let $A\in\mathbb{F}^{n\times n}$ be an $n\times n$-matrix. The
\emph{centralizer} of $A$ is defined to be the set of all $n\times n$-matrices
$B\in\mathbb{F}^{n\times n}$ such that $AB=BA$. We denote this set by
$\operatorname*{Cent}A$.
\end{definition}

We thus want to know what $\operatorname*{Cent}A$ is.

We begin with some general properties:

\begin{proposition}
\label{prop.jnf.cent.subalg}Let $A\in\mathbb{F}^{n\times n}$ be an $n\times
n$-matrix. Then, $\operatorname*{Cent}A$ is a subset of $\mathbb{F}^{n\times
n}$ that is closed under addition, scaling and multiplication and contains
$\lambda I_{n}$ for all $\lambda\in\mathbb{F}$. In other words:

\textbf{(a)} For any $B,C\in\operatorname*{Cent}A$, we have $B+C\in
\operatorname*{Cent}A$.

\textbf{(b)} For any $B\in\operatorname*{Cent}A$ and $\lambda\in\mathbb{F}$,
we have $\lambda B\in\operatorname*{Cent}A$.

\textbf{(c)} For any $B,C\in\operatorname*{Cent}A$, we have $BC\in
\operatorname*{Cent}A$.

\textbf{(d)} For any $\lambda\in\mathbb{F}$, we have $\lambda I_{n}%
\in\operatorname*{Cent}A$.
\end{proposition}

This implies, in particular, that $\operatorname*{Cent}A$ is a vector subspace
of $\mathbb{F}^{n\times n}$. Furthermore, it shows that $\operatorname*{Cent}%
A$ is an $\mathbb{F}$-subalgebra of $\mathbb{F}^{n\times n}$ (in particular, a
subring of $\mathbb{F}^{n\times n}$).

\begin{proof}
[Proof of Proposition \ref{prop.jnf.cent.subalg}.]Let me just show part
\textbf{(c)}; the other parts are even easier.

\textbf{(c)} Let $B,C\in\operatorname*{Cent}A$. Thus, $AB=BA$ and $AC=CA$.
Now,%
\[
\underbrace{AB}_{=BA}C=B\underbrace{AC}_{=CA}=BCA.
\]
This shows that $BC\in\operatorname*{Cent}A$. Thus, part \textbf{(c)} is proved.
\end{proof}

Now, as an example, let us compute $\operatorname*{Cent}A$ in the case when
$A$ is a single Jordan cell $J_{n}\left(  0\right)  $. So we fix an $n>0$, and
we set%
\[
A:=J_{n}\left(  0\right)  =\left(
\begin{array}
[c]{ccccc}%
0 & 1 & 0 & \cdots & 0\\
0 & 0 & 1 & \cdots & 0\\
0 & 0 & 0 & \cdots & 0\\
\vdots & \vdots & \vdots & \ddots & \vdots\\
0 & 0 & 0 & \cdots & 0
\end{array}
\right)  .
\]
Let $B\in\mathbb{F}^{n\times n}$ be arbitrary. We want to know when
$B\in\operatorname*{Cent}A$. In other words, we want to know when $AB=BA$.

We have
\begin{align*}
AB  &  =\left(
\begin{array}
[c]{ccccc}%
0 & 1 & 0 & \cdots & 0\\
0 & 0 & 1 & \cdots & 0\\
0 & 0 & 0 & \cdots & 0\\
\vdots & \vdots & \vdots & \ddots & \vdots\\
0 & 0 & 0 & \cdots & 0
\end{array}
\right)  \left(
\begin{array}
[c]{ccccc}%
B_{1,1} & B_{1,2} & B_{1,3} & \cdots & B_{1,n}\\
B_{2,1} & B_{2,2} & B_{2,3} & \cdots & B_{2,n}\\
B_{3,1} & B_{3,2} & B_{3,3} & \cdots & B_{3,n}\\
\vdots & \vdots & \vdots & \ddots & \vdots\\
B_{n,1} & B_{n,2} & B_{n,3} & \cdots & B_{n,n}%
\end{array}
\right) \\
&  =\left(
\begin{array}
[c]{ccccc}%
B_{2,1} & B_{2,2} & B_{2,3} & \cdots & B_{2,n}\\
B_{3,1} & B_{3,2} & B_{3,3} & \cdots & B_{3,n}\\
\vdots & \vdots & \vdots & \ddots & \vdots\\
B_{n,1} & B_{n,2} & B_{n,3} & \cdots & B_{n,n}\\
0 & 0 & 0 & \cdots & 0
\end{array}
\right)
\end{align*}
and%
\begin{align*}
BA  &  =\left(
\begin{array}
[c]{ccccc}%
B_{1,1} & B_{1,2} & B_{1,3} & \cdots & B_{1,n}\\
B_{2,1} & B_{2,2} & B_{2,3} & \cdots & B_{2,n}\\
B_{3,1} & B_{3,2} & B_{3,3} & \cdots & B_{3,n}\\
\vdots & \vdots & \vdots & \ddots & \vdots\\
B_{n,1} & B_{n,2} & B_{n,3} & \cdots & B_{n,n}%
\end{array}
\right)  \left(
\begin{array}
[c]{ccccc}%
0 & 1 & 0 & \cdots & 0\\
0 & 0 & 1 & \cdots & 0\\
0 & 0 & 0 & \cdots & 0\\
\vdots & \vdots & \vdots & \ddots & \vdots\\
0 & 0 & 0 & \cdots & 0
\end{array}
\right) \\
&  =\left(
\begin{array}
[c]{ccccc}%
0 & B_{1,1} & B_{1,2} & \cdots & B_{1,n-1}\\
0 & B_{2,1} & B_{2,2} & \cdots & B_{2,n-1}\\
0 & B_{3,1} & B_{3,2} & \cdots & B_{3,n-1}\\
\vdots & \vdots & \vdots & \ddots & \ddots\\
0 & B_{n,1} & B_{n,2} & \cdots & B_{n,n-1}%
\end{array}
\right)  .
\end{align*}
Thus, $AB=BA$ holds if and only if
\[
\left(
\begin{array}
[c]{ccccc}%
B_{2,1} & B_{2,2} & B_{2,3} & \cdots & B_{2,n}\\
B_{3,1} & B_{3,2} & B_{3,3} & \cdots & B_{3,n}\\
\vdots & \vdots & \vdots & \ddots & \vdots\\
B_{n,1} & B_{n,2} & B_{n,3} & \cdots & B_{n,n}\\
0 & 0 & 0 & \cdots & 0
\end{array}
\right)  =\left(
\begin{array}
[c]{ccccc}%
0 & B_{1,1} & B_{1,2} & \cdots & B_{1,n-1}\\
0 & B_{2,1} & B_{2,2} & \cdots & B_{2,n-1}\\
0 & B_{3,1} & B_{3,2} & \cdots & B_{3,n-1}\\
\vdots & \vdots & \vdots & \ddots & \ddots\\
0 & B_{n,1} & B_{n,2} & \cdots & B_{n,n-1}%
\end{array}
\right)  ,
\]
i.e., if%
\begin{align*}
B_{2,j}  &  =B_{1,j-1}\ \ \ \ \ \ \ \ \ \ \text{for all }j\in\left[  n\right]
\text{ (where }B_{1,0}:=0\text{);}\\
B_{3,j}  &  =B_{2,j-1}\ \ \ \ \ \ \ \ \ \ \text{for all }j\in\left[  n\right]
\text{ (where }B_{2,0}:=0\text{);}\\
B_{4,j}  &  =B_{3,j-1}\ \ \ \ \ \ \ \ \ \ \text{for all }j\in\left[  n\right]
\text{ (where }B_{3,0}:=0\text{);}\\
&  \ldots;\\
B_{n,j}  &  =B_{n-1,j-1}\ \ \ \ \ \ \ \ \ \ \text{for all }j\in\left[
n\right]  \text{ (where }B_{n-1,0}:=0\text{);}\\
0  &  =B_{n,j}\ \ \ \ \ \ \ \ \ \ \text{for all }j\in\left[  n-1\right]  .
\end{align*}
The latter system of equations can be restated as follows:%
\begin{align*}
&  \ldots;\\
B_{n,n-2}  &  =B_{n-1,n-3}=B_{n-2,n-4}=\cdots=B_{3,1}=0;\\
B_{n,n-1}  &  =B_{n-1,n-2}=B_{n-2,n-3}=\cdots=B_{2,1}=0;\\
B_{n,n}  &  =B_{n-1,n-1}=B_{n-2,n-2}=\cdots=B_{1,1};\\
B_{n-1,n}  &  =B_{n-2,n-1}=B_{n-3,n-2}=\cdots=B_{1,2};\\
B_{n-2,n}  &  =B_{n-3,n-1}=B_{n-4,n-2}=\cdots=B_{1,3};\\
&  \ldots.
\end{align*}
In other words, it means that the matrix $B$ looks as follows:%
\[
B=\left(
\begin{array}
[c]{ccccc}%
b_{0} & b_{1} & b_{2} & \cdots & b_{n-1}\\
& b_{0} & b_{1} & \cdots & b_{n-2}\\
&  & b_{0} & \cdots & b_{n-3}\\
&  &  & \ddots & \vdots\\
&  &  &  & b_{0}%
\end{array}
\right)
\]
(where the empty cells have entries equal to $0$). This is called an
\emph{upper-triangular Toeplitz matrix}. We can also rewrite it as%
\[
B=b_{0}I_{n}+b_{1}A+b_{2}A^{2}+\cdots+b_{n-1}A^{n-1}.
\]


So we have proved the following:

\begin{theorem}
\label{thm.jnf.cent.Jn0}Let $n>0$. Let $A=J_{n}\left(  0\right)  $. Then,
\begin{align*}
\operatorname*{Cent}A  &  =\left\{  \left(
\begin{array}
[c]{ccccc}%
b_{0} & b_{1} & b_{2} & \cdots & b_{n-1}\\
& b_{0} & b_{1} & \cdots & b_{n-2}\\
&  & b_{0} & \cdots & b_{n-3}\\
&  &  & \ddots & \vdots\\
&  &  &  & b_{0}%
\end{array}
\right)  \ \mid\ b_{0},b_{1},\ldots,b_{n-1}\in\mathbb{F}\right\} \\
&  =\left\{  b_{0}I_{n}+b_{1}A+b_{2}A^{2}+\cdots+b_{n-1}A^{n-1}\ \mid
\ b_{0},b_{1},\ldots,b_{n-1}\in\mathbb{F}\right\} \\
&  =\left\{  f\left(  A\right)  \ \mid\ f\in\mathbb{F}\left[  t\right]  \text{
is a polynomial of degree }\leq n-1\right\}  .
\end{align*}

\end{theorem}

So this is the worst-case scenario: The only matrices commuting with $A$ are
the matrices of the form $f\left(  A\right)  $ (which, as we recall, must
always commute with $A$).

What happens for an arbitrary $A$ ? Is the answer closer to the best-case
scenario or to the worst-case scenario? The answer is that the worst-case
scenario holds for a randomly chosen matrix, but we can actually answer the
question \textquotedblleft what is $\operatorname*{Cent}A$
exactly\textquotedblright\ if we know the Jordan canonical form of $A$.

We start with simple propositions:

\begin{proposition}
\label{prop.jnf.cent.lambda}Let $A\in\mathbb{F}^{n\times n}$ and $\lambda
\in\mathbb{F}$. Then, $\operatorname*{Cent}\left(  A-\lambda I_{n}\right)
=\operatorname*{Cent}A$.
\end{proposition}

\begin{exercise}
\fbox{1} Prove this.
\end{exercise}

\begin{proposition}
\label{prop.jnf.cent.similar}Let $A$, $B$ and $S$ be three $n\times
n$-matrices such that $S$ is invertible. Then,
\[
\left(  B\in\operatorname*{Cent}A\right)  \ \Longleftrightarrow\ \left(
SBS^{-1}\in\operatorname*{Cent}\left(  SAS^{-1}\right)  \right)  .
\]

\end{proposition}

\begin{exercise}
\fbox{1} Prove this.
\end{exercise}

Thus, if $A$ is a matrix with complex entries, and if we want to compute
$\operatorname*{Cent}A$, it suffices to compute $\operatorname*{Cent}J$, where
$J$ is the JCF of $A$.

Therefore, we now focus on centralizers of Jordan matrices. One further
simplification stems from the following proposition:

\begin{proposition}
\label{prop.jnf.cent.block-separate}Let $A_{1},A_{2},\ldots,A_{k}$ be square
matrices with complex entries. Assume that the spectra of these matrices are
disjoint -- i.e., if $i\neq j$, then $\sigma\left(  A_{i}\right)  \cap
\sigma\left(  A_{j}\right)  =\varnothing$.

Then,%
\begin{align*}
&  \operatorname*{Cent}\left(
\begin{array}
[c]{cccc}%
A_{1} &  &  & \\
& A_{2} &  & \\
&  & \ddots & \\
&  &  & A_{k}%
\end{array}
\right) \\
&  =\left\{  \left(
\begin{array}
[c]{cccc}%
B_{1} &  &  & \\
& B_{2} &  & \\
&  & \ddots & \\
&  &  & B_{k}%
\end{array}
\right)  \ \mid\ B_{i}\in\operatorname*{Cent}\left(  A_{i}\right)  \text{ for
each }i\in\left[  k\right]  \right\}  .
\end{align*}

\end{proposition}

\begin{proof}
The $\supseteq$ inclusion is obvious. We thus need to prove the $\subseteq$
inclusion only.

Let $A_{i}$ be an $n_{i}\times n_{i}$-matrix for each $i\in\left[  k\right]  $.

Let $B\in\operatorname*{Cent}\left(
\begin{array}
[c]{cccc}%
A_{1} &  &  & \\
& A_{2} &  & \\
&  & \ddots & \\
&  &  & A_{k}%
\end{array}
\right)  $. We want to show that $B$ has the form $\left(
\begin{array}
[c]{cccc}%
B_{1} &  &  & \\
& B_{2} &  & \\
&  & \ddots & \\
&  &  & B_{k}%
\end{array}
\right)  $ where $B_{i}\in\operatorname*{Cent}\left(  A_{i}\right)  $ for each
$i\in\left[  k\right]  $.

Write $B$ as a block matrix%
\[
B=\left(
\begin{array}
[c]{cccc}%
B\left(  1,1\right)  & B\left(  1,2\right)  & \cdots & B\left(  1,k\right) \\
B\left(  2,1\right)  & B\left(  2,2\right)  & \cdots & B\left(  2,k\right) \\
\vdots & \vdots & \ddots & \vdots\\
B\left(  k,1\right)  & B\left(  k,2\right)  & \cdots & B\left(  k,k\right)
\end{array}
\right)  ,
\]
where each $B\left(  i,j\right)  $ is an $n_{i}\times n_{j}$-matrix. Then, by
the rule for multiplying block matrices, we have%
\begin{align*}
&  \left(
\begin{array}
[c]{cccc}%
A_{1} &  &  & \\
& A_{2} &  & \\
&  & \ddots & \\
&  &  & A_{k}%
\end{array}
\right)  \left(
\begin{array}
[c]{cccc}%
B\left(  1,1\right)  & B\left(  1,2\right)  & \cdots & B\left(  1,k\right) \\
B\left(  2,1\right)  & B\left(  2,2\right)  & \cdots & B\left(  2,k\right) \\
\vdots & \vdots & \ddots & \vdots\\
B\left(  k,1\right)  & B\left(  k,2\right)  & \cdots & B\left(  k,k\right)
\end{array}
\right) \\
&  =\left(
\begin{array}
[c]{cccc}%
A_{1}B\left(  1,1\right)  & A_{1}B\left(  1,2\right)  & \cdots & A_{1}B\left(
1,k\right) \\
A_{2}B\left(  2,1\right)  & A_{2}B\left(  2,2\right)  & \cdots & A_{2}B\left(
2,k\right) \\
\vdots & \vdots & \ddots & \vdots\\
A_{k}B\left(  k,1\right)  & A_{k}B\left(  k,2\right)  & \cdots & A_{k}B\left(
k,k\right)
\end{array}
\right)
\end{align*}
and%
\begin{align*}
&  \left(
\begin{array}
[c]{cccc}%
B\left(  1,1\right)  & B\left(  1,2\right)  & \cdots & B\left(  1,k\right) \\
B\left(  2,1\right)  & B\left(  2,2\right)  & \cdots & B\left(  2,k\right) \\
\vdots & \vdots & \ddots & \vdots\\
B\left(  k,1\right)  & B\left(  k,2\right)  & \cdots & B\left(  k,k\right)
\end{array}
\right)  \left(
\begin{array}
[c]{cccc}%
A_{1} &  &  & \\
& A_{2} &  & \\
&  & \ddots & \\
&  &  & A_{k}%
\end{array}
\right) \\
&  =\left(
\begin{array}
[c]{cccc}%
B\left(  1,1\right)  A_{1} & B\left(  1,2\right)  A_{2} & \cdots & B\left(
1,k\right)  A_{k}\\
B\left(  2,1\right)  A_{1} & B\left(  2,2\right)  A_{2} & \cdots & B\left(
2,k\right)  A_{k}\\
\vdots & \vdots & \ddots & \vdots\\
B\left(  k,1\right)  A_{1} & B\left(  k,2\right)  A_{2} & \cdots & B\left(
k,k\right)  A_{k}%
\end{array}
\right)  .
\end{align*}
However, these two matrices must be equal, since $\left(
\begin{array}
[c]{cccc}%
B\left(  1,1\right)  & B\left(  1,2\right)  & \cdots & B\left(  1,k\right) \\
B\left(  2,1\right)  & B\left(  2,2\right)  & \cdots & B\left(  2,k\right) \\
\vdots & \vdots & \ddots & \vdots\\
B\left(  k,1\right)  & B\left(  k,2\right)  & \cdots & B\left(  k,k\right)
\end{array}
\right)  \in\operatorname*{Cent}\left(
\begin{array}
[c]{cccc}%
A_{1} &  &  & \\
& A_{2} &  & \\
&  & \ddots & \\
&  &  & A_{k}%
\end{array}
\right)  $. Thus, we have%
\[
\left(
\begin{array}
[c]{cccc}%
A_{1}B\left(  1,1\right)  & A_{1}B\left(  1,2\right)  & \cdots & A_{1}B\left(
1,k\right) \\
A_{2}B\left(  2,1\right)  & A_{2}B\left(  2,2\right)  & \cdots & A_{2}B\left(
2,k\right) \\
\vdots & \vdots & \ddots & \vdots\\
A_{k}B\left(  k,1\right)  & A_{k}B\left(  k,2\right)  & \cdots & A_{k}B\left(
k,k\right)
\end{array}
\right)  =\left(
\begin{array}
[c]{cccc}%
B\left(  1,1\right)  A_{1} & B\left(  1,2\right)  A_{2} & \cdots & B\left(
1,k\right)  A_{k}\\
B\left(  2,1\right)  A_{1} & B\left(  2,2\right)  A_{2} & \cdots & B\left(
2,k\right)  A_{k}\\
\vdots & \vdots & \ddots & \vdots\\
B\left(  k,1\right)  A_{1} & B\left(  k,2\right)  A_{2} & \cdots & B\left(
k,k\right)  A_{k}%
\end{array}
\right)  .
\]
Comparing blocks, we can rewrite this as%
\[
A_{i}B\left(  i,j\right)  =B\left(  i,j\right)  A_{j}%
\ \ \ \ \ \ \ \ \ \ \text{for all }i,j\in\left[  k\right]  .
\]


Now, let $i,j\in\left[  k\right]  $ be distinct. Consider this equality
$A_{i}B\left(  i,j\right)  =B\left(  i,j\right)  A_{j}$. We can rewrite it as
$A_{i}B\left(  i,j\right)  -B\left(  i,j\right)  A_{j}=0$. Thus, $B\left(
i,j\right)  $ is an $n_{i}\times n_{j}$-matrix $X$ satisfying $A_{i}%
X-XA_{j}=0$. However, because $\sigma\left(  A_{i}\right)  \cap\sigma\left(
A_{j}\right)  =\varnothing$, a theorem we proved before (the $\mathcal{V}%
\Longrightarrow\mathcal{U}$ direction of Theorem
\ref{thm.schurtri.syl.equivalence}) tells us that there is a \textbf{unique}
$n_{i}\times n_{j}$-matrix $X$ satisfying $A_{i}X-XA_{j}=0$. Clearly, this
unique matrix $X$ must be the $0$ matrix (since the $0$ matrix satisfies
$A_{i}0-0A_{j}=0$). So we conclude that $B\left(  i,j\right)  $ is the $0$
matrix. In other words, $B\left(  i,j\right)  =0$.

So we have shown that $B\left(  i,j\right)  =0$ whenever $i$ and $j$ are
distinct. Thus,%
\[
B=\left(
\begin{array}
[c]{cccc}%
B\left(  1,1\right)  & B\left(  1,2\right)  & \cdots & B\left(  1,k\right) \\
B\left(  2,1\right)  & B\left(  2,2\right)  & \cdots & B\left(  2,k\right) \\
\vdots & \vdots & \ddots & \vdots\\
B\left(  k,1\right)  & B\left(  k,2\right)  & \cdots & B\left(  k,k\right)
\end{array}
\right)  =\left(
\begin{array}
[c]{cccc}%
B\left(  1,1\right)  &  &  & \\
& B\left(  2,2\right)  &  & \\
&  & \ddots & \\
&  &  & B\left(  k,k\right)
\end{array}
\right)  .
\]
This shows that $B$ is block-diagonal. Now, applying the equation%
\[
A_{i}B\left(  i,j\right)  =B\left(  i,j\right)  A_{j}%
\ \ \ \ \ \ \ \ \ \ \text{for all }i,j\in\left[  k\right]
\]
to $j=i$, we obtain $A_{i}B\left(  i,i\right)  =B\left(  i,i\right)  A_{i}$,
which of course means that $B\left(  i,i\right)  \in\operatorname*{Cent}%
\left(  A_{i}\right)  $. Thus, $B$ has the form $\left(
\begin{array}
[c]{cccc}%
B_{1} &  &  & \\
& B_{2} &  & \\
&  & \ddots & \\
&  &  & B_{k}%
\end{array}
\right)  $ where $B_{i}\in\operatorname*{Cent}\left(  A_{i}\right)  $ for each
$i\in\left[  k\right]  $. This completes the proof of Proposition
\ref{prop.jnf.cent.block-separate}.
\end{proof}

So we only need to compute $\operatorname*{Cent}J$ when $J$ is a Jordan matrix
with only one eigenvalue.

We can WLOG assume that this eigenvalue is $0$, since we know that
$\operatorname*{Cent}\left(  A-\lambda I_{n}\right)  =\operatorname*{Cent}A$.

So we only need to compute $\operatorname*{Cent}J$ when $J$ is a Jordan matrix
with zeroes on its diagonal.

If $J$ is just a single Jordan cell, we already know the result (by Theorem
\ref{thm.jnf.cent.Jn0}). In the general case, we have the following:

\begin{proposition}
\label{prop.jnf.cent.jordan-0}Let $J\in\mathbb{C}^{n\times n}$ be a Jordan
matrix whose Jordan blocks are%
\[
J_{n_{1}}\left(  0\right)  ,\ \ J_{n_{2}}\left(  0\right)  ,\ \ \ldots
,\ \ J_{n_{k}}\left(  0\right)  .
\]
Let $B$ be an $n\times n$-matrix, written as a block matrix%
\[
B=\left(
\begin{array}
[c]{cccc}%
B\left(  1,1\right)  & B\left(  1,2\right)  & \cdots & B\left(  1,k\right) \\
B\left(  2,1\right)  & B\left(  2,2\right)  & \cdots & B\left(  2,k\right) \\
\vdots & \vdots & \ddots & \vdots\\
B\left(  k,1\right)  & B\left(  k,2\right)  & \cdots & B\left(  k,k\right)
\end{array}
\right)  ,
\]
where each $B\left(  i,j\right)  $ is an $n_{i}\times n_{j}$-matrix. Then,
$B\in\operatorname*{Cent}J$ if and only if each of the $k^{2}$ blocks
$B\left(  i,j\right)  $ is an upper-triangular Toeplitz matrix in the wide sense.

Here, we say that a matrix is an \emph{upper-triangular Toeplitz matrix in the
wide sense} if it

\begin{itemize}
\item has the form $\left(
\begin{array}
[c]{cc}%
0 & U
\end{array}
\right)  $, where $U$ is an upper-triangular Toeplitz (square) matrix and $0$
is a zero matrix, or

\item has the form $\left(
\begin{array}
[c]{c}%
U\\
0
\end{array}
\right)  $, where $U$ is an upper-triangular Toeplitz (square) matrix and $0$
is a zero matrix.
\end{itemize}

(The zero matrices are allowed to be empty.)
\end{proposition}

\begin{proof}
Essentially the same argument that we used to prove Theorem
\ref{thm.jnf.cent.Jn0}, just with a lot more bookkeeping involved. See
\cite[Proposition 3.1.2]{OmClVi11} for details.
\end{proof}

We can summarize our results into a single theorem:

\begin{theorem}
Let $A\in\mathbb{C}^{n\times n}$ be an $n\times n$-matrix with Jordan
canonical form $J$. Then, $\operatorname*{Cent}A$ is a vector subspace of
$\mathbb{C}^{n\times n}$ with dimension%
\[
\sum_{\lambda\in\sigma\left(  A\right)  }g_{\lambda}\left(  A\right)  .
\]
Here, for each eigenvalue $\lambda$ of $A$, the number $g_{\lambda}\left(
A\right)  $ is a nonnegative integer defined as follows: Let $n_{1}%
,n_{2},\ldots,n_{k}$ be the sizes of the Jordan blocks at eigenvalue $\lambda$
that appear in $J$; then, we set%
\[
g_{\lambda}\left(  A\right)  :=\sum_{i=1}^{k}\ \ \sum_{j=1}^{k}\min\left\{
n_{i},n_{j}\right\}  .
\]

\end{theorem}

\begin{proof}
Combine Proposition \ref{prop.jnf.cent.similar}, Proposition
\ref{prop.jnf.cent.block-separate}, Proposition \ref{prop.jnf.cent.lambda},
and Proposition \ref{prop.jnf.cent.jordan-0}, and count the degrees of freedom.
\end{proof}

Now, let us return to the worst-case scenario: When is $\operatorname*{Cent}%
A=\left\{  f\left(  A\right)  \ \mid\ f\in\mathbb{C}\left[  t\right]
\right\}  $ ? We can answer this, too, although the proof takes longer.

\begin{definition}
An $n\times n$-matrix $A\in\mathbb{F}^{n\times n}$ is said to be
\emph{nonderogatory} if $q_{A}=p_{A}$ (that is, the minimal polynomial of $A$
equals the characteristic polynomial of $A$).
\end{definition}

\textquotedblleft Most\textquotedblright\ matrices are nonderogatory (in the
sense that a \textquotedblleft randomly chosen\textquotedblright\ matrix with
complex entries will be nonderogatory with probability $1$); but there are
exceptions. It is easy to see that if a matrix $A$ has $n$ distinct
eigenvalues, then $A$ is nonderogatory, but this is not an \textquotedblleft
if and only if\textquotedblright; a single Jordan cell is also nonderogatory.
Here is a necessary and sufficient criterion:

\begin{proposition}
An $n\times n$-matrix $A\in\mathbb{C}^{n\times n}$ is nonderogatory if and
only if its Jordan canonical form has exactly one Jordan block for each eigenvalue.
\end{proposition}

\begin{exercise}
\fbox{2} Prove this.
\end{exercise}

\begin{theorem}
\label{thm.jnf.cent.nonderog}Let $A\in\mathbb{C}^{n\times n}$ be an $n\times
n$-matrix. Then,
\[
\operatorname*{Cent}A=\left\{  f\left(  A\right)  \ \mid\ f\in\mathbb{C}%
\left[  t\right]  \right\}
\]
if and only if $A$ is nonderogatory. Moreover, in this case,%
\[
\operatorname*{Cent}A=\left\{  f\left(  A\right)  \ \mid\ f\in\mathbb{C}%
\left[  t\right]  \text{ is a polynomial of degree }\leq n-1\right\}  .
\]

\end{theorem}

\begin{exercise}
\fbox{8} Prove this.
\end{exercise}

\section{Hermitian matrices (\cite[Chapter 4]{HorJoh13})}

\textbf{Recall:} A \emph{Hermitian matrix} is an $n\times n$-matrix
$A\in\mathbb{C}^{n\times n}$ such that $A^{\ast}=A$.

Note that this is the complex analogue of real symmetric matrices (i.e.,
matrices $A\in\mathbb{R}^{n\times n}$ such that $A^{T}=A$).

If $A$ is a Hermitian matrix, then $A_{i,i}\in\mathbb{R}$ and $A_{i,j}%
=\overline{A_{j,i}}$.

For instance, the matrix $\left(
\begin{array}
[c]{ccc}%
-1 & i & 2\\
-i & 5 & 1-i\\
2 & 1+i & 0
\end{array}
\right)  $ is Hermitian.

\subsection{Basics}

\begin{theorem}
\label{thm.herm.ABCDE}Let $A\in\mathbb{C}^{n\times n}$ be an $n\times
n$-matrix. Then, the following are equivalent:

\begin{itemize}
\item $\mathcal{A}$: The matrix $A$ is Hermitian (i.e., we have $A^{\ast}=A$).

\item $\mathcal{B}$: We have $A=UDU^{\ast}$ for some unitary matrix
$U\in\mathbb{C}^{n\times n}$ and some real diagonal matrix $D\in
\mathbb{C}^{n\times n}$ (that is, $D$ is a diagonal matrix with real entries).

\item $\mathcal{C}$: The matrix $A$ is normal and its eigenvalues are real.

\item $\mathcal{D}$: We have $\left\langle Ax,x\right\rangle \in\mathbb{R}$
for each $x\in\mathbb{C}^{n}$.

\item $\mathcal{E}$: The matrix $S^{\ast}AS$ is Hermitian for all
$S\in\mathbb{C}^{n\times k}$ (for all $k\in\mathbb{N}$).
\end{itemize}
\end{theorem}

To prove this, we will need two lemmas:

\begin{lemma}
\label{lem.herm.Muv}Let $M\in\mathbb{C}^{n\times n}$ be an $n\times n$-matrix.
Let $u=\left(
\begin{array}
[c]{c}%
u_{1}\\
u_{2}\\
\vdots\\
u_{n}%
\end{array}
\right)  $ and $v=\left(
\begin{array}
[c]{c}%
v_{1}\\
v_{2}\\
\vdots\\
v_{n}%
\end{array}
\right)  $ be two vectors in $\mathbb{C}^{n}$. Then,%
\[
\left\langle Mu,v\right\rangle =\sum_{i=1}^{n}\ \ \sum_{j=1}^{n}M_{i,j}%
u_{j}\overline{v_{i}}.
\]

\end{lemma}

\begin{proof}
For each $i\in\left[  n\right]  $, let $w_{i}$ denote the $i$-th entry of the
column vector $w$. According to the definition of matrix multiplication, this
entry is given by%
\begin{equation}
w_{i}=M_{i,1}u_{1}+M_{i,2}u_{2}+\cdots+M_{i,n}u_{n}=\sum_{j=1}^{n}M_{i,j}%
u_{j}. \label{pf.lem.herm.Muv.wi=}%
\end{equation}
However, $Mu=\left(
\begin{array}
[c]{c}%
w_{1}\\
w_{2}\\
\vdots\\
w_{n}%
\end{array}
\right)  $ and $v=\left(
\begin{array}
[c]{c}%
v_{1}\\
v_{2}\\
\vdots\\
v_{n}%
\end{array}
\right)  $; therefore, the definition of the inner product yields%
\[
\left\langle Mu,v\right\rangle =w_{1}\overline{v_{1}}+w_{2}\overline{v_{2}%
}+\cdots+w_{n}\overline{v_{n}}=\sum_{i=1}^{n}\underbrace{w_{i}}%
_{\substack{=\sum_{j=1}^{n}M_{i,j}u_{j}\\\text{(by (\ref{pf.lem.herm.Muv.wi=}%
))}}}\overline{v_{i}}=\sum_{i=1}^{n}\ \ \sum_{j=1}^{n}M_{i,j}u_{j}%
\overline{v_{i}}.
\]
This proves Lemma \ref{lem.herm.Muv}.
\end{proof}

\begin{lemma}
\label{lem.herm.M=0}Let $M\in\mathbb{C}^{n\times n}$ be an $n\times n$-matrix.
Assume that $\left\langle Mx,x\right\rangle =0$ for each $x\in\mathbb{C}^{n}$.
Then, $M=0$.
\end{lemma}

\begin{proof}
[Proof of Lemma \ref{lem.herm.M=0}.]For every $x=\left(
\begin{array}
[c]{c}%
x_{1}\\
x_{2}\\
\vdots\\
x_{n}%
\end{array}
\right)  \in\mathbb{C}^{n}$, we have%
\begin{align*}
\left\langle Mx,x\right\rangle  &  =\sum_{i=1}^{n}\ \ \sum_{j=1}^{n}%
M_{i,j}x_{j}\overline{x_{i}}\ \ \ \ \ \ \ \ \ \ \left(  \text{by Lemma
\ref{lem.herm.Muv}}\right) \\
&  =\sum_{i=1}^{n}\ \ \sum_{j=1}^{n}M_{i,j}\overline{x_{i}}x_{j}%
\end{align*}
and therefore%
\begin{equation}
\sum_{i=1}^{n}\ \ \sum_{j=1}^{n}M_{i,j}\overline{x_{i}}x_{j}=\left\langle
Mx,x\right\rangle =0 \label{pf.lem.herm.M=0.=0}%
\end{equation}
(since we assumed that $\left\langle Mx,x\right\rangle =0$ for each
$x\in\mathbb{C}^{n}$). In particular:

\begin{itemize}
\item We can apply (\ref{pf.lem.herm.M=0.=0}) to $x=e_{1}=\left(
1,0,0,\ldots,0\right)  ^{T}$, and we obtain $M_{1,1}\cdot\overline{1}\cdot
1=0$, which means $M_{1,1}=0$. Similarly, we can find that%
\begin{equation}
M_{i,i}=0\ \ \ \ \ \ \ \ \ \ \text{for all }i\in\left[  n\right]  .
\label{pf.lem.herm.M=0.Mii=0}%
\end{equation}


\item We can apply (\ref{pf.lem.herm.M=0.=0}) to $x=e_{1}+e_{2}=\left(
1,1,0,0,\ldots,0\right)  ^{T}$, and we obtain
\[
M_{1,1}\cdot\overline{1}\cdot1+M_{1,2}\cdot\overline{1}\cdot1+M_{2,1}%
\cdot\overline{1}\cdot1\cdot M_{2,2}\cdot\overline{1}\cdot1=0.
\]
This simplifies to%
\[
M_{1,1}+M_{1,2}+M_{2,1}+M_{2,2}=0.
\]
However, the previous bullet point yields $M_{1,1}=0$ and $M_{2,2}=0$, so this
simplifies further to%
\[
M_{1,2}+M_{2,1}=0.
\]


\item We can apply (\ref{pf.lem.herm.M=0.=0}) to $x=e_{1}+ie_{2}=\left(
1,i,0,0,\ldots,0\right)  ^{T}$ (where $i=\sqrt{-1}$), and we obtain
\[
M_{1,1}\cdot\overline{1}\cdot1+M_{1,2}\cdot\overline{1}\cdot i+M_{2,1}%
\cdot\overline{i}\cdot1\cdot M_{2,2}\cdot\overline{i}\cdot i=0.
\]
This simplifies to%
\[
M_{1,1}+iM_{1,2}-iM_{2,1}+M_{2,2}=0.
\]
However, we know that $M_{1,1}=0$ and $M_{2,2}=0$, so this simplifies further
to%
\[
iM_{1,2}-iM_{2,1}=0.
\]
Thus,
\[
M_{1,2}-M_{2,1}=0.
\]
Adding this to%
\[
M_{1,2}+M_{2,1}=0,
\]
we obtain $2M_{1,2}=0$. In other words, $M_{1,2}=0$. Similarly, we can show
that%
\begin{equation}
M_{i,j}=0\ \ \ \ \ \ \ \ \ \ \text{for all }i\neq j.
\label{pf.lem.herm.M=0.Mij=0}%
\end{equation}

\end{itemize}

Combining (\ref{pf.lem.herm.M=0.Mii=0}) with (\ref{pf.lem.herm.M=0.Mij=0}), we
conclude that all entries of $M$ are $0$. In other words, $M=0$. This proves
Lemma \ref{lem.herm.M=0}.
\end{proof}

Now we can prove the theorem:

\begin{proof}
[Proof of Theorem \ref{thm.herm.ABCDE}.]The equivalence $\mathcal{A}%
\Longleftrightarrow\mathcal{B}$ has already been proved (it is Corollary
\ref{cor.schurtri.normal.hermitian-iff}). The implication $\mathcal{A}%
\Longrightarrow\mathcal{C}$ follows from Proposition
\ref{prop.schurtri.normal.classes} \textbf{(a)} and Proposition
\ref{prop.schurtri.normal.hermitian-spec} and Theorem
\ref{thm.schurtri.normal.spectral} \textbf{(b)}. The implication
$\mathcal{C}\Longrightarrow\mathcal{B}$ follows from Theorem
\ref{thm.schurtri.normal.spectral}. Combining these facts, we obtain the
equivalence $\mathcal{A}\Longleftrightarrow\mathcal{B}\Longleftrightarrow
\mathcal{C}$. So we only need to prove the equivalence $\mathcal{A}%
\Longleftrightarrow\mathcal{D}\Longleftrightarrow\mathcal{E}$.

\begin{itemize}
\item \textit{Proof of }$\mathcal{A}\Longrightarrow\mathcal{D}$\textit{:}
Assume that $\mathcal{A}$ holds. Thus, $A=A^{\ast}$. Now, let $x\in
\mathbb{C}^{n}$. Then, $\left\langle x,Ax\right\rangle =\overline{\left\langle
Ax,x\right\rangle }$ (by Proposition \ref{prop.unitary.innerprod.props}
\textbf{(b)}). However, by Proposition \ref{prop.unitary.innerprod.props}
\textbf{(a)}, we have%
\[
\left\langle Ax,x\right\rangle =x^{\ast}Ax\ \ \ \ \ \ \ \ \ \ \text{and}%
\ \ \ \ \ \ \ \ \ \ \left\langle x,Ax\right\rangle =\underbrace{\left(
Ax\right)  ^{\ast}}_{=x^{\ast}A^{\ast}}x=x^{\ast}\underbrace{A^{\ast}}%
_{=A}x=x^{\ast}Ax.
\]
Comparing these two equalities, we see that $\left\langle Ax,x\right\rangle
=\left\langle x,Ax\right\rangle =\overline{\left\langle Ax,x\right\rangle }$.
This entails $\left\langle Ax,x\right\rangle \in\mathbb{R}$ (since the only
complex numbers $z\in\mathbb{C}$ that satisfy $z=\overline{z}$ are the real
numbers). Thus, statement $\mathcal{D}$ is proved.

\item \textit{Proof of }$\mathcal{D}\Longrightarrow\mathcal{A}$\textit{:}
Assume that statement $\mathcal{D}$ holds. Thus, $\left\langle
Ax,x\right\rangle \in\mathbb{R}$ for each $x\in\mathbb{C}^{n}$. Again,
Proposition \ref{prop.unitary.innerprod.props} \textbf{(a)} shows that each
$x\in\mathbb{C}^{n}$ satisfies%
\[
\left\langle Ax,x\right\rangle =x^{\ast}Ax\ \ \ \ \ \ \ \ \ \ \text{and}%
\ \ \ \ \ \ \ \ \ \ \left\langle x,Ax\right\rangle =\underbrace{\left(
Ax\right)  ^{\ast}}_{=x^{\ast}A^{\ast}}x=x^{\ast}A^{\ast}x.
\]
Thus, each $x\in\mathbb{C}^{n}$ satisfies%
\begin{align*}
x^{\ast}Ax  &  =\left\langle Ax,x\right\rangle =\overline{\left\langle
Ax,x\right\rangle }\ \ \ \ \ \ \ \ \ \ \left(  \text{since }\left\langle
Ax,x\right\rangle \in\mathbb{R}\right) \\
&  =\left\langle x,Ax\right\rangle \ \ \ \ \ \ \ \ \ \ \left(  \text{by
Proposition \ref{prop.unitary.innerprod.props} \textbf{(b)}}\right) \\
&  =x^{\ast}A^{\ast}x
\end{align*}
and thus%
\[
x^{\ast}\left(  A^{\ast}-A\right)  x=x^{\ast}A^{\ast}x-\underbrace{x^{\ast}%
Ax}_{=x^{\ast}A^{\ast}x}=x^{\ast}A^{\ast}x-x^{\ast}A^{\ast}x=0.
\]
Applying Lemma \ref{lem.herm.M=0} to $M=A^{\ast}-A$, we thus conclude that
$A^{\ast}-A=0$. In other words, $A^{\ast}=A$. This proves statement
$\mathcal{A}$.

\item \textit{Proof of }$\mathcal{A}\Longrightarrow\mathcal{E}$\textit{:} If
$A$ is Hermitian, then $A^{\ast}=A$, so that every matrix $S\in\mathbb{C}%
^{n\times k}$ satisfies%
\[
\left(  S^{\ast}AS\right)  ^{\ast}=S^{\ast}\underbrace{A^{\ast}}%
_{=A}\underbrace{\left(  S^{\ast}\right)  ^{\ast}}_{=S}=S^{\ast}AS,
\]
and therefore $S^{\ast}AS$ is again Hermitian. This proves the implication
$\mathcal{A}\Longrightarrow\mathcal{E}$.

\item \textit{Proof of }$\mathcal{E}\Longrightarrow\mathcal{A}$\textit{:} If
statement $\mathcal{E}$ holds, then we can apply it to $S=I_{n}$ (and $k=n$),
and conclude that $I_{n}^{\ast}AI_{n}$ is Hermitian; but this is simply saying
that $A$ is Hermitian. So the implication $\mathcal{E}\Longrightarrow
\mathcal{A}$ follows.
\end{itemize}

Theorem \ref{thm.herm.ABCDE} is thus proved.
\end{proof}

\begin{exercise}
\fbox{1} \textbf{(a)} Prove the converse of Proposition
\ref{prop.unitary.innerprod.isometry.len}: If a matrix $A\in\mathbb{C}%
^{n\times k}$ satisfies $\left\vert \left\vert Ax\right\vert \right\vert
=\left\vert \left\vert x\right\vert \right\vert $ for each $x\in\mathbb{C}%
^{k}$, then $A$ is an isometry. \medskip

\textbf{(b)} Prove the converse of Exercise \ref{exe.schurtri.normal.A*x}
\textbf{(a)}: If a matrix $A\in\mathbb{C}^{n\times n}$ satisfies $\left\vert
\left\vert Ax\right\vert \right\vert =\left\vert \left\vert A^{\ast
}x\right\vert \right\vert $ for each $x\in\mathbb{C}^{n}$, then $A$ is normal.
\end{exercise}

Let us recall again that sums of Hermitian matrices are Hermitian, but
products are not (in general).

\subsection{Definiteness and semidefiniteness}

\begin{definition}
Let $A\in\mathbb{C}^{n\times n}$ be a Hermitian matrix. \medskip

\textbf{(a)} We say that $A$ is \emph{positive semidefinite} if it satisfies%
\[
\left\langle Ax,x\right\rangle \geq0\ \ \ \ \ \ \ \ \ \ \text{for all }%
x\in\mathbb{C}^{n}.
\]


\textbf{(b)} We say that $A$ is \emph{positive definite} if it satisfies%
\[
\left\langle Ax,x\right\rangle >0\ \ \ \ \ \ \ \ \ \ \text{for all nonzero
}x\in\mathbb{C}^{n}.
\]


\textbf{(c)} We say that $A$ is \emph{negative semidefinite} if it satisfies%
\[
\left\langle Ax,x\right\rangle \leq0\ \ \ \ \ \ \ \ \ \ \text{for all }%
x\in\mathbb{C}^{n}.
\]


\textbf{(d)} We say that $A$ is \emph{negative definite} if it satisfies%
\[
\left\langle Ax,x\right\rangle <0\ \ \ \ \ \ \ \ \ \ \text{for all nonzero
}x\in\mathbb{C}^{n}.
\]


\textbf{(e)} We say that $A$ is \emph{indefinite} if it is neither positive
semidefinite nor negative semidefinite, i.e., if there exist vectors
$x,y\in\mathbb{C}^{n}$ such that%
\[
\left\langle Ax,x\right\rangle <0<\left\langle Ay,y\right\rangle .
\]

\end{definition}

Here are some examples of matrices that are definite, semidefinite or neither:

\begin{example}
Let $n\in\mathbb{N}$. Let $J=\left(
\begin{array}
[c]{cccc}%
1 & 1 & \cdots & 1\\
1 & 1 & \cdots & 1\\
\vdots & \vdots & \ddots & \vdots\\
1 & 1 & \cdots & 1
\end{array}
\right)  $. This matrix $J$ is real symmetric, thus Hermitian. Is it positive
definite? Positive semidefinite? Let us see.

Let $x=\left(  x_{1},x_{2},\ldots,x_{n}\right)  ^{T}\in\mathbb{C}^{n}$. Then,
Lemma \ref{lem.herm.Muv} yields%
\begin{align*}
\left\langle Jx,x\right\rangle  &  =\sum_{i=1}^{n}\ \ \sum_{j=1}^{n}%
\overline{x_{j}}x_{i}=\left(  \sum_{j=1}^{n}\overline{x_{j}}\right)  \left(
\sum_{i=1}^{n}x_{i}\right)  =\left(  \sum_{i=1}^{n}\overline{x_{i}}\right)
\left(  \sum_{i=1}^{n}x_{i}\right) \\
&  =\left(  \overline{\sum_{i=1}^{n}x_{i}}\right)  \left(  \sum_{i=1}^{n}%
x_{i}\right)  =\left\vert \sum_{i=1}^{n}x_{i}\right\vert ^{2}\geq0.
\end{align*}
So $J$ is positive semidefinite.

Is $J$ positive definite? Again, let $x=\left(  x_{1},x_{2},\ldots
,x_{n}\right)  ^{T}\in\mathbb{C}^{n}$. We just have shown that $\left\langle
Jx,x\right\rangle =\left\vert \sum_{i=1}^{n}x_{i}\right\vert ^{2}$. Therefore,
to have $\left\langle Jx,x\right\rangle =0$ is equivalent to having
$\sum_{i=1}^{n}x_{i}=0$. When $n=1$ (or $n=0$), this is equivalent to having
$x=0$, so we can conclude that $J$ is positive definite in this case. However,
if $n>1$, then this is not equivalent to having $x=0$, and in fact the vector
$e_{1}-e_{2}$ is an example of a nonzero vector $x\in\mathbb{C}^{n}$ such that
$\left\langle Jx,x\right\rangle =0$. So $J$ is not positive definite unless
$n\leq1$.
\end{example}

\begin{example}
Consider a diagonal matrix
\[
D:=\operatorname*{diag}\left(  \lambda_{1},\lambda_{2},\ldots,\lambda
_{n}\right)  =\left(
\begin{array}
[c]{cccc}%
\lambda_{1} & 0 & \cdots & 0\\
0 & \lambda_{2} & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & \lambda_{n}%
\end{array}
\right)
\]
with $\lambda_{1},\lambda_{2},\ldots,\lambda_{n}\in\mathbb{R}$. When is $D$
positive semidefinite?

We want $\left\langle Dx,x\right\rangle \geq0$ for all $x\in\mathbb{C}^{n}$.
Let $x=\left(  x_{1},x_{2},\ldots,x_{n}\right)  ^{T}\in\mathbb{C}^{n}$. Then,%
\[
\left\langle Dx,x\right\rangle =\sum_{i=1}^{n}\lambda_{i}\underbrace{\overline
{x_{i}}x_{i}}_{=\left\vert x_{i}\right\vert ^{2}}=\sum_{i=1}^{n}\lambda
_{i}\left\vert x_{i}\right\vert ^{2}.
\]
If $\lambda_{1},\lambda_{2},\ldots,\lambda_{n}\geq0$, then we therefore
conclude that $\left\langle Dx,x\right\rangle \geq0$, so that $D$ is positive
semidefinite. Otherwise, $D$ is not positive semidefinite, since we can pick
an $x=e_{j}$ where $j$ satisfies $\lambda_{j}<0$. So $D$ is positive
semidefinite if and only if $\lambda_{1},\lambda_{2},\ldots,\lambda_{n}\geq0$.
A similar argument shows that $D$ is positive definite if and only if
$\lambda_{1},\lambda_{2},\ldots,\lambda_{n}>0$.
\end{example}

\begin{example}
The Hilbert matrix%
\[
\left(
\begin{array}
[c]{cccc}%
\dfrac{1}{1} & \dfrac{1}{2} & \cdots & \dfrac{1}{n}\\
\dfrac{1}{2} & \dfrac{1}{3} & \cdots & \dfrac{1}{n+1}\\
\vdots & \vdots & \ddots & \vdots\\
\dfrac{1}{n} & \dfrac{1}{n+1} & \cdots & \dfrac{1}{2n}%
\end{array}
\right)
\]
(i.e., the $n\times n$-matrix whose $\left(  i,j\right)  $-th entry is
$\dfrac{1}{i+j-1}$) is positive definite. In other words, for any $x=\left(
x_{1},x_{2},\ldots,x_{n}\right)  ^{T}\in\mathbb{C}^{n}$, we have%
\[
\sum_{i=1}^{n}\ \ \sum_{j=1}^{n}\dfrac{\overline{x_{i}}x_{j}}{i+j-1}\geq0.
\]
This is not obvious at all, and the proof will be the content of the next
exercise. More generally, if $a_{1},a_{2},\ldots,a_{n}$ are positive reals,
then the $n\times n$-matrix whose $\left(  i,j\right)  $-th entry is
$\dfrac{1}{a_{i}+a_{j}}$ is positive definite.
\end{example}

\begin{exercise}
\fbox{4} Let $a_{1},a_{2},\ldots,a_{n}$ be positive reals. Let $A$ be the
$n\times n$-matrix whose $\left(  i,j\right)  $-th entry is $\dfrac{1}%
{a_{i}+a_{j}}$. Prove that $A$ is positive definite. \medskip

[\textbf{Hint:} Recall that $\dfrac{1}{m}=\int_{0}^{1}t^{m-1}dt$ for each
$m>0$. Also, integrating an $\mathbb{R}_{\geq0}$-valued function over $\left[
0,1\right]  $ yields a nonnegative real.]
\end{exercise}

\begin{exercise}
\fbox{5} Let $a_{1},a_{2},\ldots,a_{n}$ be reals. Let $A\in\mathbb{R}^{n\times
n}$ be the $n\times n$-matrix%
\[
\left(
\begin{array}
[c]{ccccc}%
a_{1} & a_{1} & \cdots & a_{1} & a_{1}\\
a_{1} & a_{2} & \cdots & a_{2} & a_{2}\\
\vdots & \vdots & \ddots & \vdots & \vdots\\
a_{1} & a_{2} & \cdots & a_{n-1} & a_{n-1}\\
a_{1} & a_{2} & \cdots & a_{n-1} & a_{n}%
\end{array}
\right)
\]
(that is, the $n\times n$-matrix whose $\left(  i,j\right)  $-th entry is
$a_{\min\left\{  i,j\right\}  }$). This matrix $A$ is real symmetric and thus
Hermitian. \medskip

\textbf{(a)} Set $a_{0}:=0$, and let $d_{i}:=a_{i}-a_{i-1}$ for each
$i\in\left[  n\right]  $. Let $D$ be the diagonal matrix $\operatorname*{diag}%
\left(  d_{1},d_{2},\ldots,d_{n}\right)  \in\mathbb{R}^{n\times n}$. Let $U$
be the upper-triangular matrix
\[
\left(
\begin{array}
[c]{ccccc}%
1 & 1 & 1 & \cdots & 1\\
0 & 1 & 1 & \cdots & 1\\
0 & 0 & 1 & \cdots & 1\\
\vdots & \vdots & \vdots & \ddots & \vdots\\
0 & 0 & 0 & \cdots & 1
\end{array}
\right)  \in\mathbb{R}^{n\times n},
\]
all of whose entries on and above the main diagonal are $1$. Prove that
$A=U^{\ast}DU$. \medskip

\textbf{(b)} Prove that $A$ is positive definite if and only if $0<a_{1}%
<a_{2}<\cdots<a_{n}$.
\end{exercise}

Note that a Hermitian matrix $A$ is negative definite if and only if $-A$ is
positive definite. Similarly, a Hermitian matrix $A$ is negative semidefinite
if and only if $-A$ is positive semidefinite. (These claims follow easily from
the definitions.)

As an application of positive semidefiniteness, the Schoenberg theorem
generalizes the triangle inequality. Recall that the triangle inequality says
that three nonnegative real numbers $x,y,z$ are the mutual distances of $3$
points in the plane if and only if $x\leq y+z$ and $y\leq z+x$ and $z\leq
x+y$. More generally, Schoenberg's theorem gives a criterion for when a bunch
of nonnegative reals can be realized as mutual distances of $n$ points in an
$r$-dimensional real vector space:

\begin{theorem}
[Schoenberg's theorem]Let $n\in\mathbb{N}$ and $r\in\mathbb{N}$. Let $d_{i,j}$
be a nonnegative real for each $i,j\in\left[  n\right]  $. Assume that
$d_{i,i}=0$ for all $i\in\left[  n\right]  $, and furthermore $d_{i,j}%
=d_{j,i}$ for all $i,j\in\left[  n\right]  $. Then, there exist $n$ points
$P_{1},P_{2},\ldots,P_{n}\in\mathbb{R}^{r}$ satisfying%
\[
\left\vert P_{i}-P_{j}\right\vert =d_{i,j}\ \ \ \ \ \ \ \ \ \ \text{for all
}i,j\in\left[  n\right]
\]
if and only if the $\left(  n-1\right)  \times\left(  n-1\right)  $-matrix
whose $\left(  i,j\right)  $-th entry is
\[
d_{i,n}^{2}+d_{j,n}^{2}-d_{i,j}^{2}\ \ \ \ \ \ \ \ \ \ \text{for all }%
i,j\in\left[  n-1\right]
\]
is positive semidefinite and has rank $\leq r$.
\end{theorem}

We will not prove this here. (See \cite[Theorem 7.1]{LibLav15} for a proof.)

\begin{remark}
If $A\in\mathbb{R}^{n\times n}$ and $\left\langle Ax,x\right\rangle \geq0$ for
all $x\in\mathbb{R}^{n}$, then we \textbf{cannot} conclude that $A$ is
positive semidefinite. The reason is that it does not follow that $A$ is
symmetric. For example, $A=\left(
\begin{array}
[c]{cc}%
2 & 1\\
0 & 2
\end{array}
\right)  $ satisfies
\[
\left\langle Ax,x\right\rangle =2x_{1}^{2}+x_{1}x_{2}+2x_{2}^{2}=\dfrac{1}%
{2}\left(  x_{1}+x_{2}\right)  ^{2}+\dfrac{3}{2}\left(  x_{1}^{2}+x_{2}%
^{2}\right)  \geq0
\]
for each $x=\left(
\begin{array}
[c]{c}%
x_{1}\\
x_{2}%
\end{array}
\right)  \in\mathbb{R}^{2}$, but it is not symmetric.
\end{remark}

\begin{theorem}
\label{thm.herm.pos-semdef-using-evals}Let $A\in\mathbb{C}^{n\times n}$ be a
Hermitian matrix. Then: \medskip

\textbf{(a)} The matrix $A$ is positive semidefinite if and only if all
eigenvalues of $A$ are nonnegative. \medskip

\textbf{(b)} The matrix $A$ is positive definite if and only if all
eigenvalues of $A$ are positive. \medskip

(Recall that the eigenvalues of $A$ are real by the spectral theorem.)
\end{theorem}

\begin{proof}
By the spectral theorem (Corollary \ref{cor.schurtri.normal.hermitian-iff}),
the matrix $A$ is unitarily similar to a diagonal matrix with real entries. In
other words, $A=UDU^{\ast}$ for some unitary matrix $U\in\operatorname*{U}%
\nolimits_{n}\left(  \mathbb{C}\right)  $ and some diagonal matrix
$D\in\mathbb{C}^{n\times n}$ that has real entries. Consider these $U$ and
$D$. From Theorem \ref{thm.schurtri.normal.spectral} \textbf{(b)}, we know
that the diagonal entries of $D$ are the eigenvalues of $A$. Let $\lambda
_{1},\lambda_{2},\ldots,\lambda_{n}$ be the diagonal entries of $D$, so that
$D=\operatorname*{diag}\left(  \lambda_{1},\lambda_{2},\ldots,\lambda
_{n}\right)  $. Then, $\lambda_{1},\lambda_{2},\ldots,\lambda_{n}$ are the
eigenvalues of $A$ (since the diagonal entries of $D$ are the eigenvalues of
$A$). \medskip

\textbf{(a)} $\Longrightarrow:$ Assume that $A$ is positive semidefinite. Let
$\lambda$ be an eigenvalue of $A$. Let $x\neq0$ be a corresponding
eigenvector. Then, $Ax=\lambda x$. However, $\left\langle Ax,x\right\rangle
\geq0$ since $A$ is positive semidefinite. However, from $Ax=\lambda x$, we
obtain $\left\langle Ax,x\right\rangle =\left\langle \lambda x,x\right\rangle
=\lambda\left\langle x,x\right\rangle $. Thus, $\lambda\left\langle
x,x\right\rangle =\left\langle Ax,x\right\rangle \geq0$. We can cancel
$\left\langle x,x\right\rangle $ from this inequality (since $\left\langle
x,x\right\rangle >0$). Thus, we get $\lambda\geq0$. Therefore, all eigenvalues
of $A$ are $\geq0$.

$\Longleftarrow:$ Assume that all eigenvalues of $A$ are $\geq0$. In other
words, $\lambda_{1},\lambda_{2},\ldots,\lambda_{n}\geq0$ (since $\lambda
_{1},\lambda_{2},\ldots,\lambda_{n}$ are the eigenvalues of $A$). Thus, the
square roots $\sqrt{\lambda_{1}},\sqrt{\lambda_{2}},\ldots,\sqrt{\lambda_{n}}$
are well-defined nonnegative reals. Set $E:=\operatorname*{diag}\left(
\sqrt{\lambda_{1}},\sqrt{\lambda_{2}},\ldots,\sqrt{\lambda_{n}}\right)
\in\mathbb{C}^{n\times n}$. Then,%
\begin{align*}
E^{2}  &  =\left(  \operatorname*{diag}\left(  \sqrt{\lambda_{1}}%
,\sqrt{\lambda_{2}},\ldots,\sqrt{\lambda_{n}}\right)  \right)  ^{2}%
=\operatorname*{diag}\left(  \left(  \sqrt{\lambda_{1}}\right)  ^{2},\left(
\sqrt{\lambda_{2}}\right)  ^{2},\ldots,\left(  \sqrt{\lambda_{n}}\right)
^{2}\right) \\
&  =\operatorname*{diag}\left(  \lambda_{1},\lambda_{2},\ldots,\lambda
_{n}\right)  =D,
\end{align*}
so that $D=E^{2}=EE$. Moreover, $E^{\ast}=E$, since $E$ is a diagonal matrix
with real entries. Now,%
\[
A=U\underbrace{D}_{=EE}U^{\ast}=UE\underbrace{E}_{=E^{\ast}}U^{\ast
}=\underbrace{UE}_{=\left(  \left(  UE\right)  ^{\ast}\right)  ^{\ast}%
}\underbrace{E^{\ast}U^{\ast}}_{=\left(  UE\right)  ^{\ast}}=\left(  \left(
UE\right)  ^{\ast}\right)  ^{\ast}\left(  UE\right)  ^{\ast}.
\]
Hence, for each $x\in\mathbb{C}^{n}$, we have%
\begin{align*}
\left\langle Ax,x\right\rangle  &  =x^{\ast}Ax\ \ \ \ \ \ \ \ \ \ \left(
\text{by the formula }\left\langle u,v\right\rangle =v^{\ast}u\right) \\
&  =x^{\ast}\left(  \left(  UE\right)  ^{\ast}\right)  ^{\ast}\left(
UE\right)  ^{\ast}x\ \ \ \ \ \ \ \ \ \ \left(  \text{since }A=\left(  \left(
UE\right)  ^{\ast}\right)  ^{\ast}\left(  UE\right)  ^{\ast}\right) \\
&  =\left\langle \left(  UE\right)  ^{\ast}x,\ \left(  UE\right)  ^{\ast
}x\right\rangle \ \ \ \ \ \ \ \ \ \ \left(  \text{by the formula }\left\langle
u,v\right\rangle =v^{\ast}u\right) \\
&  \geq0\ \ \ \ \ \ \ \ \ \ \left(  \text{since }\left\langle u,u\right\rangle
\geq0\text{ for each }u\in\mathbb{C}^{n}\right)  .
\end{align*}
Thus, $A$ is positive semidefinite.

We have thus proved Proposition \ref{thm.herm.pos-semdef-using-evals}
\textbf{(a)}. The proof of Proposition \ref{thm.herm.pos-semdef-using-evals}
\textbf{(b)} is similar, except that we need to also observe that $x\neq0$
entails $\left(  UE\right)  ^{\ast}x\neq0$ (because $U$ and $E$ are
invertible, thus $UE$ is invertible, thus $\left(  UE\right)  ^{\ast}$ is invertible).
\end{proof}

\begin{exercise}
\fbox{4} Let $A,B\in\mathbb{C}^{n\times n}$ be two positive definite Hermitian
matrices. \medskip

\textbf{(a)} Prove that $A+B$ is positive definite. \medskip

\textbf{(b)} Find a counterexample showing that $AB$ is not necessarily
Hermitian. \medskip

\textbf{(c)} Now assume that $AB=BA$. Prove that $AB$ is Hermitian and
positive definite. \medskip

[\textbf{Hint:} In part \textbf{(c)}, use Exercise \ref{exe.schurtri.normal.F}.]
\end{exercise}

\subsection{The Cholesky decomposition}

\begin{theorem}
[Cholesky decomposition for positive definite matrices]%
\label{thm.herm.cholesky.pd}Let $A\in\mathbb{C}^{n\times n}$ be a positive
definite Hermitian matrix. Then, $A$ has a unique factorization of the form%
\[
A=LL^{\ast},
\]
where $L\in\mathbb{C}^{n\times n}$ is a lower-triangular matrix whose diagonal
entries are positive reals.
\end{theorem}

\begin{example}
\label{exa.herm.cholesky.pd.1}For $n=1$, the theorem is trivial: In this case,
$A=\left(
\begin{array}
[c]{c}%
a
\end{array}
\right)  $ for some $a\in\mathbb{R}$, and this $a$ is $>0$ because $A$ is
positive definite. Thus, setting $L=\left(
\begin{array}
[c]{c}%
\sqrt{a}%
\end{array}
\right)  $, we obtain $A=LL^{\ast}$. Moreover, this is clearly the only choice
for $L$.
\end{example}

\begin{example}
Let us manually verify Theorem \ref{thm.herm.cholesky.pd} for $n=2$. Let
$A=\left(
\begin{array}
[c]{cc}%
a & b\\
c & d
\end{array}
\right)  $ be a positive definite Hermitian matrix. We are looking for a
lower-triangular matrix $L=\left(
\begin{array}
[c]{cc}%
\lambda & 0\\
x & \delta
\end{array}
\right)  $ whose diagonal entries $\lambda$ and $\delta$ are positive reals
that satisfies $A=LL^{\ast}$.

So we need%
\begin{align*}
\left(
\begin{array}
[c]{cc}%
a & b\\
c & d
\end{array}
\right)   &  =A=LL^{\ast}=\left(
\begin{array}
[c]{cc}%
\lambda & 0\\
x & \delta
\end{array}
\right)  \left(
\begin{array}
[c]{cc}%
\lambda & 0\\
x & \delta
\end{array}
\right)  ^{\ast}\\
&  =\left(
\begin{array}
[c]{cc}%
\lambda & 0\\
x & \delta
\end{array}
\right)  \left(
\begin{array}
[c]{cc}%
\lambda & \overline{x}\\
0 & \delta
\end{array}
\right)  \ \ \ \ \ \ \ \ \ \ \left(  \text{since }\lambda,\delta\text{ are
real}\right) \\
&  =\left(
\begin{array}
[c]{cc}%
\lambda^{2} & \lambda\overline{x}\\
\lambda x & x\overline{x}+\delta^{2}%
\end{array}
\right)  =\left(
\begin{array}
[c]{cc}%
\lambda^{2} & \lambda\overline{x}\\
\lambda x & \left\vert x\right\vert ^{2}+\delta^{2}%
\end{array}
\right)  .
\end{align*}
So we need to solve the system of equations%
\[
\left\{
\begin{array}
[c]{c}%
a=\lambda^{2};\\
b=\lambda\overline{x};\\
c=\lambda x;\\
d=\left\vert x\right\vert ^{2}+\delta^{2}.
\end{array}
\right.
\]


First, we solve the equation $a=\lambda^{2}$ by setting $\lambda=\sqrt{a}$.
Since $A$ is positive definite, we have $a=\left\langle Ae_{1},e_{1}%
\right\rangle >0$, so that $\sqrt{a}$ is well-defined, and we get a positive
real $\lambda$. Next, we solve the equation $c=\lambda x$ by setting
$x=\dfrac{c}{\lambda}$. Next, the equation $b=\lambda\overline{x}$ is
automatically satisfied, since the Hermitianness of $A$ entails $b=\overline
{c}=\overline{\lambda x}=\lambda\overline{x}$ (since $\lambda$ is real).
Finally, we solve the equation $d=\left\vert x\right\vert ^{2}+\delta^{2}$ by
setting $\delta=\sqrt{d-\left\vert x\right\vert ^{2}}$. Here, we need to
convince ourselves that $d-\left\vert x\right\vert ^{2}$ is a positive real,
i.e., that $d>\left\vert x\right\vert ^{2}$. Why is this the case?

I claim that this follows from applying $\left\langle Az,z\right\rangle \geq0$
to the vector $z=\left(
\begin{array}
[c]{c}%
b\\
-a
\end{array}
\right)  $. Indeed, setting $z=\left(
\begin{array}
[c]{c}%
b\\
-a
\end{array}
\right)  $, we obtain%
\[
Az=\left(
\begin{array}
[c]{cc}%
a & b\\
c & d
\end{array}
\right)  \left(
\begin{array}
[c]{c}%
b\\
-a
\end{array}
\right)  =\left(
\begin{array}
[c]{c}%
0\\
bc-ad
\end{array}
\right)  ,
\]
so that%
\[
\left\langle Az,z\right\rangle =\left\langle \left(
\begin{array}
[c]{c}%
0\\
bc-ad
\end{array}
\right)  ,\left(
\begin{array}
[c]{c}%
b\\
-a
\end{array}
\right)  \right\rangle =\left(  bc-ad\right)  \overline{-a}=a\left(
ad-bc\right)
\]
and thus $a\left(  ad-bc\right)  =\left\langle Az,z\right\rangle >0$ (by the
positive definiteness of $A$, since $z\neq0$). We can divide this inequality
by $a$ (since $a>0$), and obtain $ad-bc>0$. Now, recall that $x=\dfrac
{c}{\lambda}$ and $\lambda=\sqrt{a}$. Hence,%
\begin{align*}
d-\left\vert x\right\vert ^{2}  &  =d-\left\vert \dfrac{c}{\lambda}\right\vert
^{2}=d-\dfrac{c\overline{c}}{\lambda^{2}}=d-\dfrac{cb}{a}%
\ \ \ \ \ \ \ \ \ \ \left(  \text{since }\overline{c}=b\text{ and }\lambda
^{2}=a\right) \\
&  =\dfrac{ad-bc}{a}>0\ \ \ \ \ \ \ \ \ \ \left(  \text{since }ad-bc>0\text{
and }a>0\right)  .
\end{align*}
This is what we need. So Theorem \ref{thm.herm.cholesky.pd} is proved for
$n=2$.
\end{example}

To prove Theorem \ref{thm.herm.cholesky.pd} in general, we need a lemma that
essentially generalizes our above argument for $d-\left\vert x\right\vert
^{2}>0$:

\begin{lemma}
\label{lem.herm.cholesky.x2d}Let $Q\in\mathbb{C}^{n\times n}$ be a invertible
matrix. Let $x\in\mathbb{C}^{n}$ be some column vector. Let $d\in\mathbb{R}$.
Let%
\[
A:=\left(
\begin{array}
[c]{cc}%
QQ^{\ast} & Qx\\
\left(  Qx\right)  ^{\ast} & d
\end{array}
\right)  \in\mathbb{C}^{\left(  n+1\right)  \times\left(  n+1\right)  }.
\]
Assume that $A$ is positive definite. Then, $\left\vert \left\vert
x\right\vert \right\vert ^{2}<d$.
\end{lemma}

\begin{proof}
[Proof of Lemma \ref{lem.herm.cholesky.x2d}.]Set $Q^{-\ast}:=\left(
Q^{-1}\right)  ^{\ast}=\left(  Q^{\ast}\right)  ^{-1}$. (This is well-defined,
since $Q$ is invertible.) Set $u=\left(
\begin{array}
[c]{c}%
Q^{-\ast}x\\
-1
\end{array}
\right)  \in\mathbb{C}^{n+1}$. (This is in block-matrix notation. Explicitly,
this is the column vector obtained by appending the extra entry $-1$ at the
bottom of $Q^{-\ast}x$.)

The definitions of $A$ and $u$ yield%
\begin{align*}
Au  &  =\left(
\begin{array}
[c]{cc}%
QQ^{\ast} & Qx\\
\left(  Qx\right)  ^{\ast} & d
\end{array}
\right)  \left(
\begin{array}
[c]{c}%
Q^{-\ast}x\\
-1
\end{array}
\right)  =\left(
\begin{array}
[c]{c}%
QQ^{\ast}Q^{-\ast}x+Qx\left(  -1\right) \\
\left(  Qx\right)  ^{\ast}Q^{-\ast}x+d\left(  -1\right)
\end{array}
\right) \\
&  =\left(
\begin{array}
[c]{c}%
0\\
x^{\ast}Q^{\ast}Q^{-\ast}x-d
\end{array}
\right)  =\left(
\begin{array}
[c]{c}%
0\\
x^{\ast}x-d
\end{array}
\right)  =\left(
\begin{array}
[c]{c}%
0\\
\left\vert \left\vert x\right\vert \right\vert ^{2}-d
\end{array}
\right)
\end{align*}
(since $x^{\ast}x=\left\langle x,x\right\rangle =\left\vert \left\vert
x\right\vert \right\vert ^{2}$). Hence,%
\[
\left\langle Au,u\right\rangle =\left\langle \left(
\begin{array}
[c]{c}%
0\\
\left\vert \left\vert x\right\vert \right\vert ^{2}-d
\end{array}
\right)  ,\left(
\begin{array}
[c]{c}%
Q^{-\ast}x\\
-1
\end{array}
\right)  \right\rangle =\left(  \left\vert \left\vert x\right\vert \right\vert
^{2}-d\right)  \left(  \overline{-1}\right)  =d-\left\vert \left\vert
x\right\vert \right\vert ^{2}.
\]
However, the vector $u$ is nonzero (since its last entry is $-1$), and the
matrix $A$ is positive definite (by assumption). Thus, $\left\langle
Au,u\right\rangle >0$. Since $\left\langle Au,u\right\rangle =d-\left\vert
\left\vert x\right\vert \right\vert ^{2}$, we thus obtain $d-\left\vert
\left\vert x\right\vert \right\vert ^{2}>0$. In other words, $d>\left\vert
\left\vert x\right\vert \right\vert ^{2}$. This proves Lemma
\ref{lem.herm.cholesky.x2d}.
\end{proof}

Now, let us prove the Cholesky factorization theorem:

\begin{proof}
[Proof of Theorem \ref{thm.herm.cholesky.pd}.]We proceed by induction on $n$.

The \textit{base cases} $n=0$ and $n=1$ are essentially obvious ($n=1$ was
done in Example \ref{exa.herm.cholesky.pd.1}).

\textit{Induction step:} Assume that Theorem \ref{thm.herm.cholesky.pd} holds
for some $n$. We must prove that it holds for $n+1$ as well.

Let $A\in\mathbb{C}^{\left(  n+1\right)  \times\left(  n+1\right)  }$ be a
positive definite Hermitian matrix. Write $A$ in the block-matrix form%
\[
A=\left(
\begin{array}
[c]{cc}%
B & b\\
b^{\ast} & d
\end{array}
\right)  ,
\]
where $B\in\mathbb{C}^{n\times n}$ and $b\in\mathbb{C}^{n}$ and $d\in
\mathbb{C}$. Note that the $b^{\ast}$ on the bottom of the right hand side is
because $A$ is Hermitian, so all entries in the last row of $A$ are the
complex conjugates of the corresponding entries in the last column of $A$.
Also, $d=d^{\ast}$ for the same reason, so $d\in\mathbb{R}$. Moreover, $B$ is
Hermitian (since $A$ is Hermitian).

Next, we claim that $B$ is positive definite. Indeed, for any nonzero vector
$x\in\mathbb{C}^{n}$, we have $\left\langle Bx,x\right\rangle =\left\langle
Ax^{\prime},x^{\prime}\right\rangle $, where $x^{\prime}$ is the nonzero
vector $\left(
\begin{array}
[c]{c}%
x\\
0
\end{array}
\right)  \in\mathbb{C}^{n+1}$. Thus, positive definiteness of $B$ follows from
positive definiteness of $A$. (More generally, any principal submatrix of a
positive definite matrix is positive definite.)

Therefore, by the induction hypothesis, we can apply Theorem
\ref{thm.herm.cholesky.pd} to the $n\times n$-matrix $B$ instead of $A$. We
conclude that $B$ can be uniquely written as a product $B=QQ^{\ast}$, where
$Q\in\mathbb{C}^{n\times n}$ is a lower-triangular matrix whose diagonal
entries are positive reals. Consider this $Q$. Note that the matrix $Q$ is
invertible (since it is lower-triangular and its diagonal entries are positive).

Now, we want to find a vector $x\in\mathbb{C}^{n}$ and a positive real
$\delta$ such that if we set%
\[
L:=\left(
\begin{array}
[c]{cc}%
Q & 0\\
x^{\ast} & \delta
\end{array}
\right)  ,
\]
then $A=LL^{\ast}$. If we can find such $x$ and $\delta$, then at least the
existence part of Theorem \ref{thm.herm.cholesky.pd} will be settled.

So let us set $L:=\left(
\begin{array}
[c]{cc}%
Q & 0\\
x^{\ast} & \delta
\end{array}
\right)  $, and see what conditions $A=LL^{\ast}$ places on $x$ and $\delta$.
We want%
\begin{align*}
\left(
\begin{array}
[c]{cc}%
B & b\\
b^{\ast} & d
\end{array}
\right)   &  =A=LL^{\ast}=\left(
\begin{array}
[c]{cc}%
Q & 0\\
x^{\ast} & \delta
\end{array}
\right)  \left(
\begin{array}
[c]{cc}%
Q^{\ast} & \left(  x^{\ast}\right)  ^{\ast}\\
0 & \overline{\delta}%
\end{array}
\right) \\
&  =\left(
\begin{array}
[c]{cc}%
Q & 0\\
x^{\ast} & \delta
\end{array}
\right)  \left(
\begin{array}
[c]{cc}%
Q^{\ast} & x\\
0 & \delta
\end{array}
\right)  \ \ \ \ \ \ \ \ \ \ \left(  \text{since }\delta\in\mathbb{R}\text{
entails }\overline{\delta}=\delta\right) \\
&  =\left(
\begin{array}
[c]{cc}%
QQ^{\ast} & Qx\\
x^{\ast}Q^{\ast} & x^{\ast}x+\delta^{2}%
\end{array}
\right)  .
\end{align*}
In other words, we want%
\[
\left\{
\begin{array}
[c]{c}%
B=QQ^{\ast};\\
b=Qx;\\
b^{\ast}=x^{\ast}Q^{\ast};\\
d=x^{\ast}x+\delta^{2}.
\end{array}
\right.
\]
The first of these four equations is already satisfied (we know that
$B=QQ^{\ast}$). The second equation will be satisfied if we set $x=Q^{-1}b$.
We can indeed set $x=Q^{-1}b$, since the matrix $Q$ is invertible. The third
equation follows automatically from the second (indeed, $b=Qx$ entails
$b^{\ast}=\left(  Qx\right)  ^{\ast}=x^{\ast}Q^{\ast}$). Finally, the fourth
equation rewrites as $d=\left\vert \left\vert x\right\vert \right\vert
^{2}+\delta^{2}$. We can satisfy it by setting $\delta=\sqrt{d-\left\vert
\left\vert x\right\vert \right\vert ^{2}}$, as long as we can show that
$d-\left\vert \left\vert x\right\vert \right\vert ^{2}>0$. Fortunately, we can
indeed show this, because Lemma \ref{lem.herm.cholesky.x2d} yields that
$\left\vert \left\vert x\right\vert \right\vert ^{2}<d$. Thus, we have found
$x$ and $\delta$, and constructed a lower-triangular matrix $L$ whose diagonal
entries are positive reals and which satisfies $A=LL^{\ast}$.

It remains to show that this $L$ is unique. Indeed, we can basically read our
argument above backwards. \textbf{If} $L\in\mathbb{C}^{\left(  n+1\right)
\times\left(  n+1\right)  }$ is a lower-triangular matrix whose diagonal
entries are positive reals and which satisfies $A=LL^{\ast}$, then we can
write $A$ in the form $A=\left(
\begin{array}
[c]{cc}%
Q & 0\\
x^{\ast} & \delta
\end{array}
\right)  $ for some $Q\in\mathbb{C}^{n\times n}$ and $x\in\mathbb{C}^{n}$ and
some positive real $\delta$, where $Q$ is lower-triangular with its diagonal
entries being real. The equation $A=LL^{\ast}$ then rewrites as%
\begin{equation}
\left(
\begin{array}
[c]{cc}%
B & b\\
b^{\ast} & d
\end{array}
\right)  =\left(
\begin{array}
[c]{cc}%
QQ^{\ast} & Qx\\
x^{\ast}Q^{\ast} & x^{\ast}x+\delta^{2}%
\end{array}
\right)  . \label{pf.thm.herm.cholesky.pd.uniq.2}%
\end{equation}
Thus, in particular, $B=QQ^{\ast}$. By the induction hypothesis, the
lower-triangular matrix $Q\in\mathbb{C}^{n\times n}$ with real diagonal
entries that satisfies $B=QQ^{\ast}$ is unique. Hence, our new $Q$ is exactly
the $Q$ that was constructed above. Furthermore,
(\ref{pf.thm.herm.cholesky.pd.uniq.2}) shows that $b=Qx$, so that $x=Q^{-1}b$,
so again our new $x$ is our old $x$. Finally,
(\ref{pf.thm.herm.cholesky.pd.uniq.2}) yields $d=x^{\ast}x+\delta^{2}$, whence
$\delta^{2}=d-x^{\ast}x=d-\left\vert \left\vert x\right\vert \right\vert ^{2}%
$. Thus, $\delta=\sqrt{d-\left\vert \left\vert x\right\vert \right\vert ^{2}}%
$, because $\delta$ has to be positive. So our $\delta$ is our old $\delta$.
Thus, our $L$ is the $L$ that we constructed above. This proves the uniqueness
of the $L$. Theorem \ref{thm.herm.cholesky.pd} is proved.
\end{proof}

Theorem \ref{thm.herm.cholesky.pd} can be used to prove several facts about
positive definite matrices:

\begin{exercise}
\fbox{2} Let $A\in\mathbb{C}^{n\times n}$ be a positive definite Hermitian
matrix. Prove that $\det A$ is a positive real.
\end{exercise}

\begin{exercise}
\fbox{4} Let $n>0$. Let $A$ and $B$ be two positive definite Hermitian
matrices in $\mathbb{C}^{n\times n}$. Prove that $\operatorname*{Tr}\left(
AB\right)  $ is real and $\operatorname*{Tr}\left(  AB\right)  >0$.
\end{exercise}

\begin{exercise}
\fbox{4} Let $A\in\mathbb{R}^{n\times n}$ be a symmetric matrix with real
entries. Assume that every nonzero vector $x\in\mathbb{R}^{n}$ (with
\textbf{real} entries) satisfies $\left\langle Ax,x\right\rangle >0$. Prove
that $A$ is positive definite.
\end{exercise}

Some properties of positive semidefinite matrices can be deduced from
corresponding properties of positive definite matrices:

\begin{exercise}
\fbox{4} Let $A\in\mathbb{C}^{n\times n}$ be a positive semidefinite Hermitian
matrix. \medskip

\textbf{(a)} Prove that $A+\varepsilon I_{n}$ is positive definite whenever
$\varepsilon$ is a positive real number. \medskip

\textbf{(b)} Prove that $\det A$ is a nonnegative real. \medskip

\textbf{(c)} Let $B\in\mathbb{C}^{n\times n}$ be a further positive
semidefinite Hermitian matrix. Prove that $\operatorname*{Tr}\left(
AB\right)  $ is real and $\operatorname*{Tr}\left(  AB\right)  \geq0$.
\end{exercise}

There is a version of Cholesky decomposition for positive semidefinite
matrices, but we omit it for now.

\begin{noncompile}
TODO: Add exercises on Cholesky for positive semidefinite matrices.

TODO: Add exercises on determinantal criteria?
\end{noncompile}

\subsection{Rayleigh quotients}

\subsubsection{Definition and basic properties}

\begin{definition}
\label{def.herm.rayleigh}Let $A\in\mathbb{C}^{n\times n}$ be a Hermitian
matrix, and $x\in\mathbb{C}^{n}$ be a nonzero vector. Then, the \emph{Rayleigh
quotient} for $A$ and $x$ is defined to be the real number%
\[
R\left(  A,x\right)  :=\dfrac{\left\langle Ax,x\right\rangle }{\left\langle
x,x\right\rangle }=\dfrac{x^{\ast}Ax}{x^{\ast}x}=\dfrac{x^{\ast}Ax}{\left\vert
\left\vert x\right\vert \right\vert ^{2}}.
\]

\end{definition}

\begin{proposition}
\label{prop.herm.rayleigh-y}Let $A\in\mathbb{C}^{n\times n}$ be a Hermitian
matrix, and $x\in\mathbb{C}^{n}$ be a nonzero vector. Let $y=\dfrac
{x}{\left\vert \left\vert x\right\vert \right\vert }$. Then,
\[
R\left(  A,x\right)  =R\left(  A,y\right)  =y^{\ast}Ay.
\]

\end{proposition}

\begin{proof}
Let $\lambda=\left\vert \left\vert x\right\vert \right\vert $. Thus,
$y=\dfrac{x}{\lambda}$, so that $x=\lambda y$. Hence,%
\[
R\left(  A,x\right)  =\dfrac{\left\langle Ax,x\right\rangle }{\left\langle
x,x\right\rangle }=\dfrac{\left\langle A\cdot\lambda y,\lambda y\right\rangle
}{\left\langle \lambda y,\lambda y\right\rangle }=\dfrac{\overline{\lambda
}\lambda\left\langle Ay,y\right\rangle }{\overline{\lambda}\lambda\left\langle
y,y\right\rangle }=\dfrac{\left\langle Ay,y\right\rangle }{\left\langle
y,y\right\rangle }=R\left(  A,y\right)
\]
(by the definition of $R\left(  A,y\right)  $). Moreover, the definition of
$y$ yields $\left\vert \left\vert y\right\vert \right\vert =\left\vert
\left\vert \dfrac{x}{\left\vert \left\vert x\right\vert \right\vert
}\right\vert \right\vert =\dfrac{\left\vert \left\vert x\right\vert
\right\vert }{\left\vert \left\vert x\right\vert \right\vert }=1$. Now, the
definition of $R\left(  A,y\right)  $ yields%
\[
R\left(  A,y\right)  =\dfrac{y^{\ast}Ay}{\left\vert \left\vert y\right\vert
\right\vert ^{2}}=y^{\ast}Ay\ \ \ \ \ \ \ \ \ \ \left(  \text{since
}\left\vert \left\vert y\right\vert \right\vert =1\right)  .
\]
The proof of Proposition \ref{prop.herm.rayleigh-y} is thus complete.
\end{proof}

\subsubsection{The Courant--Fisher theorem: statement}

Let us explore what Rayleigh quotients can tell us about the eigenvalues of a
Hermitian matrix.

Let $A\in\mathbb{C}^{n\times n}$ be a Hermitian matrix with $n>0$. By the
spectral theorem, we have $A=UDU^{\ast}$ for some unitary $U\in\mathbb{C}%
^{n\times n}$ and some real diagonal matrix $D\in\mathbb{R}^{n\times n}$.
Consider these $U$ and $D$. We have $D=\operatorname*{diag}\left(  \lambda
_{1},\lambda_{2},\ldots,\lambda_{n}\right)  $, where $\lambda_{1},\lambda
_{2},\ldots,\lambda_{n}$ are the eigenvalues of $A$. We WLOG assume that
\[
\lambda_{1}\leq\lambda_{2}\leq\cdots\leq\lambda_{n}%
\]
(indeed, we can always achieve this by permuting rows/columns of $D$ and
integrating the permutation matrices into $U$\ \ \ \ \footnote{Alternatively,
we can achieve $\lambda_{1}\leq\lambda_{2}\leq\cdots\leq\lambda_{n}$ right
away by applying Theorem \ref{thm.jnf.schurtri-diag}.}). We set%
\[
\lambda_{\min}\left(  A\right)  :=\lambda_{1}\ \ \ \ \ \ \ \ \ \ \text{and}%
\ \ \ \ \ \ \ \ \ \ \lambda_{\max}\left(  A\right)  :=\lambda_{n}.
\]


Let us now pick some vector $x\in\mathbb{C}^{n}$ of length $1$ (that is,
$\left\vert \left\vert x\right\vert \right\vert =1$). Set $z=U^{\ast}x$. Then,
$z^{\ast}=\left(  U^{\ast}x\right)  ^{\ast}=x^{\ast}\left(  U^{\ast}\right)
^{\ast}=x^{\ast}U$. Also, the matrix $U^{\ast}$ is an isometry (since $U$ is
unitary), and thus we have $\left\vert \left\vert U^{\ast}x\right\vert
\right\vert =\left\vert \left\vert x\right\vert \right\vert =1$. In other
words, $\left\vert \left\vert z\right\vert \right\vert =1$ (since $z=U^{\ast
}x$). In other words, $\sum\limits_{k=1}^{n}\left\vert z_{k}\right\vert
^{2}=1$ (since $\left\vert \left\vert z\right\vert \right\vert =\sqrt
{\sum\limits_{k=1}^{n}\left\vert z_{k}\right\vert ^{2}}$). Furthermore,
writing $z$ as $\left(
\begin{array}
[c]{c}%
z_{1}\\
z_{2}\\
\vdots\\
z_{n}%
\end{array}
\right)  $, we have%
\begin{align*}
x^{\ast}\underbrace{A}_{=UDU^{\ast}}x  &  =\underbrace{x^{\ast}U}_{=z^{\ast}%
}D\underbrace{U^{\ast}x}_{=z}=z^{\ast}Dz=\sum_{k=1}^{n}\lambda_{k}%
\overline{z_{k}}z_{k}=\sum_{k=1}^{n}\underbrace{\lambda_{k}}_{\leq\lambda_{n}%
}\left\vert z_{k}\right\vert ^{2}\\
&  \leq\sum_{k=1}^{n}\lambda_{n}\left\vert z_{k}\right\vert ^{2}=\lambda
_{n}\underbrace{\sum_{k=1}^{n}\left\vert z_{k}\right\vert ^{2}}_{=1}%
=\lambda_{n}.
\end{align*}


Thus, we have shown that each vector $x\in\mathbb{C}^{n}$ of length $1$
satisfies $x^{\ast}Ax\leq\lambda_{n}$. This inequality becomes an equality at
least for one vector $x$ of length $1$: namely, for the vector $x=Ue_{n}$
(because for this vector, we have $z=\underbrace{U^{\ast}U}_{=I_{n}}%
e_{n}=e_{n}$, so that $z_{k}=0$ for all $k<n$, and therefore the inequality
$\sum_{k=1}^{n}\lambda_{k}\left\vert z_{k}\right\vert ^{2}\leq\sum_{k=1}%
^{n}\lambda_{n}\left\vert z_{k}\right\vert ^{2}$ becomes an
equality\footnote{and because the vector $Ue_{n}$ does have length $1$ (since
$U$ is an isometry, so that $\left\vert \left\vert Ue_{n}\right\vert
\right\vert =\left\vert \left\vert e_{n}\right\vert \right\vert =1$)}). Thus,%
\begin{align*}
\lambda_{n}  &  =\max\left\{  x^{\ast}Ax\ \mid\ x\in\mathbb{C}^{n}\text{ is a
vector of length }1\right\} \\
&  =\max\left\{  \dfrac{x^{\ast}Ax}{x^{\ast}x}\ \mid\ x\in\mathbb{C}^{n}\text{
is nonzero}\right\} \\
&  =\max\left\{  R\left(  A,x\right)  \ \mid\ x\in\mathbb{C}^{n}\text{ is
nonzero}\right\}  .
\end{align*}
Since $\lambda_{n}=\lambda_{\max}\left(  A\right)  $, we thus have proved the
following fact:

\begin{proposition}
\label{prop.herm.cour-fish-max}Let $A\in\mathbb{C}^{n\times n}$ be a Hermitian
matrix with $n>0$. Then, the largest eigenvalue of $A$ is%
\begin{align*}
\lambda_{\max}\left(  A\right)   &  =\max\left\{  x^{\ast}Ax\ \mid
\ x\in\mathbb{C}^{n}\text{ is a vector of length }1\right\} \\
&  =\max\left\{  R\left(  A,x\right)  \ \mid\ x\in\mathbb{C}^{n}\text{ is
nonzero}\right\}  .
\end{align*}

\end{proposition}

Similarly, we can prove the following:

\begin{proposition}
\label{prop.herm.cour-fish-min}Let $A\in\mathbb{C}^{n\times n}$ be a Hermitian
matrix with $n>0$. Then, the smallest eigenvalue of $A$ is%
\begin{align*}
\lambda_{\min}\left(  A\right)   &  =\min\left\{  x^{\ast}Ax\ \mid
\ x\in\mathbb{C}^{n}\text{ is a vector of length }1\right\} \\
&  =\min\left\{  R\left(  A,x\right)  \ \mid\ x\in\mathbb{C}^{n}\text{ is
nonzero}\right\}  .
\end{align*}

\end{proposition}

What about the other eigenvalues? Can we characterize $\lambda_{2}$ (for
example) in terms of Rayleigh quotients? Yes, but the characterization is more complicated:

\begin{theorem}
[Courant--Fisher theorem]\label{thm.herm.cour-fish}Let $A\in\mathbb{C}%
^{n\times n}$ be a Hermitian matrix. Let $\lambda_{1},\lambda_{2}%
,\ldots,\lambda_{n}$ be the eigenvalues of $A$, with $\lambda_{1}\leq
\lambda_{2}\leq\cdots\leq\lambda_{n}$. Then, for each $k\in\left[  n\right]
$, we have%
\begin{equation}
\lambda_{k}=\min\limits_{\substack{S\subseteq\mathbb{C}^{n}\text{ is a
subspace;}\\\dim S=k}}\ \ \max\limits_{\substack{x\in S;\\x\neq0}}\ \ R\left(
A,x\right)  \label{thm.herm.cour-fish.minmax}%
\end{equation}
and%
\begin{equation}
\lambda_{k}=\max\limits_{\substack{S\subseteq\mathbb{C}^{n}\text{ is a
subspace;}\\\dim S=n-k+1}}\ \ \min\limits_{\substack{x\in S;\\x\neq
0}}\ \ R\left(  A,x\right)  . \label{thm.herm.cour-fish.maxmin}%
\end{equation}


(The notation \textquotedblleft$\min\limits_{x\in\Omega}f\left(  x\right)
$\textquotedblright\ we are using here is a synonym for $\min\left\{  f\left(
x\right)  \ \mid\ x\in\Omega\right\}  $ whenever $\Omega$ is a set and $f$ is
a function defined on this set. The same applies to maxima. For instance,
$\max\limits_{\substack{x\in S;\\x\neq0}}R\left(  A,x\right)  $ means
$\max\left\{  R\left(  A,x\right)  \ \mid\ x\in S\text{ and }x\neq0\right\}  $.)
\end{theorem}

To prove this theorem, we will use some elementary facts about subspaces of
finite-dimensional vector spaces. We begin by recalling a classical definition:

\subsubsection{The Courant--Fisher theorem: lemmas}

\begin{definition}
\label{def.subspaces.sum}Let $S_{1}$ and $S_{2}$ be two subspaces of a vector
space $V$. Then,
\[
S_{1}+S_{2}:=\left\{  s_{1}+s_{2}\ \mid\ s_{1}\in S_{1}\text{ and }s_{2}\in
S_{2}\right\}  .
\]
This is again a subspace of $V$. (This is the smallest subspace of $V$ that
contains both $S_{1}$ and $S_{2}$ as subspaces.)
\end{definition}

\begin{proposition}
\label{prop.subspaces.dimS1+S2}Let $\mathbb{F}$ be a field. Let $V$ be a
finite-dimensional $\mathbb{F}$-vector space. Let $S_{1}$ and $S_{2}$ be two
subspaces of $V$. Then,%
\[
\dim\left(  S_{1}\cap S_{2}\right)  +\dim\left(  S_{1}+S_{2}\right)  =\dim
S_{1}+\dim S_{2}.
\]

\end{proposition}

\begin{proof}
Pick any basis $\left(  x_{1},x_{2},\ldots,x_{k}\right)  $ of the vector space
$S_{1}\cap S_{2}$.

Then, $\left(  x_{1},x_{2},\ldots,x_{k}\right)  $ is a linearly independent
list of vectors in $S_{1}$. Thus, we can extend it to a basis of $S_{1}$ by
inserting some new vectors $y_{1},y_{2},\ldots,y_{p}$. Hence,%
\[
\left(  x_{1},x_{2},\ldots,x_{k},y_{1},y_{2},\ldots,y_{p}\right)  \text{ is a
basis of }S_{1}\text{.}%
\]


On the other hand, $\left(  x_{1},x_{2},\ldots,x_{k}\right)  $ is a linearly
independent list of vectors in $S_{2}$. Thus, we can extend it to a basis of
$S_{2}$ by inserting some new vectors $z_{1},z_{2},\ldots,z_{q}$. Hence,%
\[
\left(  x_{1},x_{2},\ldots,x_{k},z_{1},z_{2},\ldots,z_{q}\right)  \text{ is a
basis of }S_{2}\text{.}%
\]


The above three bases yield $\dim\left(  S_{1}\cap S_{2}\right)  =k$ and $\dim
S_{1}=k+p$ and $\dim S_{2}=k+q$.

Now, we claim that
\[
\mathbf{w}:=\left(  x_{1},x_{2},\ldots,x_{k},y_{1},y_{2},\ldots,y_{p}%
,z_{1},z_{2},\ldots,z_{q}\right)  \text{ is a basis of }S_{1}+S_{2}.
\]
Once this is proved, we will conclude that $\dim\left(  S_{1}+S_{2}\right)
=k+p+q$, and then Proposition \ref{prop.subspaces.dimS1+S2} will follow by a
simple computation (namely, $k+\left(  k+p+q\right)  =\left(  k+p\right)
+\left(  k+q\right)  $).

So let us prove our claim. To prove that $\mathbf{w}$ is a basis of
$S_{1}+S_{2}$, we need to check the following two statements:

\begin{enumerate}
\item The list $\mathbf{w}$ is linearly independent.

\item The list $\mathbf{w}$ spans $S_{1}+S_{2}$.
\end{enumerate}

Proving statement 2 is easy: Any element of $S_{1}+S_{2}$ is an element of
$S_{1}$ plus an element of $S_{2}$, and thus can be written as
\begin{align*}
&  \left(  \text{a linear combination of }x_{1},x_{2},\ldots,x_{k},y_{1}%
,y_{2},\ldots,y_{p}\right) \\
&  \ \ \ \ \ \ \ \ \ \ +\left(  \text{a linear combination of }x_{1}%
,x_{2},\ldots,x_{k},z_{1},z_{2},\ldots,z_{q}\right) \\
&  =\lambda_{1}x_{1}+\lambda_{2}x_{2}+\cdots+\lambda_{k}x_{k}+\alpha_{1}%
y_{1}+\alpha_{2}y_{2}+\cdots+\alpha_{p}y_{p}\\
&  \ \ \ \ \ \ \ \ \ \ +\mu_{1}x_{1}+\mu_{2}x_{2}+\cdots+\mu_{k}x_{k}%
+\beta_{1}z_{1}+\beta_{2}z_{2}+\cdots+\beta_{q}z_{q}\\
&  \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \left(  \text{for some scalars
}\lambda_{i},\alpha_{j},\mu_{i},\beta_{v}\in\mathbb{F}\right) \\
&  =\left(  \lambda_{1}+\mu_{1}\right)  x_{1}+\left(  \lambda_{2}+\mu
_{2}\right)  x_{2}+\cdots+\left(  \lambda_{k}+\mu_{k}\right)  x_{k}\\
&  \ \ \ \ \ \ \ \ \ \ +\alpha_{1}y_{1}+\alpha_{2}y_{2}+\cdots+\alpha_{p}%
y_{p}+\beta_{1}z_{1}+\beta_{2}z_{2}+\cdots+\beta_{q}z_{q}\\
&  =\left(  \text{a linear combination of }x_{1},x_{2},\ldots,x_{k}%
,y_{1},y_{2},\ldots,y_{p},z_{1},z_{2},\ldots,z_{q}\right)  ;
\end{align*}
thus it belongs to the span of $\mathbf{w}$.

Let us now prove statement 1. We need to show that $\mathbf{w}$ is linearly
independent. So let us assume that%
\[
\lambda_{1}x_{1}+\lambda_{2}x_{2}+\cdots+\lambda_{k}x_{k}+\alpha_{1}%
y_{1}+\alpha_{2}y_{2}+\cdots+\alpha_{p}y_{p}+\beta_{1}z_{1}+\beta_{2}%
z_{2}+\cdots+\beta_{q}z_{q}=0
\]
for some coefficients $\lambda_{m},\alpha_{i},\beta_{j}$ that are not all
equal to $0$. We want a contradiction.

Let
\[
v:=\lambda_{1}x_{1}+\lambda_{2}x_{2}+\cdots+\lambda_{k}x_{k}+\alpha_{1}%
y_{1}+\alpha_{2}y_{2}+\cdots+\alpha_{p}y_{p}.
\]
Then,%
\begin{align*}
v  &  =-\left(  \beta_{1}z_{1}+\beta_{2}z_{2}+\cdots+\beta_{q}z_{q}\right)
\ \ \ \ \ \ \ \ \ \ \left(  \text{by the above equation}\right) \\
&  \in S_{2}\ \ \ \ \ \ \ \ \ \ \left(  \text{since the }z_{j}\text{'s lie in
}S_{2}\right)  .
\end{align*}
On the other hand, the definition of $v$ yields $v\in S_{1}$ (since the
$x_{m}$'s and the $y_{i}$'s lie in $S_{1}$). Thus, $v$ lies in both $S_{1}$
and $S_{2}$. This entails that $v\in S_{1}\cap S_{2}$. Since $\left(
x_{1},x_{2},\ldots,x_{k}\right)  $ is a basis of $S_{1}\cap S_{2}$, this
entails that%
\[
v=\xi_{1}x_{1}+\xi_{2}x_{2}+\cdots+\xi_{k}x_{k}\ \ \ \ \ \ \ \ \ \ \text{for
some }\xi_{1},\xi_{2},\ldots,\xi_{k}\in\mathbb{F}.
\]
Comparing this with%
\[
v=-\left(  \beta_{1}z_{1}+\beta_{2}z_{2}+\cdots+\beta_{q}z_{q}\right)  ,
\]
we obtain%
\[
\xi_{1}x_{1}+\xi_{2}x_{2}+\cdots+\xi_{k}x_{k}=-\left(  \beta_{1}z_{1}%
+\beta_{2}z_{2}+\cdots+\beta_{q}z_{q}\right)  .
\]
In other words,%
\[
\xi_{1}x_{1}+\xi_{2}x_{2}+\cdots+\xi_{k}x_{k}+\beta_{1}z_{1}+\beta_{2}%
z_{2}+\cdots+\beta_{q}z_{q}=0.
\]
Since the list $\left(  x_{1},x_{2},\ldots,x_{k},z_{1},z_{2},\ldots
,z_{q}\right)  $ is linearly independent (being a basis of $S_{2}$), this
entails that all coefficients $\xi_{m}$ and $\beta_{j}$ are $0$. Thus, $v=0$
(since $v=\xi_{1}x_{1}+\xi_{2}x_{2}+\cdots+\xi_{k}x_{k}$). In view of%
\[
v=\lambda_{1}x_{1}+\lambda_{2}x_{2}+\cdots+\lambda_{k}x_{k}+\alpha_{1}%
y_{1}+\alpha_{2}y_{2}+\cdots+\alpha_{p}y_{p},
\]
this rewrites as%
\[
\lambda_{1}x_{1}+\lambda_{2}x_{2}+\cdots+\lambda_{k}x_{k}+\alpha_{1}%
y_{1}+\alpha_{2}y_{2}+\cdots+\alpha_{p}y_{p}=0.
\]
Since the list $\left(  x_{1},x_{2},\ldots,x_{k},y_{1},y_{2},\ldots
,y_{p}\right)  $ is linearly independent (being a basis of $S_{1}$), this
entails that all coefficients $\lambda_{m}$ and $\alpha_{i}$ are $0$.

Now we know that all $\lambda_{m}$ and $\alpha_{i}$ and $\beta_{j}$ are $0$,
which contradicts our assumption that some of them are nonzero. This completes
the proof of Statement 1.

As we said, we now conclude that the list $\mathbf{w}$ is a basis of the
vector space $S_{1}+S_{2}$. Since this list $\mathbf{w}$ contains $k+p+q$
vectors, we thus obtain $\dim\left(  S_{1}+S_{2}\right)  =k+p+q$, so that%
\begin{align*}
\underbrace{\dim\left(  S_{1}\cap S_{2}\right)  }_{=k}+\underbrace{\dim\left(
S_{1}+S_{2}\right)  }_{=k+p+q}  &  =k+\left(  k+p+q\right) \\
&  =\underbrace{\left(  k+p\right)  }_{=\dim S_{1}}+\underbrace{\left(
k+q\right)  }_{=\dim S_{2}}\\
&  =\dim S_{1}+\dim S_{2}.
\end{align*}
This proves Proposition \ref{prop.subspaces.dimS1+S2}.
\end{proof}

\begin{remark}
A well-known fact in elementary set theory says that if $A_{1}$ and $A_{2}$
are two finite sets, then%
\[
\left\vert A_{1}\cap A_{2}\right\vert +\left\vert A_{1}\cup A_{2}\right\vert
=\left\vert A_{1}\right\vert +\left\vert A_{2}\right\vert .
\]
Proposition \ref{prop.subspaces.dimS1+S2} is an analogue of this fact for
vector spaces (noticing that the sum $S_{1}+S_{2}$ is a vector-space analogue
of the union).

Note, however, that the \textquotedblleft next level\textquotedblright\ of the
above formula has no vector space analogue. We do have%
\begin{align*}
&  \left\vert A_{1}\cup A_{2}\cup A_{3}\right\vert +\left\vert A_{1}\cap
A_{2}\right\vert +\left\vert A_{1}\cap A_{3}\right\vert +\left\vert A_{2}\cap
A_{3}\right\vert \\
&  =\left\vert A_{1}\right\vert +\left\vert A_{2}\right\vert +\left\vert
A_{3}\right\vert +\left\vert A_{1}\cap A_{2}\cap A_{3}\right\vert
\end{align*}
for any three finite sets $A_{1},A_{2},A_{3}$, but no such relation holds for
three subspaces of a vector space.
\end{remark}

\begin{corollary}
\label{cor.subspaces.dim-geq-del}Let $\mathbb{F}$ be a field, and let
$n\in\mathbb{N}$. Let $V$ be an $n$-dimensional $\mathbb{F}$-vector space. Let
$S_{1},S_{2},\ldots,S_{k}$ be subspaces of $V$ (with $k\geq1$). Let
\[
\delta:=\dim\left(  S_{1}\right)  +\dim\left(  S_{2}\right)  +\cdots
+\dim\left(  S_{k}\right)  -\left(  k-1\right)  n.
\]


\textbf{(a)} Then, $\dim\left(  S_{1}\cap S_{2}\cap\cdots\cap S_{k}\right)
\geq\delta$. \medskip

\textbf{(b)} If $\mathbb{F}=\mathbb{C}$ and $V=\mathbb{C}^{n}$ and $\delta>0$,
then there exists a vector $x\in S_{1}\cap S_{2}\cap\cdots\cap S_{k}$ with
$\left\vert \left\vert x\right\vert \right\vert =1$.
\end{corollary}

\begin{proof}
\textbf{(a)} We induct on $k$. The \textit{base case} ($k=1$) is obvious
(since $\dim\left(  S_{1}\cap S_{2}\cap\cdots\cap S_{k}\right)  =\dim\left(
S_{1}\right)  =\delta$ in this case).

\textit{Induction step:} Suppose the statement holds for some $k$. Now
consider $k+1$ subspaces $S_{1},S_{2},\ldots,S_{k+1}$ of $V$, and let%
\[
\delta_{k+1}:=\dim\left(  S_{1}\right)  +\dim\left(  S_{2}\right)
+\cdots+\dim\left(  S_{k+1}\right)  -kn.
\]
We want to prove that $\dim\left(  S_{1}\cap S_{2}\cap\cdots\cap
S_{k+1}\right)  \geq\delta_{k+1}$.

Then,%
\begin{align*}
&  \dim\left(  S_{1}\cap S_{2}\cap\cdots\cap S_{k+1}\right) \\
&  =\dim\left(  S_{1}\cap S_{2}\cap\cdots\cap S_{k-1}\cap\left(  S_{k}\cap
S_{k+1}\right)  \right)  .
\end{align*}
Now, set
\[
\delta_{k}:=\dim\left(  S_{1}\right)  +\dim\left(  S_{2}\right)  +\cdots
+\dim\left(  S_{k-1}\right)  +\dim\left(  S_{k}\cap S_{k+1}\right)  -\left(
k-1\right)  n.
\]
By the induction hypothesis, we can apply Corollary
\ref{cor.subspaces.dim-geq-del} \textbf{(a)} to $S_{k}\cap S_{k+1}$ and
$\delta_{k}$ instead of $S_{k}$ and $\delta$. Thus, we obtain%
\[
\dim\left(  S_{1}\cap S_{2}\cap\cdots\cap S_{k-1}\cap\left(  S_{k}\cap
S_{k+1}\right)  \right)  \geq\delta_{k}.
\]
It remains to show that $\delta_{k}\geq\delta_{k+1}$. Equivalently, we need to
show that%
\[
\dim\left(  S_{k}\cap S_{k+1}\right)  -\left(  k-1\right)  n\geq\dim\left(
S_{k}\right)  +\dim\left(  S_{k+1}\right)  -kn.
\]
In other words, we need to show that%
\[
\dim\left(  S_{k}\cap S_{k+1}\right)  +n\geq\dim\left(  S_{k}\right)
+\dim\left(  S_{k+1}\right)  .
\]
However, $S_{k}+S_{k+1}$ is a subspace of $V$, so its dimension is
$\dim\left(  S_{k}+S_{k+1}\right)  \leq\dim V=n$. Therefore,
\[
\dim\left(  S_{k}\cap S_{k+1}\right)  +\underbrace{n}_{\geq\dim\left(
S_{k}+S_{k+1}\right)  }\geq\dim\left(  S_{k}\cap S_{k+1}\right)  +\dim\left(
S_{k}+S_{k+1}\right)  =\dim\left(  S_{k}\right)  +\dim\left(  S_{k+1}\right)
\]
(by Proposition \ref{prop.subspaces.dimS1+S2}). So the induction step is
complete, and Corollary \ref{cor.subspaces.dim-geq-del} \textbf{(a)} is
proved. \medskip

\textbf{(b)} Assume that $\mathbb{F}=\mathbb{C}$ and $V=\mathbb{C}^{n}$ and
$\delta>0$. Then, part \textbf{(a)} yields%
\[
\dim\left(  S_{1}\cap S_{2}\cap\cdots\cap S_{k}\right)  \geq\delta>0.
\]
Thus, the subspace $S_{1}\cap S_{2}\cap\cdots\cap S_{k}$ is not just $\left\{
0\right\}  $. Therefore, it contains a nonzero vector. Scaling this vector by
the reciprocal of its length, we obtain a vector of length $1$. This proves
Corollary \ref{cor.subspaces.dim-geq-del} \textbf{(b)}.
\end{proof}

\begin{lemma}
\label{lem.herm.cour-fish-lem}Let $A\in\mathbb{C}^{n\times n}$ be a Hermitian
matrix. Let $\left(  v_{1},v_{2},\ldots,v_{k}\right)  $ be an orthonormal
tuple of eigenvectors of $A$, and let $\mu_{1},\mu_{2},\ldots,\mu_{k}$ be the
corresponding eigenvalues (so that $A_{i}v_{i}=\mu_{i}v_{i}$ for each
$i\in\left[  k\right]  $). Assume that $\mu_{1}\leq\mu_{2}\leq\cdots\leq
\mu_{k}$. Then, each $v\in\operatorname*{span}\left(  v_{1},v_{2},\ldots
,v_{k}\right)  $ satisfies%
\begin{equation}
\left\langle Av,v\right\rangle \geq\mu_{1}\left\langle v,v\right\rangle
\label{eq.lem.herm.cour-fish-lem.geq}%
\end{equation}
and%
\begin{equation}
\left\langle Av,v\right\rangle \leq\mu_{k}\left\langle v,v\right\rangle .
\label{eq.lem.herm.cour-fish-lem.leq}%
\end{equation}

\end{lemma}

\begin{proof}
Let $v\in\operatorname*{span}\left(  v_{1},v_{2},\ldots,v_{k}\right)  $. Thus,
we can write $v$ in the form%
\begin{equation}
v=\alpha_{1}v_{1}+\alpha_{2}v_{2}+\cdots+\alpha_{k}v_{k}
\label{pf.lem.herm.cour-fish-lem.v=}%
\end{equation}
for some $\alpha_{1},\alpha_{2},\ldots,\alpha_{k}\in\mathbb{C}$. Consider
these $\alpha_{1},\alpha_{2},\ldots,\alpha_{k}$. From
(\ref{pf.lem.herm.cour-fish-lem.v=}), we obtain%
\[
v=\alpha_{1}v_{1}+\alpha_{2}v_{2}+\cdots+\alpha_{k}v_{k}=\sum_{i=1}^{k}%
\alpha_{i}v_{i}%
\]
and thus%
\[
Av=A\sum_{i=1}^{k}\alpha_{i}v_{i}=\sum_{i=1}^{k}\alpha_{i}\underbrace{Av_{i}%
}_{=\mu_{i}v_{i}}=\sum_{i=1}^{k}\alpha_{i}\mu_{i}v_{i}.
\]
Combining this with $v=\sum_{i=1}^{k}\alpha_{i}v_{i}=\sum_{j=1}^{k}\alpha
_{j}v_{j}$, we obtain%
\begin{equation}
\left\langle Av,v\right\rangle =\left\langle \sum_{i=1}^{k}\alpha_{i}\mu
_{i}v_{i},\ \sum_{j=1}^{k}\alpha_{j}v_{j}\right\rangle =\sum_{i=1}^{k}%
\ \ \sum_{j=1}^{k}\alpha_{i}\mu_{i}\overline{\alpha_{j}}\left\langle
v_{i},v_{j}\right\rangle . \label{pf.lem.herm.cour-fish-lem.2}%
\end{equation}
However, the inner products $\left\langle v_{i},v_{j}\right\rangle $ in this
sum are $0$ whenever $i\neq j$ (since the tuple $\left(  v_{1},v_{2}%
,\ldots,v_{k}\right)  $ is orthonormal). Thus, we can simplify this sum as
follows:%
\begin{align*}
\sum_{i=1}^{k}\ \ \sum_{j=1}^{k}\alpha_{i}\mu_{i}\overline{\alpha_{j}%
}\left\langle v_{i},v_{j}\right\rangle  &  =\sum_{i=1}^{k}\alpha_{i}\mu
_{i}\overline{\alpha_{i}}\underbrace{\left\langle v_{i},v_{i}\right\rangle
}_{\substack{=\left\vert \left\vert v_{i}\right\vert \right\vert
^{2}=1\\\text{(since the tuple }\left(  v_{1},v_{2},\ldots,v_{k}\right)
\\\text{is orthonormal)}}}=\sum_{i=1}^{k}\underbrace{\alpha_{i}\mu
_{i}\overline{\alpha_{i}}}_{=\alpha_{i}\overline{\alpha_{i}}\cdot\mu_{i}}\\
&  =\sum_{i=1}^{k}\underbrace{\alpha_{i}\overline{\alpha_{i}}}_{=\left\vert
\alpha_{i}\right\vert ^{2}}\cdot\mu_{i}=\sum_{i=1}^{k}\left\vert \alpha
_{i}\right\vert ^{2}\mu_{i}.
\end{align*}
Thus, (\ref{pf.lem.herm.cour-fish-lem.2}) rewrites as%
\begin{equation}
\left\langle Av,v\right\rangle =\sum_{i=1}^{k}\left\vert \alpha_{i}\right\vert
^{2}\mu_{i}. \label{pf.lem.herm.cour-fish-lem.3}%
\end{equation}


On the other hand, from $v=\sum_{i=1}^{k}\alpha_{i}v_{i}$ and $v=\sum
_{j=1}^{k}\alpha_{j}v_{j}$, we obtain%
\begin{align}
\left\langle v,v\right\rangle  &  =\left\langle \sum_{i=1}^{k}\alpha_{i}%
v_{i},\ \sum_{j=1}^{k}\alpha_{j}v_{j}\right\rangle =\sum_{i=1}^{k}%
\ \ \sum_{j=1}^{k}\alpha_{i}\overline{\alpha_{j}}\left\langle v_{i}%
,v_{j}\right\rangle \nonumber\\
&  =\sum_{i=1}^{k}\underbrace{\alpha_{i}\overline{\alpha_{i}}}_{=\left\vert
\alpha_{i}\right\vert ^{2}}\underbrace{\left\langle v_{i},v_{i}\right\rangle
}_{\substack{=\left\vert \left\vert v_{i}\right\vert \right\vert
^{2}=1\\\text{(since the tuple }\left(  v_{1},v_{2},\ldots,v_{k}\right)
\\\text{is orthonormal)}}}\nonumber\\
&  \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \left(  \text{since the inner
products }\left\langle v_{i},v_{j}\right\rangle \text{ are }0\text{ whenever
}i\neq j\right) \nonumber\\
&  =\sum_{i=1}^{k}\left\vert \alpha_{i}\right\vert ^{2}.
\label{pf.lem.herm.cour-fish-lem.4}%
\end{align}


Now, (\ref{pf.lem.herm.cour-fish-lem.3}) becomes%
\begin{align*}
\left\langle Av,v\right\rangle  &  =\sum_{i=1}^{k}\left\vert \alpha
_{i}\right\vert ^{2}\underbrace{\mu_{i}}_{\substack{\geq\mu_{1}\\\text{(since
}\mu_{1}\leq\mu_{2}\leq\cdots\leq\mu_{k}\text{)}}}\geq\sum_{i=1}^{k}\left\vert
\alpha_{i}\right\vert ^{2}\mu_{1}=\mu_{1}\underbrace{\sum_{i=1}^{k}\left\vert
\alpha_{i}\right\vert ^{2}}_{\substack{=\left\langle v,v\right\rangle
\\\text{(by (\ref{pf.lem.herm.cour-fish-lem.4}))}}}\\
&  =\mu_{1}\left\langle v,v\right\rangle .
\end{align*}
This proves (\ref{eq.lem.herm.cour-fish-lem.geq}). Likewise,
(\ref{pf.lem.herm.cour-fish-lem.3}) becomes%
\begin{align*}
\left\langle Av,v\right\rangle  &  =\sum_{i=1}^{k}\left\vert \alpha
_{i}\right\vert ^{2}\underbrace{\mu_{i}}_{\substack{\leq\mu_{k}\\\text{(since
}\mu_{1}\leq\mu_{2}\leq\cdots\leq\mu_{k}\text{)}}}\leq\sum_{i=1}^{k}\left\vert
\alpha_{i}\right\vert ^{2}\mu_{k}=\mu_{k}\underbrace{\sum_{i=1}^{k}\left\vert
\alpha_{i}\right\vert ^{2}}_{\substack{=\left\langle v,v\right\rangle
\\\text{(by (\ref{pf.lem.herm.cour-fish-lem.4}))}}}\\
&  =\mu_{k}\left\langle v,v\right\rangle .
\end{align*}
This proves (\ref{eq.lem.herm.cour-fish-lem.leq}).
\end{proof}

\subsubsection{The Courant--Fisher theorem: proof}

Now, we are ready for the proof of the Courant--Fisher theorem:

\begin{proof}
[Proof of Theorem \ref{thm.herm.cour-fish}.]The spectral theorem (Theorem
\ref{thm.schurtri.normal.spectral} \textbf{(a)}) says that $A=UDU^{\ast}$ for
some unitary $U$ and some diagonal matrix $D$. Consider these $U$ and $D$.
Proposition \ref{prop.schurtri.normal.hermitian-spec} shows that the diagonal
entries of $D$ are real.

The columns of $U$ form an orthonormal basis of $\mathbb{C}^{n}$ (since $U$ is
unitary); let $\left(  u_{1},u_{2},\ldots,u_{n}\right)  $ be this basis. Then,
$u_{1},u_{2},\ldots,u_{n}$ are eigenvectors of $A$ (by Theorem
\ref{thm.schurtri.normal.spectral} \textbf{(b)}). We WLOG assume that the
corresponding eigenvalues are $\lambda_{1},\lambda_{2},\ldots,\lambda_{n}$
(otherwise, permute the diagonal entries of $D$ and correspondingly permute
the columns of $U$). Thus, $D=\operatorname*{diag}\left(  \lambda_{1}%
,\lambda_{2},\ldots,\lambda_{n}\right)  $ (by the proof of Theorem
\ref{thm.schurtri.normal.spectral} \textbf{(b)}).

Let $k\in\left[  n\right]  $.

Let $S$ be a vector subspace of $\mathbb{C}^{n}$ with $\dim S=k$. Let
$S^{\prime}=\operatorname*{span}\left(  u_{k},u_{k+1},\ldots,u_{n}\right)  $.
Then, by Proposition \ref{prop.subspaces.dimS1+S2}, we have%
\[
\dim\left(  S\cap S^{\prime}\right)  +\dim\left(  S+S^{\prime}\right)
=\underbrace{\dim S}_{=k}+\underbrace{\dim S^{\prime}}_{=n-k+1}=n+1>n\geq
\dim\left(  S+S^{\prime}\right)
\]
(since $S+S^{\prime}$ is a subspace of $\mathbb{C}^{n}$). Subtracting
$\dim\left(  S+S^{\prime}\right)  $ from this inequality, we obtain
$\dim\left(  S\cap S^{\prime}\right)  >0$. Thus, $S\cap S^{\prime}$ contains a
nonzero vector. Thus, $\sup\limits_{\substack{x\in S\cap S^{\prime};\\x\neq
0}}\ \ R\left(  A,x\right)  $ and $\inf\limits_{\substack{x\in S\cap
S^{\prime};\\x\neq0}}\ \ R\left(  A,x\right)  $ are well-defined.

Now,%
\begin{align}
\sup\limits_{\substack{x\in S;\\x\neq0}}\ \ R\left(  A,x\right)   &  \geq
\sup\limits_{\substack{x\in S\cap S^{\prime};\\x\neq0}}\ \ R\left(
A,x\right)  \geq\inf\limits_{\substack{x\in S\cap S^{\prime};\\x\neq
0}}\ \ R\left(  A,x\right) \nonumber\\
&  \geq\inf\limits_{\substack{x\in S^{\prime};\\x\neq0}}\ \ R\left(
A,x\right)  . \label{pf.thm.herm.cour-fish.4}%
\end{align}


However, I claim that $\inf\limits_{\substack{x\in S^{\prime};\\x\neq
0}}\ \ R\left(  A,x\right)  \geq\lambda_{k}$ (actually, this is an equality,
but we will not need this). Indeed, $\left(  u_{k},u_{k+1},\ldots
,u_{n}\right)  $ is an orthonormal tuple of eigenvalues of $A$ (since $\left(
u_{1},u_{2},\ldots,u_{n}\right)  $ is an orthonormal tuple of eigenvalues of
$A$) with corresponding eigenvalues $\lambda_{k},\lambda_{k+1},\ldots
,\lambda_{n}$ satisfying $\lambda_{k}\leq\lambda_{k+1}\leq\cdots\leq
\lambda_{n}$ (since $\lambda_{1}\leq\lambda_{2}\leq\cdots\leq\lambda_{n}$).
Thus, any $x\in\operatorname*{span}\left(  u_{k},u_{k+1},\ldots,u_{n}\right)
$ satisfies%
\[
\left\langle Ax,x\right\rangle \geq\lambda_{k}\left\langle x,x\right\rangle
\]
(by (\ref{eq.lem.herm.cour-fish-lem.geq}), applied to $\left(  u_{k}%
,u_{k+1},\ldots,u_{n}\right)  $ and $\left(  \lambda_{k},\lambda_{k+1}%
,\ldots,\lambda_{n}\right)  $ and $x$ instead of $\left(  v_{1},v_{2}%
,\ldots,v_{k}\right)  $ and $\left(  \mu_{1},\mu_{2},\ldots,\mu_{k}\right)  $
and $v$) and thus $R\left(  A,x\right)  =\dfrac{\left\langle Ax,x\right\rangle
}{\left\langle x,x\right\rangle }\geq\lambda_{k}$. In other words, any $x\in
S^{\prime}$ satisfies $R\left(  A,x\right)  \geq\lambda_{k}$ (since
$S^{\prime}=\operatorname*{span}\left(  u_{k},u_{k+1},\ldots,u_{n}\right)  $).
Hence, we have%
\[
\inf\limits_{\substack{x\in S^{\prime};\\x\neq0}}\ \ R\left(  A,x\right)
\geq\lambda_{k}.
\]
Combining this with (\ref{pf.thm.herm.cour-fish.4}), we obtain%
\[
\sup\limits_{\substack{x\in S;\\x\neq0}}\ \ R\left(  A,x\right)  \geq
\lambda_{k}.
\]
Furthermore, this supremum is a maximum, because
\begin{align*}
\sup\limits_{\substack{x\in S;\\x\neq0}}\ \ R\left(  A,x\right)   &
=\sup\limits_{\substack{y\in S;\\\left\vert \left\vert y\right\vert
\right\vert =1}}\ \ R\left(  A,y\right)  \ \ \ \ \ \ \ \ \ \ \left(
\text{since }R\left(  A,x\right)  =R\left(  A,y\right)  \text{ where }%
y=\dfrac{x}{\left\vert \left\vert x\right\vert \right\vert }\right) \\
&  =\max\limits_{\substack{y\in S;\\\left\vert \left\vert y\right\vert
\right\vert =1}}\ \ R\left(  A,y\right)  \ \ \ \ \ \ \ \ \ \ \left(
\begin{array}
[c]{c}%
\text{since the set of all }y\in S\text{ satisfying }\left\vert \left\vert
y\right\vert \right\vert =1\\
\text{is compact, and since a continuous function}\\
\text{on a compact set always has a maximum}%
\end{array}
\right) \\
&  =\max\limits_{\substack{x\in S;\\x\neq0}}\ \ R\left(  A,x\right)  .
\end{align*}
So we conclude that%
\[
\max\limits_{\substack{x\in S;\\x\neq0}}\ \ R\left(  A,x\right)
=\sup\limits_{\substack{x\in S;\\x\neq0}}\ \ R\left(  A,x\right)  \geq
\lambda_{k}.
\]


Forget that we fixed $S$. We thus have shown that if $S$ is any $k$%
-dimensional subspace of $\mathbb{C}^{n}$, then $\max\limits_{\substack{x\in
S;\\x\neq0}}\ \ R\left(  A,x\right)  $ exists and satisfies
\[
\max\limits_{\substack{x\in S;\\x\neq0}}\ \ R\left(  A,x\right)  \geq
\lambda_{k}.
\]
However, by choosing $S$ appropriately, we can achieve equality here; indeed,
we have to choose $S=\operatorname*{span}\left(  u_{1},u_{2},\ldots
,u_{k}\right)  $ for this. (Why? Because each $x\in\operatorname*{span}\left(
u_{1},u_{2},\ldots,u_{k}\right)  $ can easily be seen to satisfy $\left\langle
Ax,x\right\rangle \leq\lambda_{k}\left\langle x,x\right\rangle $ by a similar
argument to the one we used above\footnote{but using
(\ref{eq.lem.herm.cour-fish-lem.leq}) instead of
(\ref{eq.lem.herm.cour-fish-lem.geq})}.)

Thus, we have shown that the value $\max\limits_{\substack{x\in S;\\x\neq
0}}\ \ R\left(  A,x\right)  $ is $\geq\lambda_{k}$ for each $S$, but is
$=\lambda_{k}$ for a certain $S$. Therefore, $\lambda_{k}$ is the smallest
possible value of $\max\limits_{\substack{x\in S;\\x\neq0}}\ \ R\left(
A,x\right)  $. In other words,%
\[
\lambda_{k}=\min\limits_{\substack{S\subseteq\mathbb{C}^{n}\text{ is a
subspace;}\\\dim S=k}}\ \ \max\limits_{\substack{x\in S;\\x\neq0}}\ \ R\left(
A,x\right)  .
\]


Thus, we have proved (\ref{thm.herm.cour-fish.minmax}). It remains to prove
the other part of the theorem -- i.e., the equality
(\ref{thm.herm.cour-fish.maxmin}).

One way to prove this is by arguing similarly to the above proof.
Alternatively, we can simply apply the already proved equality
(\ref{thm.herm.cour-fish.minmax}) to $-A$ instead of $A$, after noticing that
$-A$ is a Hermitian matrix with eigenvalues%
\[
-\lambda_{n}\leq-\lambda_{n-1}\leq\cdots\leq-\lambda_{1}.
\]
Keep in mind that $-\lambda_{k}$ is not the $k$-th smallest eigenvalue of
$-A$, but rather is the $k$-th largest eigenvalue of $-A$, and thus the
$\left(  n-k+1\right)  $-st smallest eigenvalue of $-A$. Thus, we have to
apply the equality (\ref{thm.herm.cour-fish.minmax}) to $-A$ and $n-k+1$
instead of $A$ and $k$. Taking negatives turns minima into maxima and vice
versa (i.e., we have $\min\limits_{x\in\Omega}\left(  -f\left(  x\right)
\right)  =-\max\limits_{x\in\Omega}f\left(  x\right)  $ and $\max
\limits_{x\in\Omega}\left(  -f\left(  x\right)  \right)  =-\min\limits_{x\in
\Omega}f\left(  x\right)  $). Finally, it is helpful to know that $R\left(
-A,x\right)  =-R\left(  A,x\right)  $ for any vector $x\in\mathbb{C}^{n}$.
Armed with these observations, we can easily derive
(\ref{thm.herm.cour-fish.maxmin}) from (\ref{thm.herm.cour-fish.minmax}). The
proof of Theorem \ref{thm.herm.cour-fish} is now complete.
\end{proof}

\subsubsection{The Weyl inequalities}

The Courant--Fisher theorem can be used to connect the eigenvalues of $A+B$
with the eigenvalues of $A$ and $B$.

\begin{theorem}
[Weyl's inequalities]\label{thm.herm.weyl-ineq}Let $A$ and $B$ be two
Hermitian matrices in $\mathbb{C}^{n\times n}$. Let $i\in\left[  n\right]  $
and $j\in\left\{  0,1,\ldots,n-i\right\}  $. \medskip

\textbf{(a)} Then,%
\[
\lambda_{i}\left(  A+B\right)  \leq\lambda_{i+j}\left(  A\right)
+\lambda_{n-j}\left(  B\right)  .
\]
Here, $\lambda_{k}\left(  C\right)  $ means the $k$-th smallest eigenvalue of
a Hermitian matrix $C$.

Moreover, this inequality becomes an equality if and only if there exists a
nonzero vector $x\in\mathbb{C}^{n}$ satisfying
\[
Ax=\lambda_{i+j}\left(  A\right)  x,\ \ \ \ \ \ \ \ \ \ Bx=\lambda
_{n-j}\left(  B\right)  x,\ \ \ \ \ \ \ \ \ \ \left(  A+B\right)
x=\lambda_{i}\left(  A+B\right)  x
\]
(at the same time). \medskip

\textbf{(b)} Furthermore,%
\[
\lambda_{i-k+1}\left(  A\right)  +\lambda_{k}\left(  B\right)  \leq\lambda
_{i}\left(  A+B\right)  \ \ \ \ \ \ \ \ \ \ \text{for any }k\in\left[
i\right]  .
\]

\end{theorem}

Theorem \ref{thm.herm.weyl-ineq} lets us bound the eigenvalues of $A+B$ in
terms of those of $A$ and $B$ under the assumption that $A$ and $B$ are
Hermitian matrices. (In contrast, if $A$ and $B$ are arbitrary -- not
Hermitian -- matrices, then no such bounds are possible for $n>1$.)

\begin{example}
Applying Theorem \ref{thm.herm.weyl-ineq} \textbf{(a)} to $i=n$ and $j=0$, we
obtain%
\[
\lambda_{n}\left(  A+B\right)  \leq\lambda_{n}\left(  A\right)  +\lambda
_{n}\left(  B\right)  .
\]


Applying Theorem \ref{thm.herm.weyl-ineq} \textbf{(b)} to $i=1$ and $k=1$, we
obtain%
\[
\lambda_{1}\left(  A\right)  +\lambda_{1}\left(  B\right)  \leq\lambda
_{1}\left(  A+B\right)  .
\]

\end{example}

\begin{proof}
[Proof of Theorem \ref{thm.herm.weyl-ineq}.]Let $\left(  x_{1},x_{2}%
,\ldots,x_{n}\right)  $, $\left(  y_{1},y_{2},\ldots,y_{n}\right)  $ and
$\left(  z_{1},z_{2},\ldots,z_{n}\right)  $ be three orthonormal bases of
$\mathbb{C}^{n}$ with%
\[
Ax_{k}=\lambda_{k}\left(  A\right)  x_{k},\ \ \ \ \ \ \ \ \ \ By_{k}%
=\lambda_{k}\left(  B\right)  y_{k},\ \ \ \ \ \ \ \ \ \ \left(  A+B\right)
z_{k}=\lambda_{k}\left(  A+B\right)  z_{k}%
\]
for all $k\in\left[  n\right]  $. (As above, we can find such bases by using
the spectral decompositions of $A$, $B$ and $A+B$.)

Let
\begin{align*}
S_{1}  &  =\operatorname*{span}\left(  x_{1},x_{2},\ldots,x_{i+j}\right)  ;\\
S_{2}  &  =\operatorname*{span}\left(  y_{1},y_{2},\ldots,y_{n-j}\right)  ;\\
S_{3}  &  =\operatorname*{span}\left(  z_{i},z_{i+1},\ldots,z_{n}\right)  .
\end{align*}
Then,%
\[
\delta:=\underbrace{\dim\left(  S_{1}\right)  }_{=i+j}+\underbrace{\dim\left(
S_{2}\right)  }_{=n-j}+\underbrace{\dim\left(  S_{3}\right)  }_{=n-i+1}%
-2n=1>0.
\]
Hence, Corollary \ref{cor.subspaces.dim-geq-del} \textbf{(b)} yields that
there is a length-$1$ vector $v$ in $S_{1}\cap S_{2}\cap S_{3}$. This $v$
satisfies%
\begin{align*}
\lambda_{i}\left(  A+B\right)   &  \leq\left\langle \left(  A+B\right)
v,\ v\right\rangle \ \ \ \ \ \ \ \ \ \ \left(  \text{by
(\ref{eq.lem.herm.cour-fish-lem.geq}), since }v\in S_{3}=\operatorname*{span}%
\left(  z_{i},z_{i+1},\ldots,z_{n}\right)  \right) \\
&  =\left\langle Av+Bv,\ v\right\rangle =\underbrace{\left\langle
Av,\ v\right\rangle }_{\substack{\leq\lambda_{i+j}\left(  A\right)
\\\text{(by (\ref{eq.lem.herm.cour-fish-lem.leq}),}\\\text{since }v\in
S_{1}=\operatorname*{span}\left(  x_{1},x_{2},\ldots,x_{i+j}\right)  \text{)}%
}}+\underbrace{\left\langle Bv,\ v\right\rangle }_{\substack{\leq\lambda
_{n-j}\left(  B\right)  \\\text{(by (\ref{eq.lem.herm.cour-fish-lem.leq}%
),}\\\text{since }v\in S_{2}=\operatorname*{span}\left(  y_{1},y_{2}%
,\ldots,y_{n-j}\right)  \text{)}}}\\
&  \leq\lambda_{i+j}\left(  A\right)  +\lambda_{n-j}\left(  B\right)  .
\end{align*}
This proves the inequality part of Theorem \ref{thm.herm.weyl-ineq}
\textbf{(a)}. The equality case is not hard to analyze following the above
argument; we leave this to the reader.

The proof of Theorem \ref{thm.herm.weyl-ineq} \textbf{(b)} is left to the
reader as well.
\end{proof}

\subsection{([Missing lecture]) The interlacing theorem}

\subsection{Consequences of the interlacing theorem}

Recall: If $A\in\mathbb{C}^{n\times n}$ is a Hermitian matrix (i.e., a square
matrix satisfying $A^{\ast}=A$), then we denote its eigenvalues by
$\lambda_{1}\left(  A\right)  ,\lambda_{2}\left(  A\right)  ,\ldots
,\lambda_{n}\left(  A\right)  $ in weakly increasing order (with
multiplicities). This makes sense, since we know that these eigenvalues are reals.

Last time, Hugo proved:

\begin{theorem}
[Cauchy's interlacing theorem, aka eigenvalue interlacing theorem]%
\label{thm.interlacing.1}Let $A\in\mathbb{C}^{n\times n}$ be a Hermitian
matrix. Let $j\in\left[  n\right]  $. Let $B\in\mathbb{C}^{\left(  n-1\right)
\times\left(  n-1\right)  }$ be the matrix obtained from $A$ by removing the
$j$-th row and the $j$-th column. Then,%
\[
\lambda_{1}\left(  A\right)  \leq\lambda_{1}\left(  B\right)  \leq\lambda
_{2}\left(  A\right)  \leq\lambda_{2}\left(  B\right)  \leq\cdots\leq
\lambda_{n-1}\left(  A\right)  \leq\lambda_{n-1}\left(  B\right)  \leq
\lambda_{n}\left(  A\right)  .
\]
In other words,%
\[
\lambda_{i}\left(  A\right)  \leq\lambda_{i}\left(  B\right)  \leq
\lambda_{i+1}\left(  A\right)  \ \ \ \ \ \ \ \ \ \ \text{for each }i\in\left[
n-1\right]  .
\]

\end{theorem}

A converse of this theorem also holds:

\begin{proposition}
\label{prop.interlacing.converse}Let $\lambda_{1},\lambda_{2},\ldots
,\lambda_{n}$ and $\mu_{1},\mu_{2},\ldots,\mu_{n-1}$ be real numbers
satisfying
\[
\lambda_{1}\leq\mu_{1}\leq\lambda_{2}\leq\mu_{2}\leq\cdots\leq\lambda
_{n-1}\leq\mu_{n-1}\leq\lambda_{n}.
\]
Then, there exist $n-1$ reals $y_{1},y_{2},\ldots,y_{n-1}\in\mathbb{R}$ and a
real $a\in\mathbb{R}$ such that the matrix%
\[
A:=\left(
\begin{array}
[c]{ccccc}%
\mu_{1} &  &  &  & y_{1}\\
& \mu_{2} &  &  & y_{2}\\
&  & \ddots &  & \vdots\\
&  &  & \mu_{n-1} & y_{n-1}\\
y_{1} & y_{2} & \cdots & y_{n-1} & a
\end{array}
\right)
\]
(where all empty cells are supposed to be filled with $0$s) has eigenvalues
$\lambda_{1},\lambda_{2},\ldots,\lambda_{n}$. (This matrix is, of course,
Hermitian, since it is real symmetric.)
\end{proposition}

\begin{proof}
Omitted. (Exercise?)
\end{proof}

Now, let us derive some consequences from Cauchy's interlacing theorem. We
begin with a straightforward generalization:

\begin{corollary}
[Cauchy's interlacing theorem for multiple deletions]\label{cor.interlacing.r}%
Let $A\in\mathbb{C}^{n\times n}$ be a Hermitian matrix. Let $r\in\left\{
0,1,\ldots,n\right\}  $. Let $C\in\mathbb{C}^{r\times r}$ be the result of
removing $n-r$ rows and the corresponding $n-r$ columns from $A$. (That is, we
pick some $j_{1}<j_{2}<\cdots<j_{n-r}$, and we remove the $j_{1}$-st, $j_{2}%
$-nd, $\ldots$, $j_{n-r}$-th rows from $A$, and we remove the $j_{1}$-st,
$j_{2}$-nd, $\ldots$, $j_{n-r}$-th columns from $A$.) Then, for each
$j\in\left[  r\right]  $, we have%
\[
\lambda_{j}\left(  A\right)  \leq\lambda_{j}\left(  C\right)  \leq
\lambda_{j+n-r}\left(  A\right)  .
\]

\end{corollary}

\begin{proof}
Induction on $n-r$.

The \textit{base case} ($n-r=0$) is trivial, since $C=A$ in this case.

In the \textit{induction step}, we obtain $C$ from $B$ by removing a single
row and the corresponding column. Thus, by the original Cauchy interlacing
theorem, we get $\lambda_{j}\left(  B\right)  \leq\lambda_{j}\left(  C\right)
$. However, by the induction hypothesis, we get $\lambda_{j}\left(  A\right)
\leq\lambda_{j}\left(  B\right)  $. Combining these inequalities, we get
$\lambda_{j}\left(  A\right)  \leq\lambda_{j}\left(  C\right)  $. The
remaining inequality $\lambda_{j}\left(  C\right)  \leq\lambda_{j+n-r}\left(
A\right)  $ is proved similarly: By the original Cauchy interlacing theorem,
we get $\lambda_{j}\left(  C\right)  \leq\lambda_{j+1}\left(  B\right)  $.
However, by the induction hypothesis, we get $\lambda_{j+1}\left(  B\right)
\leq\lambda_{j+1+\left(  n-r-1\right)  }\left(  A\right)  =\lambda
_{j+n-r}\left(  A\right)  $.
\end{proof}

The next corollary provides a minimum/maximum description of the sum of the
first $m$ smallest/largest eigenvalues of a Hermitian matrix:

\begin{corollary}
\label{cor.interlacing.sum-first-m-eigs}Let $A\in\mathbb{C}^{n\times n}$ be a
Hermitian matrix. Let $m\in\left\{  0,1,\ldots,n\right\}  $. Then:

\textbf{(a)} We have%
\[
\lambda_{1}\left(  A\right)  +\lambda_{2}\left(  A\right)  +\cdots+\lambda
_{m}\left(  A\right)  =\min\limits_{\text{isometries }V\in\mathbb{C}^{n\times
m}}\operatorname*{Tr}\left(  V^{\ast}AV\right)  .
\]


\textbf{(b)} We have%
\[
\lambda_{n-m+1}\left(  A\right)  +\lambda_{n-m+2}\left(  A\right)
+\cdots+\lambda_{n}\left(  A\right)  =\max\limits_{\text{isometries }%
V\in\mathbb{C}^{n\times m}}\operatorname*{Tr}\left(  V^{\ast}AV\right)  .
\]

\end{corollary}

\begin{proof}
First of all, it suffices to show the first equality, because the second
follows by applying the first to $-A$ instead of $A$.

First, we shall show that%
\[
\lambda_{1}\left(  A\right)  +\lambda_{2}\left(  A\right)  +\cdots+\lambda
_{m}\left(  A\right)  \leq\operatorname*{Tr}\left(  V^{\ast}AV\right)
\ \ \ \ \ \ \ \ \ \ \text{for every isometry }V\in\mathbb{C}^{n\times m}.
\]


Indeed, let $V\in\mathbb{C}^{n\times m}$ be an isometry. Thus, $V$ is an
$n\times m$-matrix whose columns are orthonormal. As we know from Corollary
\ref{cor.unitary.orthon-extend}, we can extend each orthonormal tuple of
vectors to an orthonormal basis. Doing this to the columns of $V$, we thus
obtain an orthonormal basis of $\mathbb{C}^{n}$ whose first $m$ entries are
the columns of $V$. Let $U$ be the matrix whose columns are the entries of
this basis. Then,%
\[
U=\left(
\begin{array}
[c]{cc}%
V & \widetilde{V}%
\end{array}
\right)  \ \ \ \ \ \ \ \ \ \ \left(  \text{in block-matrix notation}\right)
\]
by construction of this basis, and furthermore the matrix $U$ is unitary since
its columns form an orthonormal basis.

Since $U$ is unitary, we have $U^{\ast}AU\sim A$ and therefore%
\[
\lambda_{j}\left(  U^{\ast}AU\right)  =\lambda_{j}\left(  A\right)
\ \ \ \ \ \ \ \ \ \ \text{for all }j\in\left[  n\right]  .
\]


However, $U=\left(
\begin{array}
[c]{cc}%
V & \widetilde{V}%
\end{array}
\right)  $ entails%
\[
U^{\ast}AU=\left(
\begin{array}
[c]{cc}%
V & \widetilde{V}%
\end{array}
\right)  ^{\ast}A\left(
\begin{array}
[c]{cc}%
V & \widetilde{V}%
\end{array}
\right)  =\left(
\begin{array}
[c]{c}%
V^{\ast}\\
\widetilde{V}^{\ast}%
\end{array}
\right)  A\left(
\begin{array}
[c]{cc}%
V & \widetilde{V}%
\end{array}
\right)  =\left(
\begin{array}
[c]{cc}%
V^{\ast}AV & \ast\\
\ast & \ast
\end{array}
\right)  ,
\]
where the three $\ast$s mean blocks that we don't care about. So the matrix
$V^{\ast}AV$ is obtained from $U^{\ast}AU$ by removing a bunch of rows and the
corresponding columns. Hence, Corollary \ref{cor.interlacing.r} yields%
\[
\lambda_{j}\left(  U^{\ast}AU\right)  \leq\lambda_{j}\left(  V^{\ast
}AV\right)  \ \ \ \ \ \ \ \ \ \ \text{for all }j\in\left[  m\right]
\]
(since $U^{\ast}AU$ is Hermitian (because $A$ is Hermitian)). In other words,%
\[
\lambda_{j}\left(  A\right)  \leq\lambda_{j}\left(  V^{\ast}AV\right)
\ \ \ \ \ \ \ \ \ \ \text{for all }j\in\left[  m\right]
\]
(since $\lambda_{j}\left(  U^{\ast}AU\right)  =\lambda_{j}\left(  A\right)
$). Adding these inequalities together, we obtain%
\begin{align*}
&  \lambda_{1}\left(  A\right)  +\lambda_{2}\left(  A\right)  +\cdots
+\lambda_{m}\left(  A\right) \\
&  \leq\lambda_{1}\left(  V^{\ast}AV\right)  +\lambda_{2}\left(  V^{\ast
}AV\right)  +\cdots+\lambda_{m}\left(  V^{\ast}AV\right) \\
&  =\left(  \text{the sum of all eigenvalues of }V^{\ast}AV\right) \\
&  \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \left(
\begin{array}
[c]{c}%
\text{since }V^{\ast}AV\text{ is an }m\times m\text{-matrix}\\
\text{and thus has }m\text{ eigenvalues}%
\end{array}
\right) \\
&  =\operatorname*{Tr}\left(  V^{\ast}AV\right)
\end{align*}
(since the sum of all eigenvalues of a matrix is the trace of this matrix).

Now, we need to show that there exists a unitary matrix $V\in\mathbb{C}%
^{n\times m}$ such that%
\[
\lambda_{1}\left(  A\right)  +\lambda_{2}\left(  A\right)  +\cdots+\lambda
_{m}\left(  A\right)  =\operatorname*{Tr}\left(  V^{\ast}AV\right)  .
\]
To do this, we construct $V$ as follows: We pick an eigenvector $x_{i}$ of $A$
at eigenvalue $\lambda_{i}\left(  A\right)  $ for each $i\in\left[  n\right]
$ in such a way that $\left(  x_{1},x_{2},\ldots,x_{n}\right)  $ is an
orthonormal basis of $\mathbb{C}^{n}$. (This is possible because of Theorem
2.6.1 \textbf{(b)}.) Now, let $V\in\mathbb{C}^{n\times m}$ be the matrix whose
columns are $x_{1},x_{2},\ldots,x_{m}$. This matrix $V$ is an isometry, since
$x_{1},x_{2},\ldots,x_{m}$ are orthonormal. Moreover,%
\begin{align*}
V^{\ast}AV  &  =\left(
\begin{array}
[c]{c}%
x_{1}^{\ast}\\
x_{2}^{\ast}\\
\vdots\\
x_{m}^{\ast}%
\end{array}
\right)  A\left(
\begin{array}
[c]{cccc}%
x_{1} & x_{2} & \cdots & x_{m}%
\end{array}
\right) \\
&  =\left(
\begin{array}
[c]{cccc}%
x_{1}^{\ast}Ax_{1} & x_{1}^{\ast}Ax_{2} & \cdots & x_{1}^{\ast}Ax_{m}\\
x_{2}^{\ast}Ax_{1} & x_{2}^{\ast}Ax_{2} & \cdots & x_{2}^{\ast}Ax_{m}\\
\vdots & \vdots & \ddots & \vdots\\
x_{m}^{\ast}Ax_{1} & x_{m}^{\ast}Ax_{2} & \cdots & x_{m}^{\ast}Ax_{m}%
\end{array}
\right)  ,
\end{align*}
so that%
\begin{align*}
\operatorname*{Tr}\left(  V^{\ast}AV\right)   &  =\sum_{j=1}^{m}x_{j}^{\ast
}\underbrace{Ax_{j}}_{\substack{=\lambda_{j}\left(  A\right)  x_{j}%
\\\text{(since }x_{j}\text{ is an eigenvector of }A\\\text{at eigenvalue
}\lambda_{j}\left(  A\right)  \text{)}}}=\sum_{j=1}^{m}\lambda_{j}\left(
A\right)  \underbrace{x_{j}^{\ast}x_{j}}_{\substack{=\left\vert \left\vert
x_{j}\right\vert \right\vert ^{2}=1\\\text{(since }\left(  x_{1},x_{2}%
,\ldots,x_{n}\right)  \\\text{is an orthonormal basis)}}}\\
&  =\sum_{j=1}^{m}\lambda_{j}\left(  A\right)  =\lambda_{1}\left(  A\right)
+\lambda_{2}\left(  A\right)  +\cdots+\lambda_{m}\left(  A\right)  .
\end{align*}
This is precisely what we needed. Thus, we conclude that%
\[
\lambda_{1}\left(  A\right)  +\lambda_{2}\left(  A\right)  +\cdots+\lambda
_{m}\left(  A\right)  =\min\limits_{\text{isometries }V\in\mathbb{C}^{n\times
m}}\operatorname*{Tr}\left(  V^{\ast}AV\right)  .
\]
As we said above, this completes the proof.
\end{proof}

\begin{corollary}
\label{cor.interlacing.maj-1}Let $A\in\mathbb{C}^{n\times n}$ be a Hermitian
$n\times n$-matrix. Let $m\in\left\{  0,1,\ldots,n\right\}  $. Let
$i_{1},i_{2},\ldots,i_{m}\in\left[  n\right]  $ be distinct. Then,%
\begin{align*}
\lambda_{1}\left(  A\right)  +\lambda_{2}\left(  A\right)  +\cdots+\lambda
_{m}\left(  A\right)   &  \leq A_{i_{1},i_{1}}+A_{i_{2},i_{2}}+\cdots
+A_{i_{m},i_{m}}\\
&  \leq\lambda_{n-m+1}\left(  A\right)  +\lambda_{n-m+2}\left(  A\right)
+\cdots+\lambda_{n}\left(  A\right)  .
\end{align*}

\end{corollary}

In words: For a Hermitian matrix $A$, each sum of $m$ distinct diagonal
entries of $A$ is sandwiched between the sum of the $m$ smallest eigenvalues
of $A$ and the sum of the $m$ largest eigenvalues of $A$.

\begin{proof}
Let $C$ be the matrix obtained from $A$ by removing all but the $i_{1}$-st,
$i_{2}$-nd, $\ldots$, $i_{m}$-th rows and the corresponding columns of $A$.
Then,%
\[
\operatorname*{Tr}C=A_{i_{1},i_{1}}+A_{i_{2},i_{2}}+\cdots+A_{i_{m},i_{m}}.
\]


However, Cauchy's interlacing theorem for multiple deletions yields%
\[
\lambda_{j}\left(  A\right)  \leq\lambda_{j}\left(  C\right)
\ \ \ \ \ \ \ \ \ \ \text{for each }j\in\left[  m\right]  .
\]
Summing these up, we obtain%
\begin{align*}
\lambda_{1}\left(  A\right)  +\lambda_{2}\left(  A\right)  +\cdots+\lambda
_{m}\left(  A\right)   &  \leq\lambda_{1}\left(  C\right)  +\lambda_{2}\left(
C\right)  +\cdots+\lambda_{m}\left(  C\right) \\
&  =\left(  \text{the sum of all eigenvalues of }C\right) \\
&  =\operatorname*{Tr}C=A_{i_{1},i_{1}}+A_{i_{2},i_{2}}+\cdots+A_{i_{m},i_{m}%
}.
\end{align*}
So we have proved the first of the required two inequalities. The second
follows by applying the first to $-A$ instead of $A$.
\end{proof}

The above corollary has a bunch of consequences that are obtained by restating
it in terms of something called \emph{majorization}. Let us define this
concept and see what it entails.

\subsection{Introduction to majorization theory (\cite[\S 4.3]{HorJoh13})}

We will now give a brief introduction to majorization theory. Much more can be
found in \cite{MaOlAr11} (see also \cite{Nathan21} for an elementary introduction).

\subsubsection{Notations and definition}

\begin{convention}
\label{conv.major.notations}Let $x\in\mathbb{R}^{n}$ be a column vector with
real entries. Then: \medskip

\textbf{(a)} For each $i\in\left[  n\right]  $, we let $x_{i}$ denote the
$i$-th coordinate of $x$. (\textquotedblleft Coordinate\textquotedblright\ is
just a synonym for \textquotedblleft entry\textquotedblright\ in the context
of a vector.) Thus,%
\[
x=\left(
\begin{array}
[c]{c}%
x_{1}\\
x_{2}\\
\vdots\\
x_{n}%
\end{array}
\right)  =\left(  x_{1},x_{2},\ldots,x_{n}\right)  ^{T}.
\]


\textbf{(b)} We say that $x$ is \emph{weakly decreasing} if $x_{1}\geq
x_{2}\geq\cdots\geq x_{n}$. We say that $x$ is \emph{weakly increasing} if
$x_{1}\leq x_{2}\leq\cdots\leq x_{n}$. \medskip

\textbf{(c)} We let $x^{\downarrow}$ denote the weakly decreasing permutation
of $x$ (that is, the column vector obtained by sorting the entries of $x$ in
weakly decreasing order). In other words, $x^{\downarrow}$ is the unique
weakly decreasing column vector that can be obtained by permuting the entries
of $x$.

Thus, $x_{i}^{\downarrow}$ is the $i$-th largest entry of $x$ for each
$i\in\left[  n\right]  $; in particular, $x_{1}^{\downarrow}\geq
x_{2}^{\downarrow}\geq\cdots\geq x_{n}^{\downarrow}$. \medskip

\textbf{(d)} We let $x^{\uparrow}$ denote the weakly increasing permutation of
$x$ (that is, the column vector obtained by sorting the entries of $x$ in
weakly increasing order). In other words, $x^{\uparrow}$ is the unique weakly
increasing column vector that can be obtained by permuting the entries of $x$.

Thus, $x_{i}^{\uparrow}$ is the $i$-th smallest entry of $x$ for each
$i\in\left[  n\right]  $; in particular, $x_{1}^{\uparrow}\leq x_{2}%
^{\uparrow}\leq\cdots\leq x_{n}^{\uparrow}$.
\end{convention}

For example, if $x=\left(  3,5,2\right)  ^{T}$, then $x_{1}=3$ and $x_{2}=5$
and $x_{3}=2$ and%
\begin{align*}
x^{\downarrow}  &  =\left(  5,3,2\right)  ^{T}\ \ \ \ \ \ \ \ \ \ \text{and
}x_{1}^{\downarrow}=5\text{ and }x_{2}^{\downarrow}=3\text{ and }%
x_{3}^{\downarrow}=2\ \ \ \ \ \ \ \ \ \ \text{and}\\
x^{\uparrow}  &  =\left(  2,3,5\right)  ^{T}\ \ \ \ \ \ \ \ \ \ \text{and
}x_{1}^{\uparrow}=2\text{ and }x_{2}^{\uparrow}=3\text{ and }x_{3}^{\uparrow
}=5.
\end{align*}


\begin{definition}
\label{def.major.major}Let $x\in\mathbb{R}^{n}$ and $y\in\mathbb{R}^{n}$ be
two column vectors with real entries. Then, we say that $x$ \emph{majorizes}
$y$ (and we write $x\succcurlyeq y$) if and only if we have%
\[
\sum_{i=1}^{m}x_{i}^{\downarrow}\geq\sum_{i=1}^{m}y_{i}^{\downarrow
}\ \ \ \ \ \ \ \ \ \ \text{for each }m\in\left[  n\right]
\]
and%
\[
\sum_{i=1}^{n}x_{i}^{\downarrow}=\sum_{i=1}^{n}y_{i}^{\downarrow}.
\]
In other words, $x$ majorizes $y$ if and only if we have%
\begin{align*}
x_{1}^{\downarrow}  &  \geq y_{1}^{\downarrow};\\
x_{1}^{\downarrow}+x_{2}^{\downarrow}  &  \geq y_{1}^{\downarrow}%
+y_{2}^{\downarrow};\\
x_{1}^{\downarrow}+x_{2}^{\downarrow}+x_{3}^{\downarrow}  &  \geq
y_{1}^{\downarrow}+y_{2}^{\downarrow}+y_{3}^{\downarrow};\\
&  \ldots;\\
x_{1}^{\downarrow}+x_{2}^{\downarrow}+\cdots+x_{n-1}^{\downarrow}  &  \geq
y_{1}^{\downarrow}+y_{2}^{\downarrow}+\cdots+y_{n-1}^{\downarrow};\\
x_{1}^{\downarrow}+x_{2}^{\downarrow}+\cdots+x_{n}^{\downarrow}  &
=y_{1}^{\downarrow}+y_{2}^{\downarrow}+\cdots+y_{n}^{\downarrow}%
\end{align*}
(note that the last relation in this chain is an equality, not just an inequality).
\end{definition}

\begin{example}
We have%
\[
\left(
\begin{array}
[c]{c}%
1\\
3\\
5\\
7
\end{array}
\right)  \succcurlyeq\left(
\begin{array}
[c]{c}%
2\\
2\\
6\\
6
\end{array}
\right)  ,
\]
since%
\begin{align*}
7  &  \geq6;\\
7+5  &  \geq6+6;\\
7+5+3  &  \geq6+6+2;\\
7+5+3+1  &  =6+6+2+2.
\end{align*}

\end{example}

\begin{example}
We don't have%
\[
\left(
\begin{array}
[c]{c}%
1\\
3\\
5\\
7
\end{array}
\right)  \succcurlyeq\left(
\begin{array}
[c]{c}%
0\\
2\\
6\\
8
\end{array}
\right)  ,
\]
since we don't have $7\geq8$.
\end{example}

The intuition behind majorization is the following: A column vector $x$
majorizes a column vector $y$ if and only if you can obtain $y$ from $x$ by
\textquotedblleft moving the entries closer together (while keeping the
average equal)\textquotedblright. We will soon see a rigorous way to state this.

Here are some more general examples of majorization:

\begin{exercise}
\label{exe.major.average}\fbox{2} Let $x\in\mathbb{R}^{n}$, and let
$m=\dfrac{x_{1}+x_{2}+\cdots+x_{n}}{n}$. Show that $x\succcurlyeq\left(
m,m,\ldots,m\right)  ^{T}$.
\end{exercise}

\begin{exercise}
\fbox{2} Let $a,b,c\in\mathbb{R}$, and let $x=\dfrac{b+c}{2}$ and
$y=\dfrac{c+a}{2}$ and $z=\dfrac{a+b}{2}$. Show that $\left(  a,b,c\right)
^{T}\succcurlyeq\left(  x,y,z\right)  ^{T}$.
\end{exercise}

\begin{exercise}
\fbox{4} Let $a,b,c\in\mathbb{R}$, and let $x=\dfrac{b+c}{2}$ and
$y=\dfrac{c+a}{2}$ and $z=\dfrac{a+b}{2}$ and $m=\dfrac{a+b+c}{3}$. Show that%
\[
\left(  a,b,c,m,m,m\right)  ^{T}\succcurlyeq\left(  x,x,y,y,z,z\right)  ^{T}.
\]

\end{exercise}

\begin{exercise}
\fbox{3} Let $x,y\in\mathbb{R}^{n}$. Prove that $x\succcurlyeq y$ if and only
if $-x\succcurlyeq-y$.
\end{exercise}

We note that the condition $\sum_{i=1}^{n}x_{i}^{\downarrow}=\sum_{i=1}%
^{n}y_{i}^{\downarrow}$ in the definition of majorization can be rewritten as
$\sum_{i=1}^{n}x_{i}=\sum_{i=1}^{n}y_{i}$, because the sum of all coordinates
of a vector does not change when we permute the entries of the vector.
However, the conditions $\sum_{i=1}^{m}x_{i}^{\downarrow}\geq\sum_{i=1}%
^{m}y_{i}^{\downarrow}$ cannot be rewritten as $\sum_{i=1}^{m}x_{i}\geq
\sum_{i=1}^{m}y_{i}$.

\begin{proposition}
Majorization is a partial order: That is, the binary relation $\succcurlyeq$
on the set $\mathbb{R}^{n}$ is reflexive, antisymmetric and transitive.
\end{proposition}

\begin{proof}
This is straightforward to verify. For example, if $\sum_{i=1}^{m}%
x_{i}^{\downarrow}\geq\sum_{i=1}^{m}y_{i}^{\downarrow}$ and $\sum_{i=1}%
^{m}y_{i}^{\downarrow}\geq\sum_{i=1}^{m}z_{i}^{\downarrow}$, then $\sum
_{i=1}^{m}x_{i}^{\downarrow}\geq\sum_{i=1}^{m}z_{i}^{\downarrow}$.
\end{proof}

However, majorization is not a total order: For example, the vectors%
\[
x=\left(
\begin{array}
[c]{c}%
2\\
2\\
4\\
6
\end{array}
\right)  \ \ \ \ \ \ \ \ \ \ \text{and}\ \ \ \ \ \ \ \ \ \ y=\left(
\begin{array}
[c]{c}%
1\\
3\\
5\\
5
\end{array}
\right)
\]
satisfy neither $x\succcurlyeq y$ nor $y\succcurlyeq x$, since we have $6>5$
but $6+4+2<5+5+3$. For an even simpler example, if two vectors $x\in
\mathbb{R}^{n}$ and $y\in\mathbb{R}^{n}$ have different sums of coordinates,
then we have neither $x\succcurlyeq y$ nor $y\succcurlyeq x$.

\subsubsection{Restating Schur's theorem as a majorization}

Now we can restate Corollary \ref{cor.interlacing.maj-1} as follows:

\begin{corollary}
[Schur's theorem]\label{cor.interlacing.maj-2}Let $A\in\mathbb{C}^{n\times n}$
be a Hermitian $n\times n$-matrix. Then,%
\[
\left(  \lambda_{1}\left(  A\right)  ,\lambda_{2}\left(  A\right)
,\ldots,\lambda_{n}\left(  A\right)  \right)  ^{T}\succcurlyeq\left(
A_{1,1},A_{2,2},\ldots,A_{n,n}\right)  ^{T}.
\]
In words: The tuple of all eigenvalues of $A$ majorizes the tuple of all
diagonal entries of $A$.
\end{corollary}

\begin{proof}
We need to show that

\begin{itemize}
\item the sum of the $m$ largest eigenvalues of $A$ is $\geq$ to the sum of
the $m$ largest diagonal entries of $A$ for each $m\in\left[  n\right]  $;

\item the sum of all diagonal entries of $A$ equals the sum of all eigenvalues
of $A$.
\end{itemize}

But the second of these two statements follows from the well-known theorem
that the trace of a matrix is the sum of its eigenvalues (Theorem
\ref{thm.schurtri.ch.fta-cons} \textbf{(d)}). Thus, it remains to prove the
first statement.

So let $m\in\left[  n\right]  $. We must prove that the sum of the $m$ largest
eigenvalues of $A$ is $\geq$ to the sum of the $m$ largest diagonal entries of
$A$. However, Corollary \ref{cor.interlacing.maj-1} yields that%
\begin{align*}
\lambda_{1}\left(  A\right)  +\lambda_{2}\left(  A\right)  +\cdots+\lambda
_{m}\left(  A\right)   &  \leq A_{i_{1},i_{1}}+A_{i_{2},i_{2}}+\cdots
+A_{i_{m},i_{m}}\\
&  \leq\lambda_{n-m+1}\left(  A\right)  +\lambda_{n-m+2}\left(  A\right)
+\cdots+\lambda_{n-m}\left(  A\right)
\end{align*}
for any distinct $i_{1},i_{2},\ldots,i_{m}\in\left[  n\right]  $. The second
inequality here says that%
\[
\lambda_{n-m+1}\left(  A\right)  +\lambda_{n-m+2}\left(  A\right)
+\cdots+\lambda_{n-m}\left(  A\right)  \geq A_{i_{1},i_{1}}+A_{i_{2},i_{2}%
}+\cdots+A_{i_{m},i_{m}}%
\]
for any distinct $i_{1},i_{2},\ldots,i_{m}\in\left[  n\right]  $. In other
words, the sum of the $m$ largest eigenvalues of $A$ is $\geq$ to any sum of
$m$ distinct diagonal entries of $A$. Thus, in particular, the sum of the $m$
largest eigenvalues of $A$ is $\geq$ to the sum of the $m$ largest diagonal
entries of $A$. This completes the proof of Corollary
\ref{cor.interlacing.maj-2}.
\end{proof}

\begin{exercise}
\fbox{5} For each Hermitian matrix $A\in\mathbb{C}^{n\times n}$, let
$\lambda\left(  A\right)  \in\mathbb{R}^{n}$ be the column vector $\left(
\lambda_{n}\left(  A\right)  ,\ \lambda_{n-1}\left(  A\right)  ,\ \ldots
,\ \lambda_{1}\left(  A\right)  \right)  ^{T}$ consisting of all eigenvalues
of $A$ in decreasing order. \medskip

Let $A\in\mathbb{C}^{n\times n}$ and $B\in\mathbb{C}^{n\times n}$ be two
Hermitian matrices. Prove the following: \medskip

\textbf{(a)} (\emph{Fan's theorem:}) We have%
\[
\lambda\left(  A\right)  +\lambda\left(  B\right)  \succcurlyeq\lambda\left(
A+B\right)  .
\]


\textbf{(b)} (\emph{Lidskii's theorem:}) We have%
\[
\lambda\left(  A+B\right)  \succcurlyeq\lambda\left(  A\right)  +\left(
\lambda\left(  B\right)  \right)  ^{\uparrow}.
\]


[\textbf{Hint:} For part \textbf{(a)}, use Corollary
\ref{cor.interlacing.sum-first-m-eigs} \textbf{(b)}. For part \textbf{(b)},
observe first that three weakly decreasing vectors $x,y,z\in\mathbb{R}^{n}$
satisfying $x-z^{\uparrow}\succcurlyeq y$ must always satisfy $x\succcurlyeq
y+z^{\uparrow}$.]
\end{exercise}

\subsubsection{Robin Hood moves}

Above, we briefly alluded to an intuition for majorization: We said that $x$
majorizes $y$ if and only if you can obtain $y$ from $x$ by \textquotedblleft
moving the entries closer together (while keeping the average
equal)\textquotedblright. Let us now turn this into an actual theorem. First,
a simple lemma:

\begin{lemma}
\label{lem.major.order-ignore}Let $x,y\in\mathbb{R}^{n}$. Then, $x\succcurlyeq
y$ if and only if $x^{\downarrow}\succcurlyeq y^{\downarrow}$.
\end{lemma}

\begin{proof}
The definition of $\succcurlyeq$ only involves $x^{\downarrow}$ and
$y^{\downarrow}$. In other words, whether or not we have $x\succcurlyeq y$
does not depend on the order of the coordinates of $x$ or of those of $y$.
Thus, replacing $x$ and $y$ by $x^{\downarrow}$ and $y^{\downarrow}$ doesn't
make any difference.
\end{proof}

\begin{definition}
\label{def.major.RH}Let $x\in\mathbb{R}^{n}$. Let $i$ and $j$ be two distinct
elements of $\left[  n\right]  $ such that $x_{i}\leq x_{j}$. Let $t\in\left[
x_{i},x_{j}\right]  $ (that is, $t\in\mathbb{R}$ and $x_{i}\leq t\leq x_{j}$).
Let $y\in\mathbb{R}^{n}$ be the column vector obtained from $x$ by%
\[
\text{replacing the }i\text{-th and }j\text{-th coordinates }x_{i}\text{ and
}x_{j}\text{ by }u\text{ and }v
\]
for some $u,v\in\left[  x_{i},x_{j}\right]  $ satisfying $u+v=x_{i}+x_{j}$. In
other words, we obtain $y$ by picking two numbers $u,v\in\left[  x_{i}%
,x_{j}\right]  $ satisfying $u+v=x_{i}+x_{j}$ and setting%
\[
y_{k}=%
\begin{cases}
u, & \text{if }k=i;\\
v, & \text{if }k=j;\\
x_{k}, & \text{otherwise}%
\end{cases}
\ \ \ \ \ \ \ \ \ \ \text{for all }k\in\left[  n\right]  .
\]


Then, we say that $y$ is obtained from $x$ by a \emph{Robin Hood move} (short:
\emph{RH move}), and we write
\[
x\overset{\text{RH}}{\longrightarrow}y.
\]


Moreover, if $x$ and $y$ are weakly decreasing, then this RH move is said to
be an \emph{order-preserving RH move} (short: \emph{OPRH move}), and we write
\[
x\overset{\text{OPRH}}{\longrightarrow}y.
\]

\end{definition}

A Robin Hood move is thus a \textquotedblleft local
transformation\textquotedblright\ that changes a column vector $x$ by picking
two of its entries (say, $x_{i}$ and $x_{j}$) and replacing them by two new
entries $u$ and $v$ that are \textquotedblleft closer
together\textquotedblright\ (that is, $u,v\in\left[  x_{i},x_{j}\right]  $)
and have the same sum (that is, $u+v=x_{i}+x_{j}$). If we regard the entries
$x_{1},x_{2},\ldots,x_{n}$ of $x$ as the wealths of $n$ persons, then a Robin
Hood move thus corresponds to redistributing some wealth from a richer person
to a poorer person in such a way that the disparity between these two people
does not increase. (Thus the name.) If the $n$ people were ordered in the
order of decreasing wealth at the beginning (that is, $x$ was weakly
decreasing) and this remains so after the RH move, then the RH move is an OPRH move.

\begin{example}
\textbf{(a)} Replacing two coordinates of a vector $x$ by their average is an
RH move. (Indeed, this corresponds to the case when $u=v=\dfrac{x_{i}+x_{j}%
}{2}$ in Definition \ref{def.major.RH}.) \medskip

\textbf{(b)} Swapping two coordinates of a vector $x$ is an RH move. (Indeed,
this corresponds to the case when $u=x_{j}$ and $v=x_{i}$ in Definition
\ref{def.major.RH}.) \medskip

\textbf{(c)} If $x\in\mathbb{R}^{n}$ is weakly decreasing, then replacing two
adjacent entries of $x$ by their average is an OPRH move. (Indeed, if we
replace $x_{i}$ and $x_{i+1}$ by their average $\dfrac{x_{i}+x_{i+1}}{2}$,
then the vector remains weakly decreasing, since $x_{1}\geq x_{2}\geq
\cdots\geq x_{n}$ entails $x_{1}\geq x_{2}\geq\cdots\geq x_{i-1}\geq
\dfrac{x_{i}+x_{i+1}}{2}\geq\dfrac{x_{i}+x_{i+1}}{2}\geq x_{i+2}\geq
x_{i+3}\geq\cdots\geq x_{n}$.) \medskip

\textbf{(d)} More generally: If $x\in\mathbb{R}^{n}$ is weakly decreasing,
then replacing its coordinates $x_{i}$ and $x_{i+1}$ by $u$ and $x_{i}%
+x_{i+1}-u$ is an OPRH move if and only if $u\in\left[  \dfrac{x_{i}+x_{i+1}%
}{2},x_{i}\right]  $. \medskip

\textbf{(e)} Here is an example of an RH move that is not an OPRH move:%
\[
\left(
\begin{array}
[c]{c}%
6\\
5\\
2\\
1
\end{array}
\right)  \overset{\text{RH}}{\longrightarrow}\left(
\begin{array}
[c]{c}%
4\\
5\\
2\\
3
\end{array}
\right)  .
\]
This move replaces the two entries $1$ and $6$ by $3$ and $4$ (with
$3,4\in\left[  1,6\right]  $ and $3+4=1+6$), but it changes the relative order
of the entries, so it is not order-preserving.
\end{example}

\begin{proposition}
\label{prop.major.RH.sum-equals}If $x\overset{\text{RH}}{\longrightarrow}y$,
then the sum of the entries of $x$ equals the sum of the entries of $y$.
\end{proposition}

\begin{proof}
Clear from the $u+v=x_{i}+x_{j}$ requirement in Definition \ref{def.major.RH}.
\end{proof}

\begin{lemma}
\label{lem.major.RH.OPRH-maj}Let $x,y\in\mathbb{R}^{n}$ be weakly decreasing
column vectors such that $y$ is obtained from $x$ by a (finite) sequence of
OPRH moves. Then, $x\succcurlyeq y$.
\end{lemma}

\begin{proof}
Let us first prove Lemma \ref{lem.major.RH.OPRH-maj} in the case when $y$ is
obtained from $x$ by a \textbf{single} OPRH move.

So let us assume that $y$ is obtained from $x$ by a \textbf{single} OPRH move.
Let this move be replacing $x_{i}$ and $x_{j}$ by $u$ and $v$, where
$x_{i}\leq x_{j}$ and $u,v\in\left[  x_{i},x_{j}\right]  $ with $u+v=x_{i}%
+x_{j}$. WLOG, we have $x_{i}<x_{j}$ (since otherwise, the OPRH move changes
nothing, because we have $u=v=x_{i}=x_{j}$). Therefore, $i>j$ (since $x$ is
weakly decreasing (by the definition of an OPRH move)). Thus,%
\[
y=\left(  x_{1},x_{2},\ldots,x_{j-1},v,x_{j+1},x_{j+2},\ldots,x_{i-1}%
,u,x_{i+1},x_{i+2},\ldots,x_{n}\right)  ^{T}%
\]
(since $y$ is obtained from $x$ by replacing $x_{i}$ and $x_{j}$ by $u$ and
$v$).

Now, we must prove that $x\succcurlyeq y$. In other words, we must prove that%
\begin{equation}
x_{1}+x_{2}+\cdots+x_{m}\geq y_{1}+y_{2}+\cdots+y_{m}
\label{pf.lem.major.RH.OPRH-maj.ineq-to-prove}%
\end{equation}
for each $m\in\left[  n\right]  $ (since $x$ and $y$ are weakly decreasing),
and we must prove that%
\begin{equation}
x_{1}+x_{2}+\cdots+x_{n}=y_{1}+y_{2}+\cdots+y_{n}.
\label{pf.lem.major.RH.OPRH-maj.eq-to-prove}%
\end{equation}


The latter equality follows from $u+v=x_{i}+x_{j}$. So we only need to prove
the former inequality. So let us fix an $m\in\left[  n\right]  $. We must
prove the inequality (\ref{pf.lem.major.RH.OPRH-maj.ineq-to-prove}). We are in
one of the following cases:

\begin{enumerate}
\item We have $m<j$.

\item We have $j\leq m<i$.

\item We have $i\leq m$.
\end{enumerate}

In Case 1, we have $x_{1}+x_{2}+\cdots+x_{m}=y_{1}+y_{2}+\cdots+y_{m}$
(because $x_{p}=y_{p}$ for all $p\leq m$ in this case). Thus,
(\ref{pf.lem.major.RH.OPRH-maj.ineq-to-prove}) is proved in Case 1.

In Case 2, we have%
\begin{align*}
y_{1}+y_{2}+\cdots+y_{m}  &  =x_{1}+x_{2}+\cdots+x_{j-1}+v+x_{j+1}%
+x_{j+2}+\cdots+x_{m}\\
&  =\left(  x_{1}+x_{2}+\cdots+x_{m}\right)  +\underbrace{v-x_{j}%
}_{\substack{\leq0\\\text{(since }v\in\left[  x_{i},x_{j}\right]  \text{)}}}\\
&  \leq x_{1}+x_{2}+\cdots+x_{m}.
\end{align*}
Thus, (\ref{pf.lem.major.RH.OPRH-maj.ineq-to-prove}) is proved in Case 2.

In Case 3, we have%
\begin{align*}
&  y_{1}+y_{2}+\cdots+y_{m}\\
&  =x_{1}+x_{2}+\cdots+x_{j-1}+v+x_{j+1}+x_{j+2}+\cdots+x_{i-1}+u+x_{i+1}%
+x_{i+2}+\cdots+x_{m}\\
&  =\left(  x_{1}+x_{2}+\cdots+x_{m}\right)  +\underbrace{\left(
u-x_{i}\right)  +\left(  v-x_{j}\right)  }_{\substack{=0\\\text{(since
}u+v=x_{i}+x_{j}\text{)}}}\\
&  =x_{1}+x_{2}+\cdots+x_{m}.
\end{align*}
Thus, (\ref{pf.lem.major.RH.OPRH-maj.ineq-to-prove}) is proved in Case 3.

So we have proved (\ref{pf.lem.major.RH.OPRH-maj.ineq-to-prove}) in all cases.
As we said, this concludes our proof of $x\succcurlyeq y$.

We are not completely done yet: We have only proved Lemma
\ref{lem.major.RH.OPRH-maj} in the case when $y$ is obtained from $x$ by a
\textbf{single} OPRH move.

Now, let us prove the general case: Assume that $y$ is obtained from $x$ by a
(finite) sequence of OPRH moves. That is, there is a finite sequence
$x_{\left[  0\right]  },x_{\left[  1\right]  },\ldots,x_{\left[  m\right]  }$
of vectors in $\mathbb{R}^{n}$ such that $x_{\left[  0\right]  }=x$ and
$x_{\left[  m\right]  }=y$ and%
\[
x_{\left[  0\right]  }\overset{\text{OPRH}}{\longrightarrow}x_{\left[
1\right]  }\overset{\text{OPRH}}{\longrightarrow}\cdots\overset{\text{OPRH}%
}{\longrightarrow}x_{\left[  m\right]  }.
\]
Then, by the special case we have already proved, we conclude that%
\[
x_{\left[  0\right]  }\succcurlyeq x_{\left[  1\right]  }\succcurlyeq
\cdots\succcurlyeq x_{\left[  m\right]  }.
\]
Hence, $x_{\left[  0\right]  }\succcurlyeq x_{\left[  m\right]  }$ (since the
relation $\succcurlyeq$ is reflexive and transitive). In other words,
$x\succcurlyeq y$ (since $x_{\left[  0\right]  }=x$ and $x_{\left[  m\right]
}=y$). This completes the proof of Lemma \ref{lem.major.RH.OPRH-maj}.
\end{proof}

We are now ready to state one version of the \textquotedblleft moving the
entries closer together\textquotedblright\ intuition for majorization:

\begin{theorem}
[RH criterion for majorization]\label{thm.major.RH.OPRH-crit}Let
$x,y\in\mathbb{R}^{n}$ be two weakly decreasing column vectors. Then,
$x\succcurlyeq y$ if and only if $y$ can be obtained from $x$ by a (finite)
sequence of OPRH moves.
\end{theorem}

\begin{example}
\textbf{(a)} We have $\left(  4,1,1\right)  ^{T}\succcurlyeq\left(
2,2,2\right)  ^{T}$, and indeed $\left(  2,2,2\right)  ^{T}$ can be obtained
from $\left(  4,1,1\right)  ^{T}$ by OPRH moves as follows:%
\[
\left(  4,1,1\right)  ^{T}\overset{\text{OPRH}}{\longrightarrow}\left(
3,2,1\right)  ^{T}\overset{\text{OPRH}}{\longrightarrow}\left(  2,2,2\right)
^{T}.
\]


\textbf{(b)} We have $\left(  7,5,2,0\right)  ^{T}\succcurlyeq\left(
4,4,3,3\right)  ^{T}$, and indeed $\left(  4,4,3,3\right)  ^{T}$ can be
obtained from $\left(  7,5,2,0\right)  ^{T}$ by OPRH moves as follows:%
\[
\left(  7,5,2,0\right)  ^{T}\overset{\text{OPRH}}{\longrightarrow}\left(
6,6,2,0\right)  ^{T}\overset{\text{OPRH}}{\longrightarrow}\left(
6,5,3,0\right)  ^{T}\overset{\text{OPRH}}{\longrightarrow}\left(
6,4,3,1\right)  ^{T}\overset{\text{OPRH}}{\longrightarrow}\left(
4,4,3,3\right)  ^{T}.
\]
Here is another way to do this:%
\[
\left(  7,5,2,0\right)  ^{T}\overset{\text{OPRH}}{\longrightarrow}\left(
7,4,3,0\right)  ^{T}\overset{\text{OPRH}}{\longrightarrow}\left(
4,4,3,3\right)  ^{T}.
\]

\end{example}

\begin{proof}
[Proof of Theorem \ref{thm.major.RH.OPRH-crit}.]$\Longleftarrow:$ This follows
from Lemma \ref{lem.major.RH.OPRH-maj}. \medskip

$\Longrightarrow:$ Let $x\succcurlyeq y$. We must show that $y$ can be
obtained from $x$ by a finite sequence of OPRH moves.

If $x=y$, then this is clear (just take the empty sequence). So we WLOG assume
that $x\neq y$. We \textbf{claim} now that there is a further weakly
decreasing vector $z\in\mathbb{R}^{n}$ such that

\begin{enumerate}
\item we have $x\overset{\text{OPRH}}{\longrightarrow}z$;

\item we have $z\succcurlyeq y$;

\item the vector $z$ has more entries in common with $y$ than $x$ does; in
other words, we have%
\begin{equation}
\left\vert \left\{  i\in\left[  n\right]  \ \mid\ z_{i}=y_{i}\right\}
\right\vert >\left\vert \left\{  i\in\left[  n\right]  \ \mid\ x_{i}%
=y_{i}\right\}  \right\vert . \label{pf.thm.major.RH.OPRH-crit.s3}%
\end{equation}

\end{enumerate}

In other words, we claim that by making a strategic OPRH move starting at $x$,
we can reach a vector $z$ that still majorizes $y$ but has at least one more
entry in common with $y$ than $x$ does. If we can prove this claim, then we
will automatically obtain a recursive procedure to transform $x$ into $y$ by a
sequence of OPRH moves. (And in fact, this procedure will use at most $n$
moves, because each move makes the vector agree with $y$ in at least one more
position.) \medskip

So let us prove our claim.

Since $x$ is weakly decreasing, we have $x=x^{\downarrow}$. Similarly,
$y=y^{\downarrow}$. Thus, from $x\succcurlyeq y$, we obtain%
\begin{equation}
x_{1}+x_{2}+\cdots+x_{m}\geq y_{1}+y_{2}+\cdots+y_{m}
\label{pf.thm.major.RH.OPRH-crit.1}%
\end{equation}
for all $m\in\left[  n\right]  $, as well as
\begin{equation}
x_{1}+x_{2}+\cdots+x_{n}=y_{1}+y_{2}+\cdots+y_{n}.
\label{pf.thm.major.RH.OPRH-crit.2}%
\end{equation}


We define a \emph{turn} to be a pair $\left(  a,b\right)  $ of two elements of
$\left[  n\right]  $ satisfying
\[
x_{a}>y_{a}\ \ \ \ \ \ \ \ \ \ \text{and}\ \ \ \ \ \ \ \ \ \ x_{b}%
<y_{b}\ \ \ \ \ \ \ \ \ \ \text{and}\ \ \ \ \ \ \ \ \ \ a<b.
\]
We claim that there exists at least one turn.

[\textit{Proof:} We have $x\neq y$. Thus, there exists some $a\in\left[
n\right]  $ such that $x_{a}\neq y_{a}$. Consider the smallest such $a$. Thus,
$x_{i}=y_{i}$ for each $i<a$. However, (\ref{pf.thm.major.RH.OPRH-crit.1})
(applied to $m=a$) yields $x_{1}+x_{2}+\cdots+x_{a}\geq y_{1}+y_{2}%
+\cdots+y_{a}$. Thus, $x_{a}\geq y_{a}$ (since $x_{i}=y_{i}$ for each $i<a$).
Hence, $x_{a}>y_{a}$ (since $x_{a}\neq y_{a}$). Therefore, $x_{1}+x_{2}%
+\cdots+x_{a}>y_{1}+y_{2}+\cdots+y_{a}$.

Next, let us pick the smallest $b\in\left\{  a+1,a+2,\ldots,n\right\}  $ such
that $x_{1}+x_{2}+\cdots+x_{b}=y_{1}+y_{2}+\cdots+y_{b}$. (This exists,
because (\ref{pf.thm.major.RH.OPRH-crit.2}) shows that $n$ is such a $b$.)
Clearly, $a<b$ (since $b\in\left\{  a+1,a+2,\ldots,n\right\}  $).

We shall now show that $x_{b}<y_{b}$. Indeed, assume the contrary. Thus,
$x_{b}\geq y_{b}$. Subtracting this inequality from the equality $x_{1}%
+x_{2}+\cdots+x_{b}=y_{1}+y_{2}+\cdots+y_{b}$, we obtain $x_{1}+x_{2}%
+\cdots+x_{b-1}\leq y_{1}+y_{2}+\cdots+y_{b-1}$. However, we have $x_{1}%
+x_{2}+\cdots+x_{b-1}\geq y_{1}+y_{2}+\cdots+y_{b-1}$ (by
(\ref{pf.thm.major.RH.OPRH-crit.1})). Combining these two inequalities, we
obtain $x_{1}+x_{2}+\cdots+x_{b-1}=y_{1}+y_{2}+\cdots+y_{b-1}$. However,
recall that $b$ was defined to be the \textbf{smallest} element of $\left\{
a+1,a+2,\ldots,n\right\}  $ such that $x_{1}+x_{2}+\cdots+x_{b}=y_{1}%
+y_{2}+\cdots+y_{b}$. This contradicts the equality $x_{1}+x_{2}%
+\cdots+x_{b-1}=y_{1}+y_{2}+\cdots+y_{b-1}$ (after all, $b-1$ is smaller than
$b$) unless $b-1$ is not an element of $\left\{  a+1,a+2,\ldots,n\right\}  $.
Thus, $b-1$ must not be an element of $\left\{  a+1,a+2,\ldots,n\right\}  $.
So we have $b\in\left\{  a+1,a+2,\ldots,n\right\}  $ but $b-1\notin\left\{
a+1,a+2,\ldots,n\right\}  $. Therefore, $b=a+1$, so that $b-1=a$. Thus, the
equality $x_{1}+x_{2}+\cdots+x_{b-1}=y_{1}+y_{2}+\cdots+y_{b-1}$ rewrites as
$x_{1}+x_{2}+\cdots+x_{a}=y_{1}+y_{2}+\cdots+y_{a}$. But this contradicts the
inequality $x_{1}+x_{2}+\cdots+x_{a}>y_{1}+y_{2}+\cdots+y_{a}$ proved above.
Thus, our proof of $x_{b}<y_{b}$ is complete.

We thus conclude that $\left(  a,b\right)  $ is a turn (since $a<b$ and
$x_{a}>y_{a}$ and $x_{b}<y_{b}$). This proves that there exists at least one
turn.] \medskip

The \emph{width} of a turn $\left(  a,b\right)  $ shall denote the positive
integer $b-a$. (This is a positive integer, since $a<b$ in a turn $\left(
a,b\right)  $.)

Now, let us pick a turn $\left(  a,b\right)  $ with the \textbf{smallest
possible width}. Then, we have%
\[
x_{a}>y_{a}\ \ \ \ \ \ \ \ \ \ \text{and}\ \ \ \ \ \ \ \ \ \ x_{b}%
<y_{b}\ \ \ \ \ \ \ \ \ \ \text{and}\ \ \ \ \ \ \ \ \ \ a<b.
\]
Moreover, for each $j\in\left\{  a+1,a+2,\ldots,b-1\right\}  $, we have
$x_{j}=y_{j}$ (because if we had $x_{j}<y_{j}$, then $\left(  a,j\right)  $
would be a turn of smaller width than $\left(  a,b\right)  $, and if we had
$x_{j}>y_{j}$, then $\left(  j,b\right)  $ would be a turn of smaller width
than $\left(  a,b\right)  $). Thus, we have
\begin{align*}
x_{a}  &  >y_{a},\\
x_{j}  &  =y_{j}\ \ \ \ \ \ \ \ \ \ \text{for all }j\in\left\{  a+1,a+2,\ldots
,b-1\right\}  ,\\
x_{b}  &  <y_{b}.
\end{align*}
Since $y$ is weakly decreasing (so that $y_{a}\geq y_{a+1}\geq\cdots\geq
y_{b}$), we thus obtain the following chain of inequalities:%
\[
x_{a}>y_{a}\geq\left(  \text{all of the }x_{j}\text{ and }y_{j}\text{ with
}j\in\left\{  a+1,a+2,\ldots,b-1\right\}  \right)  \geq y_{b}>x_{b}.
\]
(If there are no $j\in\left\{  a+1,a+2,\ldots,b-1\right\}  $, then this is
supposed to read $x_{a}>y_{a}\geq y_{b}>x_{b}$.) This shows, in particular,
that $y_{a}$ and $y_{b}$ lie in the open interval $\left(  x_{a},x_{b}\right)
$.

Now, we make an RH move (on the vector $x$) that \textquotedblleft squeezes
$x_{a}$ and $x_{b}$ together\textquotedblright\ until either $x_{a}$ reaches
$y_{a}$ or $x_{b}$ reaches $y_{b}$ (whatever happens first). In formal terms,
this means that we do the following:

\begin{itemize}
\item If $x_{a}-y_{a}\leq y_{b}-x_{b}$, then we replace $x_{a}$ and $x_{b}$ by
$y_{a}$ and $x_{a}+x_{b}-y_{a}$.

\item If $x_{a}-y_{a}\geq y_{b}-x_{b}$, then we replace $x_{a}$ and $x_{b}$ by
$x_{a}+x_{b}-y_{b}$ and $y_{b}$.
\end{itemize}

\noindent(The two cases overlap, but this is not a problem, because if
$x_{a}-y_{a}=y_{b}-x_{b}$, then both outcomes are identical.)

Let $z\in\mathbb{R}^{n}$ be the $n$-tuple resulting from this operation. We
claim that $z$ is weakly decreasing and satisfies the three requirements 1, 2,
3 above:

\begin{enumerate}
\item we have $x\overset{\text{OPRH}}{\longrightarrow}z$;

\item we have $z\succcurlyeq y$;

\item the vector $z$ has more entries in common with $y$ than $x$ does; in
other words, we have%
\[
\left\vert \left\{  i\in\left[  n\right]  \ \mid\ z_{i}=y_{i}\right\}
\right\vert >\left\vert \left\{  i\in\left[  n\right]  \ \mid\ x_{i}%
=y_{i}\right\}  \right\vert .
\]

\end{enumerate}

Indeed, let us prove that these three requirements hold. Requirement 3 is the
easiest one to verify: The only entries of $z$ that differ from the respective
entries of $x$ are the two entries $z_{a}$ and $z_{b}$; among these two
entries, at least one agrees with the corresponding entry of $y$ (because we
have either $z_{a}=y_{a}$ or $z_{b}=y_{b}$), whereas none of the corresponding
entries of $x$ agrees with the corresponding entry of $y$ (since $x_{a}>y_{a}$
and $x_{b}<y_{b}$). Thus, going from $x$ to $z$, we have increased the number
of entries that agree with the corresponding entries of $y$ by at least $1$.
Requirement 3 is therefore satisfied.

Let us next check Requirement 1. (The reader should draw a picture of the
numbers $x_{j}$ and $y_{j}$ for $j\in\left\{  a,a+1,\ldots,b\right\}  $ as
points on the real axis. As we recall, $z_{a}$ and $z_{b}$ are obtained by
moving $x_{a}$ and $x_{b}$ closer together (preserving their sum) until either
$x_{a}$ hits $y_{a}$ or $x_{b}$ hits $y_{b}$ (whatever happens first). This
picture should render some of the verifications below trivial.)

The definition of $z$ shows that $z_{a}\geq y_{a}$%
\ \ \ \ \footnote{\textit{Proof.} The definition of $z_{a}$ shows that
$z_{a}=y_{a}$ in the case when $x_{a}-y_{a}\leq y_{b}-x_{b}$, and that
$z_{a}=x_{a}+x_{b}-y_{b}$ in the case when $x_{a}-y_{a}\geq y_{b}-x_{b}$. In
the former case, the inequality $z_{a}\geq y_{a}$ is obvious (and is, in fact,
an equality). Hence, it remains to prove $z_{a}\geq y_{a}$ in the latter case.
\par
So let us assume that we are in the latter case -- i.e., that we have
$x_{a}-y_{a}\geq y_{b}-x_{b}$. Thus, $x_{a}\geq y_{b}-x_{b}+y_{a}$. Now,%
\[
z_{a}=\underbrace{x_{a}}_{\geq y_{b}-x_{b}+y_{a}}+x_{b}-y_{b}\geq y_{b}%
-x_{b}+y_{a}+x_{b}-y_{b}=y_{a},
\]
qed.} and $z_{b}\leq y_{b}$\ \ \ \ \footnote{\textit{Proof.} The definition of
$z_{b}$ shows that $z_{b}=x_{a}+x_{b}-y_{a}$ in the case when $x_{a}-y_{a}\leq
y_{b}-x_{b}$, and that $z_{b}=y_{b}$ in the case when $x_{a}-y_{a}\geq
y_{b}-x_{b}$. In the latter case, the inequality $z_{b}\leq y_{b}$ is obvious
(and is, in fact, an equality). Hence, it remains to prove $z_{b}\leq y_{b}$
in the former case.
\par
So let us assume that we are in the former case -- i.e., that we have
$x_{a}-y_{a}\leq y_{b}-x_{b}$. Thus, $x_{a}\leq y_{b}-x_{b}+y_{a}$. Now,%
\[
z_{b}=\underbrace{x_{a}}_{\leq y_{b}-x_{b}+y_{a}}+x_{b}-y_{a}\leq y_{b}%
-x_{b}+y_{a}+x_{b}-y_{a}=y_{b},
\]
qed.}. Moreover, the numbers $z_{a}$ and $z_{b}$ lie in the interval $\left[
x_{b},x_{a}\right]  $\ \ \ \ \footnote{\textit{Proof.} The definition of $z$
shows that we have either $\left(  z_{a}=y_{a}\text{ and }z_{b}=x_{a}%
+x_{b}-y_{a}\right)  $ or $\left(  z_{a}=x_{a}+x_{b}-y_{b}\text{ and }%
z_{b}=y_{b}\right)  $. Hence, we must show that the numbers $y_{a}$,
$x_{a}+x_{b}-y_{a}$, $x_{a}+x_{b}-y_{b}$ and $y_{b}$ all lie in the interval
$\left[  x_{b},x_{a}\right]  $. But this follows easily from $x_{a}>y_{a}\geq
y_{b}>x_{b}$.}, and we have $z_{a}+z_{b}=x_{a}+x_{b}$%
\ \ \ \ \footnote{\textit{Proof.} The definition of $z$ shows that we have
either $\left(  z_{a}=y_{a}\text{ and }z_{b}=x_{a}+x_{b}-y_{a}\right)  $ or
$\left(  z_{a}=x_{a}+x_{b}-y_{b}\text{ and }z_{b}=y_{b}\right)  $. Hence, we
must show that $y_{a}+\left(  x_{a}+x_{b}-y_{a}\right)  =x_{a}+x_{b}$ and
$\left(  x_{a}+x_{b}-y_{b}\right)  +y_{b}=x_{a}+x_{b}$. But both of these
equalities are clearly true.}. Thus, the operation that transformed $x$ into
$z$ was an RH move (with $b$ and $a$ playing the roles of $i$ and $j$ from
Definition \ref{def.major.RH}). Therefore, $x\overset{\text{RH}%
}{\longrightarrow}z$. It remains to prove that this RH move is
order-preserving, i.e., that $z$ is weakly decreasing. Since the only entries
of $x$ that changed in our RH move were $x_{a}$ and $x_{b}$ (and since we know
that $x$ is weakly decreasing), we only need to verify the inequalities
\begin{align*}
x_{a-1}  &  \geq z_{a}\ \ \ \ \ \ \ \ \ \ \left(  \text{if }a>1\right)
\ \ \ \ \ \ \ \ \ \ \text{and}\\
z_{a}  &  \geq x_{a+1}\ \ \ \ \ \ \ \ \ \ \left(  \text{if }a+1\neq b\right)
\ \ \ \ \ \ \ \ \ \ \text{and}\\
z_{a}  &  \geq z_{b}\ \ \ \ \ \ \ \ \ \ \left(  \text{if }a+1=b\right)
\ \ \ \ \ \ \ \ \ \ \text{and}\\
x_{b-1}  &  \geq z_{b}\ \ \ \ \ \ \ \ \ \ \left(  \text{if }a+1\neq b\right)
\ \ \ \ \ \ \ \ \ \ \text{and}\\
z_{b}  &  \geq x_{b+1}\ \ \ \ \ \ \ \ \ \ \left(  \text{if }b\neq n\right)  .
\end{align*}
Fortunately, these inequalities all follow easily from the facts that we know
(viz., from the chain of inequalities%
\[
x_{a}>y_{a}\geq\left(  \text{all of the }x_{j}\text{ and }y_{j}\text{ with
}j\in\left\{  a+1,a+2,\ldots,b-1\right\}  \right)  \geq y_{b}>x_{b}%
\]
and from the inequalities $z_{a}\geq y_{a}$ and $z_{b}\leq y_{b}$ and the fact
that $z_{a}$ and $z_{b}$ lie in the interval $\left[  x_{b},x_{a}\right]  $):
The inequality $x_{a-1}\geq z_{a}$ (if $a>1$) follows from $x_{a-1}\geq
x_{a}\geq z_{a}$ (since $z_{a}\in\left[  x_{b},x_{a}\right]  $). The
inequality $z_{a}\geq x_{a+1}$ (if $a+1\neq b$) follows from
\[
z_{a}\geq y_{a}\geq\left(  \text{all of the }x_{j}\text{ and }y_{j}\text{ with
}j\in\left\{  a+1,a+2,\ldots,b-1\right\}  \right)
\]
(since $x_{a+1}$ is one of the latter $x_{j}$). Furthermore, $z_{a}\geq z_{b}$
follows from $z_{a}\geq y_{a}\geq y_{b}\geq z_{b}$ (since $z_{b}\leq y_{b}$).
Next, $x_{b-1}\geq z_{b}$ (if $a+1\neq b$) follows from
\[
\left(  \text{all of the }x_{j}\text{ and }y_{j}\text{ with }j\in\left\{
a+1,a+2,\ldots,b-1\right\}  \right)  \geq y_{b}\geq z_{b}%
\]
(since $x_{b-1}$ is one of those former $x_{j}$). Finally, $z_{b}\geq x_{b+1}$
follows by combining $z_{b}\geq x_{b}$ (this is because $z_{b}$ lies in the
interval $\left[  x_{b},x_{a}\right]  $) and $x_{b}\geq x_{b+1}$. Thus, we
have checked all the necessary inequalities to ensure that $z$ is weakly
decreasing. Thus, requirement 1 holds.

Finally, we need to verify requirement 2. In other words, we need to show that
$z\succcurlyeq y$. Since $z$ and $y$ are weakly decreasing, this means that we
need to verify the inequalities%
\begin{equation}
z_{1}+z_{2}+\cdots+z_{m}\geq y_{1}+y_{2}+\cdots+y_{m}
\label{pf.thm.major.RH.OPRH-crit.req2-ineq}%
\end{equation}
for all $m\in\left[  n\right]  $, as well as the equality
\begin{equation}
z_{1}+z_{2}+\cdots+z_{n}=y_{1}+y_{2}+\cdots+y_{n}.
\label{pf.thm.major.RH.OPRH-crit.req2-eq}%
\end{equation}


The equality (\ref{pf.thm.major.RH.OPRH-crit.req2-eq}) is easy to verify:
Since $x\overset{\text{RH}}{\longrightarrow}z$ (and since the sum of the
entries of a vector does not change when we make an RH move), we have%
\[
z_{1}+z_{2}+\cdots+z_{n}=x_{1}+x_{2}+\cdots+x_{n}=y_{1}+y_{2}+\cdots+y_{n}%
\]
(since $x\succcurlyeq y$). Thus, it remains to prove the inequality
(\ref{pf.thm.major.RH.OPRH-crit.req2-ineq}). So let us fix $m\in\left[
n\right]  $. We must prove (\ref{pf.thm.major.RH.OPRH-crit.req2-ineq}). We are
in one of the following three cases:

\textit{Case 1:} We have $m<a$.

\textit{Case 2:} We have $a\leq m<b$.

\textit{Case 3:} We have $m\geq b$.

Let us first consider Case 1. In this case, we have $m<a$. Hence, $z_{i}%
=x_{i}$ for each $i\leq m$. Thus,%
\[
z_{1}+z_{2}+\cdots+z_{m}=x_{1}+x_{2}+\cdots+x_{m}\geq y_{1}+y_{2}+\cdots+y_{m}%
\]
(since $x\succcurlyeq y$). This proves
(\ref{pf.thm.major.RH.OPRH-crit.req2-ineq}) in Case 1.

Let us next consider Case 2. In this case, we have $a\leq m<b$. Hence,%
\begin{align*}
z_{1}+z_{2}+\cdots+z_{m}  &  =\underbrace{x_{1}+x_{2}+\cdots+x_{a-1}%
}_{\substack{\geq y_{1}+y_{2}+\cdots+y_{a-1}\\\text{(since }x\succcurlyeq
y\text{)}}}+\underbrace{z_{a}}_{\geq y_{a}}+\underbrace{x_{a+1}+x_{a+2}%
+\cdots+x_{m}}_{\substack{=y_{a+1}+y_{a+2}+\cdots+y_{m}\\\text{(since }%
x_{j}=y_{j}\text{ for all }j\in\left\{  a+1,a+2,\ldots,b-1\right\}  \text{)}%
}}\\
&  \geq y_{1}+y_{2}+\cdots+y_{a-1}+y_{a}+y_{a+1}+y_{a+2}+\cdots+y_{m}\\
&  =y_{1}+y_{2}+\cdots+y_{m}.
\end{align*}
This proves (\ref{pf.thm.major.RH.OPRH-crit.req2-ineq}) in Case 2.

Finally, let us consider Case 3. In this case, we have $m\geq b$. Hence,%
\begin{align*}
&  z_{1}+z_{2}+\cdots+z_{m}\\
&  =x_{1}+x_{2}+\cdots+x_{a-1}+z_{a}+x_{a+1}+x_{a+2}+\cdots+x_{b-1}%
+z_{b}+x_{b+1}+x_{b+2}+\cdots+x_{m}\\
&  =\left(  x_{1}+x_{2}+\cdots+x_{m}\right)  +\underbrace{\left(  z_{a}%
-x_{a}\right)  +\left(  z_{b}-x_{b}\right)  }_{\substack{=0\\\text{(since
}z_{a}+z_{b}=x_{a}+x_{b}\text{)}}}\\
&  =x_{1}+x_{2}+\cdots+x_{m}\geq y_{1}+y_{2}+\cdots+y_{m}%
\ \ \ \ \ \ \ \ \ \ \left(  \text{since }x\succcurlyeq y\right)  .
\end{align*}
This proves (\ref{pf.thm.major.RH.OPRH-crit.req2-ineq}) in Case 3.

We have now proved (\ref{pf.thm.major.RH.OPRH-crit.req2-ineq}) in all three
Cases 1, 2 and 3. Hence, (\ref{pf.thm.major.RH.OPRH-crit.req2-ineq}) always
holds. Thus, we have verified Requirement 2. Now, all three requirements 1, 2
and 3 are satisfied. As we explained, this means that our vector $z$ fits our
bill, and this completes the proof of Theorem \ref{thm.major.RH.OPRH-crit}.
\end{proof}

\begin{exercise}
\label{exe.major.RH.OPRH-n-1}\fbox{2} Let $x,y\in\mathbb{R}^{n}$ be two weakly
decreasing column vectors. Prove that $x\succcurlyeq y$ if and only if $y$ can
be obtained from $x$ by a sequence of at most $n-1$ OPRH moves.
\end{exercise}

Theorem \ref{thm.major.RH.OPRH-crit} formalizes our intuition about
majorization as \textquotedblleft moving entries closer
together\textquotedblright\ for weakly decreasing vectors. There is a version
for arbitrary vectors as well:

\begin{theorem}
[RH criterion for majorization, non-decreasing form]%
\label{thm.major.RH.RH-crit}Let $x,y\in\mathbb{R}^{n}$ be two column vectors.
Then, $x\succcurlyeq y$ if and only if $y$ can be obtained from $x$ by a
(finite) sequence of RH moves.
\end{theorem}

The proof of this theorem will be an exercise, but we delay this exercise
until after Theorem \ref{thm.major.abs-crit}, since the latter theorem
provides a good tool for the proof.

\begin{exercise}
\fbox{7} Let $x,y\in\mathbb{R}^{n}$ be two column vectors such that
$x\succcurlyeq y$. Prove that there exists a real symmetric matrix
$A\in\mathbb{R}^{n\times n}$ with diagonal entries $y_{1},y_{2},\ldots,y_{n}$
and eigenvalues $x_{1},x_{2},\ldots,x_{n}$.

[\textbf{Remark:} This can be viewed as a converse to Corollary
\ref{cor.interlacing.maj-2} (but is in fact even stronger, since $A$ is not
just Hermitian but real symmetric).]
\end{exercise}

\subsubsection{Karamata's inequality}

Now, what can we do with majorization? Probably the most important property of
majorizing pairs of vectors is the so-called \emph{Karamata inequality}. To
state it, we recall the concept of a convex function:

\begin{definition}
\label{def.major.convex}Let $I\subseteq\mathbb{R}$ be an interval. Let
$f:I\rightarrow\mathbb{R}$ be a function. We say that $f$ is \emph{convex} if
and only if it has the following property: For any $a,b\in I$ and any
$\lambda\in\left[  0,1\right]  $, we have%
\[
\lambda f\left(  a\right)  +\left(  1-\lambda\right)  f\left(  b\right)  \geq
f\left(  \lambda a+\left(  1-\lambda\right)  b\right)  .
\]

\end{definition}

This property is best conceptualized using the plot of the function: A
function $f:I\rightarrow\mathbb{R}$ is convex if and only if, for any two
points $\left(  a,f\left(  a\right)  \right)  $ and $\left(  b,f\left(
b\right)  \right)  $ on the plot of $f$, the entire segment connecting these
two points lies weakly above (i.e., on or above) the plot of $f$. (In fact,
the segment connecting these two points can be parametrized as
\[
\left\{  \left(  \lambda a+\left(  1-\lambda\right)  b,\ \lambda f\left(
a\right)  +\left(  1-\lambda\right)  f\left(  b\right)  \right)
\ \mid\ \lambda\in\left[  0,1\right]  \right\}  ,
\]
so a typical point on this segment has the form
\[
\left(  \lambda a+\left(  1-\lambda\right)  b,\ \lambda f\left(  a\right)
+\left(  1-\lambda\right)  f\left(  b\right)  \right)
\]
for some $\lambda\in\left[  0,1\right]  $, whereas the corresponding point on
the plot of $f$ is
\[
\left(  \lambda a+\left(  1-\lambda\right)  b,\ f\left(  \lambda a+\left(
1-\lambda\right)  b\right)  \right)  .
\]
Thus, the former point lies weakly above the latter point if and only if
$\lambda f\left(  a\right)  +\left(  1-\lambda\right)  f\left(  b\right)  \geq
f\left(  \lambda a+\left(  1-\lambda\right)  b\right)  $ holds.)

Here are some examples of convex functions:

\begin{itemize}
\item $f\left(  t\right)  =t^{n}$ defines a convex function $f:\mathbb{R}%
\rightarrow\mathbb{R}$ whenever $n\in\mathbb{N}$ is even.

\item $f\left(  t\right)  =t^{n}$ defines a convex function $f:\mathbb{R}%
_{+}\rightarrow\mathbb{R}$ whenever $n\in\mathbb{R}\setminus\left(
0,1\right)  $. Otherwise, it defines a concave function\footnote{A function
$f:I\rightarrow\mathbb{R}$ is said to be \emph{concave} if $-f$ is convex.}.

\item $f\left(  t\right)  =\sin t$ defines a concave function $f:\left[
0,\pi\right]  \rightarrow\mathbb{R}$ and a convex function $f:\left[  \pi
,2\pi\right]  \rightarrow\mathbb{R}$.
\end{itemize}

Before we state Karamata's inequality, let us recall three fundamental
properties of convex functions:

\begin{theorem}
[second derivative test]Let $I\subseteq\mathbb{R}$ be an interval. Let
$f:I\rightarrow\mathbb{R}$ be a twice differentiable function. Then, $f$ is
convex if and only if each $x\in I$ satisfies $f^{\prime\prime}\left(
x\right)  \geq0$.
\end{theorem}

Note that there are convex functions that are not twice differentiable. For
instance, the absolute value function $f:\mathbb{R}\rightarrow\mathbb{R}$
(given by $f\left(  z\right)  =\left\vert z\right\vert $ for each
$z\in\mathbb{R}$) is convex. This cannot be proved by the second derivative
test, but it is easy to check using the triangle inequality.

A classical property of convex functions is \emph{Jensen's inequality}. Let us
give it in two of its forms -- a simple unweighted and a more general weighted one:

\begin{theorem}
[Jensen's inequality]\label{thm.major.jensen}Let $I\subseteq\mathbb{R}$ be an
interval. Let $f:I\rightarrow\mathbb{R}$ be a convex function. Let
$x_{1},x_{2},\ldots,x_{n}\in I$. Let $m=\dfrac{x_{1}+x_{2}+\cdots+x_{n}}{n}$.
Then,%
\[
f\left(  x_{1}\right)  +f\left(  x_{2}\right)  +\cdots+f\left(  x_{n}\right)
\geq nf\left(  m\right)  .
\]

\end{theorem}

\begin{theorem}
[weighted Jensen's inequality]\label{thm.major.jensen-wt}Let $I\subseteq
\mathbb{R}$ be an interval. Let $f:I\rightarrow\mathbb{R}$ be a convex
function. Let $x_{1},x_{2},\ldots,x_{n}\in I$. Let $\lambda_{1},\lambda
_{2},\ldots,\lambda_{n}$ be $n$ nonnegative reals satisfying $\lambda
_{1}+\lambda_{2}+\cdots+\lambda_{n}=1$. Then,%
\[
\lambda_{1}f\left(  x_{1}\right)  +\lambda_{2}f\left(  x_{2}\right)
+\cdots+\lambda_{n}f\left(  x_{n}\right)  \geq f\left(  \lambda_{1}%
x_{1}+\lambda_{2}x_{2}+\cdots+\lambda_{n}x_{n}\right)  .
\]

\end{theorem}

The easiest way to prove these two theorems is to first prove Theorem
\ref{thm.major.jensen-wt} by induction on $n$ (this is commonly done in
textbooks on probability theory, where this inequality is used quite
often\footnote{Or see \url{https://en.wikipedia.org/wiki/Jensen's_inequality}
or
\url{https://www.ucd.ie/mathstat/t4media/convex-sets-and-jensen-inequalities-mathstat.pdf}
.}) and then obtain Theorem \ref{thm.major.jensen} from it by setting
$\lambda_{1}=\lambda_{2}=\cdots=\lambda_{n}=\dfrac{1}{n}$.

Note that both Jensen's inequalities are highly useful for proving various
kinds of inequalities, even ones in which there is no convex function easily
visible. See \cite[Chapter 4]{Hung07} for some such applications.

Jensen's inequality can be interpreted as saying that a sum of values of a
convex function $f$ at several points $x_{1},x_{2},\ldots,x_{n}$ becomes
smaller (or at least not larger) if all these points are replaced by their
average $m=\dfrac{x_{1}+x_{2}+\cdots+x_{n}}{n}$. Karamata's inequality
generalizes this by saying that we don't need to replace all the points by
their average right away, but rather it suffices to \textquotedblleft move
them closer together\textquotedblright\ (not necessarily going all the way to
the average). Here, \textquotedblleft moving them closer
together\textquotedblright\ is formalized using majorization:

\begin{theorem}
[Karamata's inequality]\label{thm.major.karamata}Let $I\subseteq\mathbb{R}$ be
an interval. Let $f:I\rightarrow\mathbb{R}$ be a convex function. Let $x\in
I^{n}$ and $y\in I^{n}$ be two vectors such that $x\succcurlyeq y$. Then,%
\[
f\left(  x_{1}\right)  +f\left(  x_{2}\right)  +\cdots+f\left(  x_{n}\right)
\geq f\left(  y_{1}\right)  +f\left(  y_{2}\right)  +\cdots+f\left(
y_{n}\right)  .
\]

\end{theorem}

Karamata's inequality has many applications (see, e.g., \cite[\S 2]{KDLM05},
which incidentally also gives a proof of Karamata's inequality different from
the ones we shall give below). In particular, Jensen's inequality follows from
Karamata's inequality, since Exercise \ref{exe.major.average} says that
$\left(  x_{1},x_{2},\ldots,x_{n}\right)  ^{T}\succcurlyeq\left(
m,m,\ldots,m\right)  ^{T}$.

The weighted Jensen's inequality can, incidentally, be derived from a weighted
Karamata's inequality (see Exercise \ref{exe.major.karamata-wt} below).

Let us now prove Karamata's inequality:

\begin{proof}
[Proof of Theorem \ref{thm.major.karamata}.]It is enough to prove the claim in
the case when $x$ and $y$ are weakly decreasing (because permuting the entries
of any of $x$ and $y$ does not change anything).

Furthermore, it is enough to prove the claim in the case when
$x\overset{\text{OPRH}}{\longrightarrow}y$ (this means that $y$ is obtained
from $x$ by a single OPRH move). Indeed, if we have shown this, then it will
mean that the sum $f\left(  x_{1}\right)  +f\left(  x_{2}\right)
+\cdots+f\left(  x_{n}\right)  $ decreases (weakly) every time we apply an
OPRH move to the vector $x$. Therefore, if $y$ is obtained from $x$ by a
(finite) sequence of OPRH moves, then $f\left(  x_{1}\right)  +f\left(
x_{2}\right)  +\cdots+f\left(  x_{n}\right)  \geq f\left(  y_{1}\right)
+f\left(  y_{2}\right)  +\cdots+f\left(  y_{n}\right)  $. Hence, if if
$x\succcurlyeq y$, then $f\left(  x_{1}\right)  +f\left(  x_{2}\right)
+\cdots+f\left(  x_{n}\right)  \geq f\left(  y_{1}\right)  +f\left(
y_{2}\right)  +\cdots+f\left(  y_{n}\right)  $ (since Theorem
\ref{thm.major.RH.OPRH-crit} shows that $y$ can be obtained from $x$ by a
(finite) sequence of OPRH moves).

So let us assume that $x\overset{\text{OPRH}}{\longrightarrow}y$. Thus, $y$ is
obtained from $x$ by picking two entries $x_{i}$ and $x_{j}$ with $x_{i}\leq
x_{j}$ and replacing them by $u$ and $v$, where $u,v\in\left[  x_{i}%
,x_{j}\right]  $ with $u+v=x_{i}+x_{j}$. Consider these $x_{i},x_{j},u,v$. We
must prove that%
\[
f\left(  x_{1}\right)  +f\left(  x_{2}\right)  +\cdots+f\left(  x_{n}\right)
\geq f\left(  y_{1}\right)  +f\left(  y_{2}\right)  +\cdots+f\left(
y_{n}\right)  .
\]
It clearly suffices to show that%
\[
f\left(  x_{i}\right)  +f\left(  x_{j}\right)  \geq f\left(  u\right)
+f\left(  v\right)
\]
(since $y_{k}=x_{k}$ for all $k$ other than $i$ and $j$).

But showing this is easy: From $u\in\left[  x_{i},x_{j}\right]  $, we obtain
\[
u=\lambda x_{i}+\left(  1-\lambda\right)  x_{j}\ \ \ \ \ \ \ \ \ \ \text{for
some }\lambda\in\left[  0,1\right]
\]
(namely, $\lambda=\dfrac{u-x_{j}}{x_{i}-x_{j}}$). Consider this $\lambda$.
Then,%
\[
v=\left(  1-\lambda\right)  x_{i}+\lambda x_{j}%
\]
(this follows easily from substituting $u=\lambda x_{i}+\left(  1-\lambda
\right)  x_{j}$ into $u+v=x_{i}+x_{j}$ and solving for $v$).

From $u=\lambda x_{i}+\left(  1-\lambda\right)  x_{j}$, we obtain%
\[
f\left(  u\right)  =f\left(  \lambda x_{i}+\left(  1-\lambda\right)
x_{j}\right)  \leq\lambda f\left(  x_{i}\right)  +\left(  1-\lambda\right)
f\left(  x_{j}\right)  \ \ \ \ \ \ \ \ \ \ \left(  \text{since }f\text{ is
convex}\right)  .
\]
From $v=\left(  1-\lambda\right)  x_{i}+\lambda x_{j}$, we obtain
\[
f\left(  v\right)  =f\left(  \left(  1-\lambda\right)  x_{i}+\lambda
x_{j}\right)  \leq\left(  1-\lambda\right)  f\left(  x_{i}\right)  +\lambda
f\left(  x_{j}\right)  \ \ \ \ \ \ \ \ \ \ \left(  \text{since }f\text{ is
convex}\right)  .
\]
Adding together these two inequalities, we obtain%
\begin{align*}
f\left(  u\right)  +f\left(  v\right)   &  \leq\left(  \lambda f\left(
x_{i}\right)  +\left(  1-\lambda\right)  f\left(  x_{j}\right)  \right)
+\left(  \left(  1-\lambda\right)  f\left(  x_{i}\right)  +\lambda f\left(
x_{j}\right)  \right) \\
&  =f\left(  x_{i}\right)  +f\left(  x_{j}\right)
,\ \ \ \ \ \ \ \ \ \ \text{qed.}%
\end{align*}
Thus, Theorem \ref{thm.major.karamata} is proved.
\end{proof}

Karamata's inequality has a converse: If $x,y\in\mathbb{R}^{n}$ are two
vectors such that \textbf{every} convex function $f:\mathbb{R}\rightarrow
\mathbb{R}$ satisfies%
\[
f\left(  x_{1}\right)  +f\left(  x_{2}\right)  +\cdots+f\left(  x_{n}\right)
\geq f\left(  y_{1}\right)  +f\left(  y_{2}\right)  +\cdots+f\left(
y_{n}\right)  ,
\]
then $x\succcurlyeq y$. Even better, we don't even need to require this to
hold for \textbf{every} convex function $f$; instead, it suffices to require
for the special class of convex functions $f:\mathbb{R}\rightarrow\mathbb{R}$
that have the form $z\mapsto\left\vert z-t\right\vert $ for constants
$t\in\mathbb{R}$. In other words, we have the following:

\begin{theorem}
[absolute-value criterion for majorization]\label{thm.major.abs-crit}Let
$x\in\mathbb{R}^{n}$ and $y\in\mathbb{R}^{n}$ be two vectors. Then,
$x\succcurlyeq y$ if and only if all $t\in\mathbb{R}$ satisfy%
\[
\left\vert x_{1}-t\right\vert +\left\vert x_{2}-t\right\vert +\cdots
+\left\vert x_{n}-t\right\vert \geq\left\vert y_{1}-t\right\vert +\left\vert
y_{2}-t\right\vert +\cdots+\left\vert y_{n}-t\right\vert .
\]

\end{theorem}

\begin{proof}
$\Longrightarrow:$ Assume that $x\succcurlyeq y$. Let $t\in\mathbb{R}$.
Consider the function%
\begin{align*}
f_{t}:\mathbb{R}  &  \rightarrow\mathbb{R},\\
z  &  \mapsto\left\vert z-t\right\vert .
\end{align*}
This function $f_{t}$ is convex (this follows easily from the triangle
inequality). Hence, Karamata's inequality (Theorem \ref{thm.major.karamata})
yields%
\[
f_{t}\left(  x_{1}\right)  +f_{t}\left(  x_{2}\right)  +\cdots+f_{t}\left(
x_{n}\right)  \geq f_{t}\left(  y_{1}\right)  +f_{t}\left(  y_{2}\right)
+\cdots+f_{t}\left(  y_{n}\right)  .
\]
By the definition of $f_{t}$, this means
\[
\left\vert x_{1}-t\right\vert +\left\vert x_{2}-t\right\vert +\cdots
+\left\vert x_{n}-t\right\vert \geq\left\vert y_{1}-t\right\vert +\left\vert
y_{2}-t\right\vert +\cdots+\left\vert y_{n}-t\right\vert .
\]
So we have proved the \textquotedblleft$\Longrightarrow$\textquotedblright%
\ direction of Theorem \ref{thm.major.abs-crit}. \medskip

$\Longleftarrow:$ We assume that the inequality%
\begin{align}
&  \left\vert x_{1}-t\right\vert +\left\vert x_{2}-t\right\vert +\cdots
+\left\vert x_{n}-t\right\vert \nonumber\\
&  \geq\left\vert y_{1}-t\right\vert +\left\vert y_{2}-t\right\vert
+\cdots+\left\vert y_{n}-t\right\vert \label{pf.thm.major.abs-crit.denn.avi}%
\end{align}
holds for all $t\in\mathbb{R}$. (Actually, it will suffice to assume that it
holds for all $t\in\left\{  x_{1},x_{2},\ldots,x_{n}\right\}  $.)

We must prove that $x\succcurlyeq y$.

WLOG assume that $x$ and $y$ are weakly decreasing (since permuting the
entries changes neither the inequality (\ref{pf.thm.major.abs-crit.denn.avi})
nor the claim $x\succcurlyeq y$).

For each $t\in\mathbb{R}$, we have%
\begin{align}
\sum_{i=1}^{n}\left\vert x_{i}-t\right\vert  &  =\left\vert x_{1}-t\right\vert
+\left\vert x_{2}-t\right\vert +\cdots+\left\vert x_{n}-t\right\vert
\nonumber\\
&  \geq\left\vert y_{1}-t\right\vert +\left\vert y_{2}-t\right\vert
+\cdots+\left\vert y_{n}-t\right\vert \ \ \ \ \ \ \ \ \ \ \left(  \text{by
(\ref{pf.thm.major.abs-crit.denn.avi})}\right) \nonumber\\
&  =\sum_{i=1}^{n}\left\vert y_{i}-t\right\vert .
\label{pf.thm.major.abs-crit.denn.avi2}%
\end{align}


Let $k\in\left\{  0,1,\ldots,n\right\}  $. Pick some $t\in\left\{  x_{1}%
,x_{2},\ldots,x_{n}\right\}  $ satisfying $x_{k}\geq t\geq x_{k+1}%
$\ \ \ \ \footnote{Here is how to find such a $t$: If $k>0$, then we pick
$t=x_{k}$; otherwise, we pick $t=x_{k+1}=x_{1}$.}. Then, since $x$ is weakly
decreasing, we have%
\[
x_{1}\geq x_{2}\geq\cdots\geq x_{k}\geq t\geq x_{k+1}\geq x_{k+2}\geq
\cdots\geq x_{n}.
\]
Thus, each $i\in\left\{  1,2,\ldots,k\right\}  $ satisfies $x_{i}\geq t$ and
therefore%
\begin{equation}
\left\vert x_{i}-t\right\vert =x_{i}-t,
\label{pf.thm.major.abs-crit.denn.abs1}%
\end{equation}
whereas each $i\in\left\{  k+1,k+2,\ldots,n\right\}  $ satisfies $t\geq x_{i}$
and therefore%
\begin{equation}
\left\vert x_{i}-t\right\vert =t-x_{i}.
\label{pf.thm.major.abs-crit.denn.abs2}%
\end{equation}
Now,%
\begin{align}
\sum_{i=1}^{n}\left\vert x_{i}-t\right\vert  &  =\sum_{i=1}^{k}%
\underbrace{\left\vert x_{i}-t\right\vert }_{\substack{=x_{i}-t\\\text{(by
(\ref{pf.thm.major.abs-crit.denn.abs1}))}}}+\sum_{i=k+1}^{n}%
\underbrace{\left\vert x_{i}-t\right\vert }_{\substack{=t-x_{i}\\\text{(by
(\ref{pf.thm.major.abs-crit.denn.abs2}))}}}\nonumber\\
&  =\underbrace{\sum_{i=1}^{k}\left(  x_{i}-t\right)  }_{=\sum_{i=1}^{k}%
x_{i}-kt}+\underbrace{\sum_{i=k+1}^{n}\left(  t-x_{i}\right)  }_{=\left(
n-k\right)  t-\sum_{i=k+1}^{n}x_{i}}=\sum_{i=1}^{k}x_{i}-kt+\left(
n-k\right)  t-\underbrace{\sum_{i=k+1}^{n}x_{i}}_{=\sum_{i=1}^{n}x_{i}%
-\sum_{i=1}^{k}x_{i}}\nonumber\\
&  =\sum_{i=1}^{k}x_{i}-kt+\left(  n-k\right)  t-\left(  \sum_{i=1}^{n}%
x_{i}-\sum_{i=1}^{k}x_{i}\right) \nonumber\\
&  =2\sum_{i=1}^{k}x_{i}+\left(  n-2k\right)  t-\sum_{i=1}^{n}x_{i}
\label{pf.thm.major.abs-crit.denn.abs-x}%
\end{align}
and%
\begin{align}
\sum_{i=1}^{n}\left\vert y_{i}-t\right\vert  &  =\sum_{i=1}^{k}%
\underbrace{\left\vert y_{i}-t\right\vert }_{\substack{\geq y_{i}%
-t\\\text{(since }\left\vert z\right\vert \geq z\text{ for each }%
z\in\mathbb{R}\text{)}}}+\sum_{i=k+1}^{n}\underbrace{\left\vert y_{i}%
-t\right\vert }_{\substack{\geq t-y_{i}\\\text{(since }\left\vert z\right\vert
\geq-z\text{ for each }z\in\mathbb{R}\text{)}}}\nonumber\\
&  \geq\underbrace{\sum_{i=1}^{k}\left(  y_{i}-t\right)  }_{=\sum_{i=1}%
^{k}y_{i}-kt}+\underbrace{\sum_{i=k+1}^{n}\left(  t-y_{i}\right)  }_{=\left(
n-k\right)  t-\sum_{i=k+1}^{n}y_{i}}=\sum_{i=1}^{k}y_{i}-kt+\left(
n-k\right)  t-\underbrace{\sum_{i=k+1}^{n}y_{i}}_{=\sum_{i=1}^{n}y_{i}%
-\sum_{i=1}^{k}y_{i}}\nonumber\\
&  =\sum_{i=1}^{k}y_{i}-kt+\left(  n-k\right)  t-\left(  \sum_{i=1}^{n}%
y_{i}-\sum_{i=1}^{k}y_{i}\right) \nonumber\\
&  =2\sum_{i=1}^{k}y_{i}+\left(  n-2k\right)  t-\sum_{i=1}^{n}y_{i}.
\label{pf.thm.major.abs-crit.denn.abs-y}%
\end{align}
Now, (\ref{pf.thm.major.abs-crit.denn.abs-x}) yields%
\begin{align*}
&  2\sum_{i=1}^{k}x_{i}+\left(  n-2k\right)  t-\sum_{i=1}^{n}x_{i}\\
&  =\sum_{i=1}^{n}\left\vert x_{i}-t\right\vert \geq\sum_{i=1}^{n}\left\vert
y_{i}-t\right\vert \ \ \ \ \ \ \ \ \ \ \left(  \text{by
(\ref{pf.thm.major.abs-crit.denn.avi2})}\right) \\
&  \geq2\sum_{i=1}^{k}y_{i}+\left(  n-2k\right)  t-\sum_{i=1}^{n}%
y_{i}\ \ \ \ \ \ \ \ \ \ \left(  \text{by
(\ref{pf.thm.major.abs-crit.denn.abs-y})}\right)  .
\end{align*}
Subtracting $\left(  n-2k\right)  t$ from both sides of this inequality, we
obtain%
\begin{equation}
2\sum_{i=1}^{k}x_{i}-\sum_{i=1}^{n}x_{i}\geq2\sum_{i=1}^{k}y_{i}-\sum
_{i=1}^{n}y_{i}. \label{pf.thm.major.abs-crit.denn.k1}%
\end{equation}


Forget that we fixed $k$. We thus have proved the inequality
(\ref{pf.thm.major.abs-crit.denn.k1}) for all $k\in\left\{  0,1,\ldots
,n\right\}  $.

Applying (\ref{pf.thm.major.abs-crit.denn.k1}) to $k=0$, we obtain%
\begin{equation}
-\sum_{i=1}^{n}x_{i}\geq-\sum_{i=1}^{n}y_{i}
\label{pf.thm.major.abs-crit.denn.k1=0}%
\end{equation}
(since both $\sum_{i=1}^{k}$ sums are empty for $k=0$). In other words,%
\begin{equation}
\sum_{i=1}^{n}x_{i}\leq\sum_{i=1}^{n}y_{i}.
\label{pf.thm.major.abs-crit.denn.k1=0b}%
\end{equation}
On the other hand, applying (\ref{pf.thm.major.abs-crit.denn.k1}) to $k=n$, we
obtain%
\begin{equation}
\sum_{i=1}^{n}x_{i}\geq\sum_{i=1}^{n}y_{i}
\label{pf.thm.major.abs-crit.denn.k1=n}%
\end{equation}
(since the left hand side simplifies to $2\sum_{i=1}^{n}x_{i}-\sum_{i=1}%
^{n}x_{i}=\sum_{i=1}^{n}x_{i}$, and likewise for the right hand side).
Combining this inequality with (\ref{pf.thm.major.abs-crit.denn.k1=0b}), we
obtain%
\begin{equation}
\sum_{i=1}^{n}x_{i}=\sum_{i=1}^{n}y_{i}. \label{pf.thm.major.abs-crit.denn.eq}%
\end{equation}


Now, for each $k\in\left[  n\right]  $, we have%
\[
2\sum_{i=1}^{k}x_{i}\geq2\sum_{i=1}^{k}y_{i}%
\]
(by adding the inequalities (\ref{pf.thm.major.abs-crit.denn.k1}) and
(\ref{pf.thm.major.abs-crit.denn.k1=n}) together) and therefore
\[
\sum_{i=1}^{k}x_{i}\geq\sum_{i=1}^{k}y_{i}%
\]
(by cancelling the factor $2$ from both sides of the previous inequality).
This fact, combined with (\ref{pf.thm.major.abs-crit.denn.eq}), shows that
$x\succcurlyeq y$ (since $x$ and $y$ are weakly decreasing). This proves the
\textquotedblleft$\Longleftarrow$\textquotedblright\ direction of Theorem
\ref{thm.major.abs-crit}.
\end{proof}

\begin{exercise}
\fbox{2} Let $x,y\in\mathbb{R}^{n}$ be two vectors such that $x\succcurlyeq
y$. Let $u,v\in\mathbb{R}^{m}$ be two further vectors such that $u\succcurlyeq
v$. Prove that $\left(
\begin{array}
[c]{c}%
x\\
u
\end{array}
\right)  \succcurlyeq\left(
\begin{array}
[c]{c}%
y\\
v
\end{array}
\right)  $. (We are using block-matrix notation here, so that $\left(
\begin{array}
[c]{c}%
x\\
u
\end{array}
\right)  $ means the vector obtained by stacking $x$ on top of $u$.)
\end{exercise}

\begin{exercise}
\fbox{3} Prove Theorem \ref{thm.major.RH.RH-crit}.
\end{exercise}

We note that the analogue of Exercise \ref{exe.major.RH.OPRH-n-1} for
arbitrary (not necessarily weakly decreasing) column vectors is false: It is
not hard to find two vectors $x,y\in\mathbb{R}^{4}$ such that $x\succcurlyeq
y$ but it takes $4$ (not $3$) RH moves to transform $x$ into $y$. (For
example, $x=\left(  5,3,2,0\right)  ^{T}$ and $y=\left(  1,1,4,4\right)  ^{T}$
are two such vectors.) On the other hand, it is easy to see (piggybacking on
Exercise \ref{exe.major.RH.OPRH-n-1}) that for any two column vectors
$x,y\in\mathbb{R}^{n}$ satisfying $x\succcurlyeq y$, it is possible to obtain
$y$ from $x$ by at most $2n-2$ RH moves. Finding the minimum number of RH
moves that always suffices to transform $x\in\mathbb{R}^{n}$ into
$y\in\mathbb{R}^{n}$ when $x\succcurlyeq y$ appears to be an interesting
question. \medskip

We can use Theorem \ref{thm.major.abs-crit} to define a \textquotedblleft
weighted\textquotedblright\ generalization of majorization. This leads to the
following generalization of Theorem \ref{thm.major.karamata}:

\begin{theorem}
[weighted Karamata's inequality]\label{thm.major.karamata-wt}Let
$I\subseteq\mathbb{R}$ be an interval. Let $f:I\rightarrow\mathbb{R}$ be a
convex function. Let $w_{1},w_{2},\ldots,w_{n}$ be $n$ nonnegative reals. Let
$x\in I^{n}$ and $y\in I^{n}$ be two vectors such that all $t\in\mathbb{R}$
satisfy%
\[
w_{1}\left\vert x_{1}-t\right\vert +w_{2}\left\vert x_{2}-t\right\vert
+\cdots+w_{n}\left\vert x_{n}-t\right\vert \geq w_{1}\left\vert y_{1}%
-t\right\vert +w_{2}\left\vert y_{2}-t\right\vert +\cdots+w_{n}\left\vert
y_{n}-t\right\vert .
\]
(Note that this is a \textquotedblleft weighted\textquotedblright\ version of
the condition $x\succcurlyeq y$.) Then,%
\[
w_{1}f\left(  x_{1}\right)  +w_{2}f\left(  x_{2}\right)  +\cdots+w_{n}f\left(
x_{n}\right)  \geq w_{1}f\left(  y_{1}\right)  +w_{2}f\left(  y_{2}\right)
+\cdots+w_{n}f\left(  y_{n}\right)  .
\]

\end{theorem}

\begin{exercise}
\label{exe.major.karamata-wt}\fbox{7} Prove Theorem
\ref{thm.major.karamata-wt}.

[\textbf{Hint:} Let $S$ be a finite subset of $I$. For each $s\in S$, let
$f_{s}:I\rightarrow\mathbb{R}$ be the piecewise-linear function that sends
each $z\in I$ to $\left\vert s-z\right\vert $. Show that the convex function
$f$ can be interpolated on $S$ by a linear combination $\sum_{s\in S}%
\alpha_{s}f_{s}$ of the functions $f_{s}$ with nonnegative coefficients
$\alpha_{s}$; that is, show that there exists a nonnegative real $\alpha_{s}$
for each $s\in S$ such that%
\[
f\left(  z\right)  =\sum_{s\in S}\alpha_{s}\left\vert s-z\right\vert
\ \ \ \ \ \ \ \ \ \ \text{for each }z\in S.
\]
Then, apply this to $S=\left\{  x_{1},x_{2},\ldots,x_{n},y_{1},y_{2}%
,\ldots,y_{n}\right\}  $.]
\end{exercise}

\begin{exercise}
\label{exe.major.weakmaj}\fbox{7} We define a new binary relation
$\succcurlyeq^{\prime}$ on the set $\mathbb{R}^{n}$ as follows: For two column
vectors $x\in\mathbb{R}^{n}$ and $y\in\mathbb{R}^{n}$, we write $x\succcurlyeq
^{\prime}y$ (and say that $x$ \emph{weakly majorizes} $y$) if and only if we
have%
\[
\sum_{i=1}^{m}x_{i}^{\downarrow}\geq\sum_{i=1}^{m}y_{i}^{\downarrow
}\ \ \ \ \ \ \ \ \ \ \text{for each }m\in\left[  n\right]
\]
(but we do not require $\sum_{i=1}^{n}x_{i}^{\downarrow}=\sum_{i=1}^{n}%
y_{i}^{\downarrow}$).

Let $x$ and $y$ be two weakly decreasing column vectors in $\mathbb{R}^{n}$.
Let $I$ be an interval of $\mathbb{R}$ that contains all entries of $x$ and of
$y$. Prove that the following statements are equivalent:

\begin{itemize}
\item $\mathcal{A}$\textit{:} We have $x\succcurlyeq^{\prime}y$.

\item $\mathcal{B}$\textit{:} For any sufficiently low $\alpha\in\mathbb{R}$,
we have $\left(
\begin{array}
[c]{c}%
x\\
\alpha-\sum x
\end{array}
\right)  \succcurlyeq\left(
\begin{array}
[c]{c}%
y\\
\alpha-\sum y
\end{array}
\right)  $, where $\sum x:=\sum_{i=1}^{n}x_{i}$ and $\sum y:=\sum_{i=1}%
^{n}y_{i}$ (and where we are using block-matrix notation, so that $\left(
\begin{array}
[c]{c}%
x\\
\alpha-\sum x
\end{array}
\right)  $ denotes the result of appending a new entry $\alpha-\sum x$ to the
bottom of the column vector $x$).

\item $\mathcal{C}$\textit{:} We can obtain $y$ from $x$ by a sequence of OPRH
moves and OPD moves. Here, an \textquotedblleft\emph{OPD move}%
\textquotedblright\ (short for \textquotedblleft order-preserving decrease
move\textquotedblright) means a move in which we decrease an entry of a weakly
decreasing vector in such a way that the vector remains weakly decreasing
(i.e., we replace an entry $z_{i}$ of a decreasing vector $z\in\mathbb{R}^{n}$
by a smaller entry $z_{i}^{\prime}\leq z_{i}$ such that we still have
$z_{1}\geq z_{2}\geq\cdots\geq z_{i-1}\geq z_{i}^{\prime}\geq z_{i+1}\geq
z_{i+2}\geq\cdots\geq z_{n}$).

\item $\mathcal{D}$\textit{:} Every weakly increasing convex function
$f:I\rightarrow\mathbb{R}$ satisfies%
\[
f\left(  x_{1}\right)  +f\left(  x_{2}\right)  +\cdots+f\left(  x_{n}\right)
\geq f\left(  y_{1}\right)  +f\left(  y_{2}\right)  +\cdots+f\left(
y_{n}\right)  .
\]


\item $\mathcal{E}$\textit{:} All $t\in\mathbb{R}$ satisfy%
\[
\left(  x_{1}-t\right)  _{+}+\left(  x_{2}-t\right)  _{+}+\cdots+\left(
x_{n}-t\right)  _{+}\geq\left(  y_{1}-t\right)  _{+}+\left(  y_{2}-t\right)
_{+}+\cdots+\left(  y_{n}-t\right)  _{+}.
\]
Here, the notation $z_{+}$ means the positive part of a real number $z$ (that
is, we have $z_{+}=z$ when $z\geq0$, and $z_{+}=0$ otherwise).
\end{itemize}
\end{exercise}

\subsubsection{Doubly stochastic matrices}

Majorizing pairs of vectors are closely related to \emph{doubly stochastic
matrices}:

\begin{definition}
\label{def.major.doublestoch}A matrix $S\in\mathbb{R}^{n\times n}$ is said to
be \emph{doubly stochastic} if its entries $S_{i,j}$ satisfy the following
three conditions:

\begin{enumerate}
\item We have $S_{i,j}\geq0$ for all $i,j$.

\item We have $\sum_{j=1}^{n}S_{i,j}=1$ for each $i\in\left[  n\right]  $.

\item We have $\sum_{i=1}^{n}S_{i,j}=1$ for each $j\in\left[  n\right]  $.
\end{enumerate}
\end{definition}

In other words, a doubly stochastic matrix is an $n\times n$-matrix whose
entries are nonnegative reals and whose rows and columns have sum $1$ each.

\begin{exercise}
\label{exe.major.doublestoch.square}\fbox{2} Show that even if we allow $S$ to
be rectangular in Definition \ref{def.major.doublestoch}, the conditions 2 and
3 still force $S$ to be a square matrix.
\end{exercise}

\begin{example}
\label{exa.major.doublestoch.exas}\textbf{(a)} The matrix $\left(
\begin{array}
[c]{ccc}%
\dfrac{1}{2} & \dfrac{1}{3} & \dfrac{1}{6}\\
\dfrac{1}{2} & \dfrac{1}{4} & \dfrac{1}{4}\\
0 & \dfrac{5}{12} & \dfrac{7}{12}%
\end{array}
\right)  $ is doubly stochastic. \medskip

\textbf{(b)} Each doubly stochastic $2\times2$-matrix has the form%
\[
\left(
\begin{array}
[c]{cc}%
\lambda & 1-\lambda\\
1-\lambda & \lambda
\end{array}
\right)  \ \ \ \ \ \ \ \ \ \ \text{for some }\lambda\in\left[  0,1\right]  .
\]


\textbf{(c)} Any permutation matrix is doubly stochastic.
\end{example}

\begin{proposition}
\label{prop.major.doublestoch.e}Let $S\in\mathbb{R}^{n\times n}$ be a matrix
whose entries are nonnegative reals. Let $e=\left(  1,1,\ldots,1\right)
^{T}\in\mathbb{R}^{n}$. Then, $S$ is doubly stochastic if and only if $Se=e$
and $e^{T}S=e^{T}$.
\end{proposition}

\begin{corollary}
\label{cor.major.doublestoch.prod}Any product of doubly stochastic matrices is
again doubly stochastic.
\end{corollary}

\begin{exercise}
\fbox{3} Prove Proposition \ref{prop.major.doublestoch.e} and Corollary
\ref{cor.major.doublestoch.prod}.
\end{exercise}

Now, we can connect doubly stochastic matrices with majorization:

\begin{theorem}
\label{thm.major.doublestoch.maj}Let $x,y\in\mathbb{R}^{n}$ be two vectors.
Then, $x\succcurlyeq y$ if and only if there exists a doubly stochastic matrix
$S\in\mathbb{R}^{n\times n}$ such that $y=Sx$.
\end{theorem}

\begin{proof}
$\Longrightarrow:$ Assume that $x\succcurlyeq y$. We must prove that there
exists a doubly stochastic matrix $S\in\mathbb{R}^{n\times n}$ such that
$y=Sx$.

By Example \ref{exa.major.doublestoch.exas} \textbf{(c)} and Corollary
\ref{cor.major.doublestoch.prod}, it suffices to show this in the case when
$x$ and $y$ are weakly decreasing (because any permutation of the entries of a
vector can be effected by multiplying this vector with a permutation matrix).

Thus, we WLOG assume that $x$ and $y$ are weakly decreasing. We must prove
that there exists a doubly stochastic matrix $S\in\mathbb{R}^{n\times n}$ such
that $y=Sx$.

By Corollary \ref{cor.major.doublestoch.prod}, it suffices to show this in the
case when $x\overset{\text{OPRH}}{\longrightarrow}y$ (because in the general
case, $y$ is obtained from $x$ by a sequence of OPRH moves\footnote{This is a
consequence of Theorem \ref{thm.major.RH.OPRH-crit}.}).

So let us WLOG assume that $x\overset{\text{OPRH}}{\longrightarrow}y$. Thus,
$y$ is obtained from $x$ by picking two entries $x_{i}$ and $x_{j}$ with
$x_{i}\leq x_{j}$ and replacing them by $u$ and $v$, where $u,v\in\left[
x_{i},x_{j}\right]  $ with $u+v=x_{i}+x_{j}$. Consider these $x_{i},x_{j},u,v$.

From $u\in\left[  x_{i},x_{j}\right]  $, we obtain
\[
u=\lambda x_{i}+\left(  1-\lambda\right)  x_{j}\ \ \ \ \ \ \ \ \ \ \text{for
some }\lambda\in\left[  0,1\right]
\]
(namely, $\lambda=\dfrac{u-x_{j}}{x_{i}-x_{j}}$). Consider this $\lambda$.
Then,%
\[
v=\left(  1-\lambda\right)  x_{i}+\lambda x_{j}\ \ \ \ \ \ \ \ \ \ \left(
\text{since }u+v=x_{i}+x_{j}\right)  .
\]


This entails that $y=Sx$, where $S\in\mathbb{R}^{n\times n}$ is the matrix
defined by%
\begin{align*}
S_{i,i}  &  =\lambda,\ \ \ \ \ \ \ \ \ \ S_{i,j}=1-\lambda
,\ \ \ \ \ \ \ \ \ \ S_{j,i}=1-\lambda,\ \ \ \ \ \ \ \ \ \ S_{j,j}=\lambda,\\
S_{k,k}  &  =1\ \ \ \ \ \ \ \ \ \ \text{for each }k\notin\left\{  i,j\right\}
,\\
S_{k,\ell}  &  =0\ \ \ \ \ \ \ \ \ \ \text{for all remaining }k,\ell.
\end{align*}
For example, if $i=2$ and $j=4$, then
\[
S=\left(
\begin{array}
[c]{cccc}%
1 &  &  & \\
& \lambda &  & 1-\lambda\\
&  & 1 & \\
& 1-\lambda &  & \lambda
\end{array}
\right)
\]
(where all empty cells are filled with zeroes). In this case,%
\[
Sx=S\left(
\begin{array}
[c]{c}%
x_{1}\\
x_{2}\\
x_{3}\\
x_{4}%
\end{array}
\right)  =\left(
\begin{array}
[c]{c}%
x_{1}\\
\lambda x_{2}+\left(  1-\lambda\right)  x_{4}\\
x_{3}\\
\left(  1-\lambda\right)  x_{2}+\lambda x_{4}%
\end{array}
\right)  =\left(
\begin{array}
[c]{c}%
x_{1}\\
u\\
x_{3}\\
v
\end{array}
\right)  =y.
\]
So we have constructed a matrix $S\in\mathbb{R}^{n\times n}$ that satisfies
$y=Sx$, and it is easy to see that this $S$ is doubly stochastic. Thus, we
have proved the \textquotedblleft$\Longrightarrow$\textquotedblright%
\ direction of Theorem \ref{thm.major.doublestoch.maj}. \medskip

$\Longleftarrow:$ Assume that $y=Sx$ for some doubly stochastic matrix
$S\in\mathbb{R}^{n\times n}$. Then, for every $i\in\left[  n\right]  $, we
have%
\begin{equation}
y_{i}=\left(  Sx\right)  _{i}=\sum_{j=1}^{n}S_{i,j}x_{j}.
\label{pf.thm.major.doublestoch.maj.denn.yi=}%
\end{equation}
Hence, for every $i\in\left[  n\right]  $ and $t\in\mathbb{R}$, we have%
\begin{align*}
y_{i}-t  &  =\sum_{j=1}^{n}S_{i,j}x_{j}-t=\sum_{j=1}^{n}S_{i,j}x_{j}%
-\sum_{j=1}^{n}S_{i,j}t\\
&  \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \left(
\begin{array}
[c]{c}%
\text{since condition 2 in Definition \ref{def.major.doublestoch}}\\
\text{yields }\sum_{j=1}^{n}S_{i,j}=1\text{, so that }\sum_{j=1}^{n}%
S_{i,j}t=\underbrace{\left(  \sum_{j=1}^{n}S_{i,j}\right)  }_{=1}t=t\\
\text{and therefore }t=\sum_{j=1}^{n}S_{i,j}t
\end{array}
\right) \\
&  =\sum_{j=1}^{n}S_{i,j}\left(  x_{j}-t\right)
\end{align*}
and therefore%
\begin{align}
\left\vert y_{i}-t\right\vert  &  =\left\vert \sum_{j=1}^{n}S_{i,j}\left(
x_{j}-t\right)  \right\vert \nonumber\\
&  \leq\sum_{j=1}^{n}\underbrace{\left\vert S_{i,j}\left(  x_{j}-t\right)
\right\vert }_{\substack{=S_{i,j}\cdot\left\vert x_{j}-t\right\vert
\\\text{(since }S_{i,j}\geq0\\\text{(by condition 1 in Definition
\ref{def.major.doublestoch}))}}}\nonumber\\
&  \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \left(
\begin{array}
[c]{c}%
\text{by the triangle inequality, which says that }\left\vert \sum_{j=1}%
^{n}\alpha_{j}\right\vert \leq\sum_{j=1}^{n}\left\vert \alpha_{j}\right\vert
\\
\text{for any }n\text{ reals }\alpha_{1},\alpha_{2},\ldots,\alpha_{n}%
\end{array}
\right) \nonumber\\
&  =\sum_{j=1}^{n}S_{i,j}\cdot\left\vert x_{j}-t\right\vert .
\label{pf.thm.major.doublestoch.maj.denn.2}%
\end{align}


Thus, for every $t\in\mathbb{R}$, we have%
\begin{align*}
&  \left\vert y_{1}-t\right\vert +\left\vert y_{2}-t\right\vert +\cdots
+\left\vert y_{n}-t\right\vert \\
&  =\sum_{i=1}^{n}\left\vert y_{i}-t\right\vert \leq\sum_{i=1}^{n}%
\ \ \sum_{j=1}^{n}S_{i,j}\cdot\left\vert x_{j}-t\right\vert
\ \ \ \ \ \ \ \ \ \ \left(  \text{by
(\ref{pf.thm.major.doublestoch.maj.denn.2})}\right) \\
&  =\sum_{j=1}^{n}\underbrace{\left(  \sum_{i=1}^{n}S_{i,j}\right)
}_{\substack{=1\\\text{(by condition 3}\\\text{in Definition
\ref{def.major.doublestoch})}}}\cdot\left\vert x_{j}-t\right\vert =\sum
_{j=1}^{n}\left\vert x_{j}-t\right\vert \\
&  =\left\vert x_{1}-t\right\vert +\left\vert x_{2}-t\right\vert
+\cdots+\left\vert x_{n}-t\right\vert .
\end{align*}
In other words, for every $t\in\mathbb{R}$, we have%
\[
\left\vert x_{1}-t\right\vert +\left\vert x_{2}-t\right\vert +\cdots
+\left\vert x_{n}-t\right\vert \geq\left\vert y_{1}-t\right\vert +\left\vert
y_{2}-t\right\vert +\cdots+\left\vert y_{n}-t\right\vert .
\]
Therefore, by Theorem \ref{thm.major.abs-crit}, we have $x\succcurlyeq y$.
This proves the \textquotedblleft$\Longleftarrow$\textquotedblright\ direction
of Theorem \ref{thm.major.doublestoch.maj}.
\end{proof}

\section{Singular value decomposition (\cite[\S 2.6]{HorJoh13})}

This will be just a brief introduction to singular value decomposition. For
much more, see \cite{TreBau97}.

\subsection{Some properties of $A^{\ast}A$}

We first state some basic properties of matrices of the form $A^{\ast}A$:

\begin{proposition}
[the $\operatorname*{Ker}\left(  A^{\ast}A\right)  $ lemma]%
\label{prop.SVD.A*A}Let $A\in\mathbb{C}^{m\times n}$ be any $m\times n$-matrix
with complex entries (not necessarily a square matrix). Then: \medskip

\textbf{(a)} The matrix $A^{\ast}A$ is Hermitian and positive semidefinite.
\medskip

\textbf{(b)} We have $\operatorname*{Ker}A=\operatorname*{Ker}\left(  A^{\ast
}A\right)  $. \medskip

\textbf{(c)} We have $\operatorname*{rank}A=\operatorname*{rank}\left(
A^{\ast}A\right)  $.
\end{proposition}

\begin{proof}
\textbf{(a)} The matrix $A^{\ast}A$ is Hermitian, since $\left(  A^{\ast
}A\right)  ^{\ast}=A^{\ast}\underbrace{\left(  A^{\ast}\right)  ^{\ast}}%
_{=A}=A^{\ast}A$. Moreover, this matrix $A^{\ast}A$ is positive semidefinite,
since each vector $x\in\mathbb{C}^{n}$ satisfies%
\[
\left\langle A^{\ast}Ax,\ x\right\rangle =\underbrace{x^{\ast}A^{\ast}%
}_{=\left(  Ax\right)  ^{\ast}}Ax=\left(  Ax\right)  ^{\ast}Ax=\left\vert
\left\vert Ax\right\vert \right\vert ^{2}\geq0.
\]
Thus, Proposition \ref{prop.SVD.A*A} \textbf{(a)} is proven. \medskip

\textbf{(b)} Each $y\in\operatorname*{Ker}A$ satisfies $y\in
\operatorname*{Ker}\left(  A^{\ast}A\right)  $ (because $y\in
\operatorname*{Ker}A$ entails $Ay=0$, so that $A^{\ast}\underbrace{Ay}_{=0}=0$
and thus $y\in\operatorname*{Ker}\left(  A^{\ast}A\right)  $). In other words,
$\operatorname*{Ker}A\subseteq\operatorname*{Ker}\left(  A^{\ast}A\right)  $.

Let us now show that $\operatorname*{Ker}\left(  A^{\ast}A\right)
\subseteq\operatorname*{Ker}A$. Indeed, let $x\in\operatorname*{Ker}\left(
A^{\ast}A\right)  $. Thus, $x\in\mathbb{C}^{n}$ and $A^{\ast}Ax=0$. Hence,
\begin{align*}
\left\vert \left\vert Ax\right\vert \right\vert ^{2}  &  =\underbrace{\left(
Ax\right)  ^{\ast}}_{=x^{\ast}A^{\ast}}Ax\ \ \ \ \ \ \ \ \ \ \left(
\text{since }\left\vert \left\vert v\right\vert \right\vert ^{2}=\left\langle
v,v\right\rangle =v^{\ast}v\text{ for any }v\in\mathbb{C}^{n}\right) \\
&  =x^{\ast}\underbrace{A^{\ast}Ax}_{=0}=0.
\end{align*}
In other words, $\left\vert \left\vert Ax\right\vert \right\vert =0$. Hence,
$Ax=0$ (since a vector whose length is $0$ must itself be $0$). In other
words, $x\in\operatorname*{Ker}A$.

Forget that we fixed $x$. We thus have shown that $x\in\operatorname*{Ker}A$
for each $x\in\operatorname*{Ker}\left(  A^{\ast}A\right)  $. In other words,
$\operatorname*{Ker}\left(  A^{\ast}A\right)  \subseteq\operatorname*{Ker}A$.
Combining this with $\operatorname*{Ker}A\subseteq\operatorname*{Ker}\left(
A^{\ast}A\right)  $, we obtain $\operatorname*{Ker}A=\operatorname*{Ker}%
\left(  A^{\ast}A\right)  $. This proves Proposition \ref{prop.SVD.A*A}
\textbf{(b)}. \medskip

\textbf{(c)} The rank-nullity theorem yields%
\begin{align*}
\operatorname*{rank}A  &  =n-\dim\left(  \operatorname*{Ker}A\right)
\ \ \ \ \ \ \ \ \ \ \text{and}\\
\operatorname*{rank}\left(  A^{\ast}A\right)   &  =n-\dim\left(
\operatorname*{Ker}\left(  A^{\ast}A\right)  \right)  .
\end{align*}
The right hand sides of these two equalities are equal (since part
\textbf{(b)} yields $\operatorname*{Ker}A=\operatorname*{Ker}\left(  A^{\ast
}A\right)  $). Thus, the left hand sides are also equal. In other words,
$\operatorname*{rank}A=\operatorname*{rank}\left(  A^{\ast}A\right)  $. This
proves Proposition \ref{prop.SVD.A*A} \textbf{(c)}.
\end{proof}

Note that Proposition \ref{prop.SVD.A*A} \textbf{(b)} really requires a matrix
with complex entries. It cannot be generalized to matrices over an arbitrary field.

\subsection{The singular value decomposition}

\begin{definition}
\label{def.svd.uniteq}Let $A$ and $B$ be two $m\times n$-matrices with complex entries.

We say that $A$ and $B$ are \emph{unitarily equivalent} if there exist unitary
matrices $U\in\operatorname*{U}\nolimits_{m}\left(  \mathbb{C}\right)  $ and
$V\in\operatorname*{U}\nolimits_{n}\left(  \mathbb{C}\right)  $ such that
$A=UBV^{\ast}$.
\end{definition}

Note that we could just as well require $A=UBV$ instead of $A=UBV^{\ast}$
here, since $V$ is unitary if and only if $V^{\ast}$ is unitary.

Note the difference between \textquotedblleft unitarily
equivalent\textquotedblright\ and \textquotedblleft unitarily
similar\textquotedblright: The latter requires $A=UBU^{\ast}$, whereas the
former only requires $A=UBV^{\ast}$.

Unitary equivalence is an equivalence relation.

\begin{exercise}
\fbox{2} Prove this!
\end{exercise}

A natural question is therefore: What is the \textquotedblleft
simplest\textquotedblright\ matrix in the equivalence class of a given matrix?
The answer is pretty nice: Each matrix is unitarily equivalent to a
\textquotedblleft more or less diagonal\textquotedblright\ matrix. We are
saying \textquotedblleft more or less\textquotedblright\ because diagonal
matrices are supposed to be square, but our matrices can have any dimensions;
thus, we introduce a separate word for rectangular matrices that
\textquotedblleft would be diagonal if they were square\textquotedblright:

\begin{definition}
Let $\mathbb{F}$ be a field. A rectangular matrix $A\in\mathbb{F}^{n\times m}$
is said to be \emph{pseudodiagonal} if it satisfies%
\[
A_{i,j}=0\ \ \ \ \ \ \ \ \ \ \text{whenever }i\neq j.
\]

\end{definition}

This is just the straightforward generalization of diagonal matrices to
non-square matrices. In particular, a square matrix is pseudodiagonal if and
only if it is diagonal. A pseudodiagonal $2\times3$-matrix looks like this:
$\left(
\begin{array}
[c]{ccc}%
\ast & 0 & 0\\
0 & \ast & 0
\end{array}
\right)  $. A pseudodiagonal $3\times2$-matrix looks like this: $\left(
\begin{array}
[c]{cc}%
\ast & 0\\
0 & \ast\\
0 & 0
\end{array}
\right)  $. (Of course, any of the $\ast$s can be a $0$ too.)

\begin{theorem}
[SVD]\label{thm.svd.svd}Let $A\in\mathbb{C}^{m\times n}$. Then: \medskip

\textbf{(a)} There exist unitary matrices $U\in\operatorname*{U}%
\nolimits_{m}\left(  \mathbb{C}\right)  $ and $V\in\operatorname*{U}%
\nolimits_{n}\left(  \mathbb{C}\right)  $ and a pseudodiagonal matrix
$\Sigma\in\mathbb{C}^{m\times n}$ such that all diagonal entries of $\Sigma$
are nonnegative reals and such that%
\[
A=U\Sigma V^{\ast}.
\]
In other words, $A$ is unitarily equivalent to a pseudodiagonal matrix whose
diagonal entries are nonnegative reals. \medskip

\textbf{(b)} The matrix $\Sigma$ is unique up to permutation of its diagonal
entries. (The matrices $U$ and $V$ are usually not unique.) \medskip

\textbf{(c)} Let $k=\operatorname*{rank}A$. Then, the matrix $\Sigma$ has
exactly $k$ nonzero diagonal entries. \medskip

\textbf{(d)} Let $\sigma_{1},\sigma_{2},\ldots,\sigma_{n}$ be the square roots
of the $n$ eigenvalues of the Hermitian matrix $A^{\ast}A$, listed in
decreasing order (so that $\sigma_{1}\geq\sigma_{2}\geq\cdots\geq\sigma_{n}$).
Then, we have $\sigma_{k+1}=\sigma_{k+2}=\cdots=\sigma_{n}=0$, and we can
take
\[
\Sigma=\left(
\begin{array}
[c]{ccccccc}%
\sigma_{1} & 0 & \cdots & 0 & 0 & \cdots & 0\\
0 & \sigma_{2} & \cdots & 0 & 0 & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & \sigma_{k} & 0 & \cdots & 0\\
0 & 0 & \cdots & 0 & 0 & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & 0 & 0 & \cdots & 0
\end{array}
\right)  \in\mathbb{C}^{m\times n}%
\]
in part \textbf{(a)}.
\end{theorem}

\begin{definition}
The triple $\left(  U,V,\Sigma\right)  $ in Theorem \ref{thm.svd.svd}
\textbf{(a)} is called a \emph{singular value decomposition} (short:
\emph{SVD}) of $A$. The numbers $\sigma_{1},\sigma_{2},\ldots,\sigma_{n}$ are
called the \emph{singular values} of $A$.
\end{definition}

Before we prove the theorem, a few words are to be said about the use of an
SVD. In practice, you often want to find a low-rank \textquotedblleft
approximation\textquotedblright\ for a given matrix $A$: that is, a matrix $B$
that is \textquotedblleft sufficiently close\textquotedblright\ to $A$ and yet
has low rank. One of the best ways to do this is by computing an SVD of $A$ --
that is, writing $A$ in the form $A=U\Sigma V^{\ast}$ with $U,\Sigma,V$ as in
Theorem \ref{thm.svd.svd} \textbf{(a)} -- and then setting all but the first
few $\sigma_{i}$'s to $0$ in $\Sigma$. We will soon see why this
\textquotedblleft approximates\textquotedblright\ $A$.

Let us now prove the theorem.

\begin{proof}
[Proof of Theorem \ref{thm.svd.svd}.]Let $k:=\operatorname*{rank}A$. Then, $k$
is the rank of an $m\times n$-matrix (namely, $A$), and thus satisfies $k\leq
m$ and $k\leq n$.

Proposition \ref{prop.SVD.A*A} \textbf{(a)} shows that the matrix $A^{\ast}A$
is Hermitian and positive semidefinite. Let $\lambda_{1},\lambda_{2}%
,\ldots,\lambda_{n}$ be the eigenvalues of this matrix $A^{\ast}A$, listed in
decreasing order. These eigenvalues $\lambda_{1},\lambda_{2},\ldots
,\lambda_{n}$ are nonnegative reals (since $A^{\ast}A$ is positive
semidefinite). Thus, we can set%
\[
\sigma_{i}:=\sqrt{\lambda_{i}}\ \ \ \ \ \ \ \ \ \ \text{for each }i\in\left[
n\right]  .
\]
Then, $\sigma_{1},\sigma_{2},\ldots,\sigma_{n}$ are nonnegative reals.
Moreover, $\lambda_{1}\geq\lambda_{2}\geq\cdots\geq\lambda_{n}$ (since
$\lambda_{1},\lambda_{2},\ldots,\lambda_{n}$ are listed in decreasing order)
and therefore $\sigma_{1}\geq\sigma_{2}\geq\cdots\geq\sigma_{n}$ (since
$\sigma_{i}=\sqrt{\lambda_{i}}$ for each $i$).

The matrix $A^{\ast}A$ is Hermitian and thus normal. Hence, Corollary
\ref{cor.schurtri.normal.given-order} (applied to $A^{\ast}A$ instead of $A$)
yields that there exists a spectral decomposition $\left(  V,D\right)  $ of
$A^{\ast}A$ with $D=\operatorname*{diag}\left(  \lambda_{1},\lambda_{2}%
,\ldots,\lambda_{n}\right)  $. Consider this spectral decomposition. Thus,
$V\in\operatorname*{U}\nolimits_{n}\left(  \mathbb{C}\right)  $ is a unitary
matrix, and $D=\operatorname*{diag}\left(  \lambda_{1},\lambda_{2}%
,\ldots,\lambda_{n}\right)  $, and we have
\begin{equation}
A^{\ast}A=VDV^{\ast}. \label{pf.thm.svd.svd.A=}%
\end{equation}


From (\ref{pf.thm.svd.svd.A=}), we obtain $A^{\ast}%
A\overset{\operatorname*{us}}{\sim}D$ (since $V$ is unitary). Now,
\begin{align*}
k  &  =\operatorname*{rank}A=\operatorname*{rank}\left(  A^{\ast}A\right)
\ \ \ \ \ \ \ \ \ \ \left(  \text{by Proposition \ref{prop.SVD.A*A}
\textbf{(c)}}\right) \\
&  =\operatorname*{rank}\left(  \operatorname*{diag}\left(  \lambda
_{1},\lambda_{2},\ldots,\lambda_{n}\right)  \right)
\ \ \ \ \ \ \ \ \ \ \left(  \text{since }A^{\ast}A\overset{\operatorname*{us}%
}{\sim}D=\operatorname*{diag}\left(  \lambda_{1},\lambda_{2},\ldots
,\lambda_{n}\right)  \right) \\
&  =\left(  \text{the number of }i\in\left[  n\right]  \text{ such that
}\lambda_{i}\neq0\right) \\
&  =\left(  \text{the number of }i\in\left[  n\right]  \text{ such that
}\sigma_{i}\neq0\right)
\end{align*}
(since $\sigma_{i}=\sqrt{\lambda_{i}}$ for each $i$). Hence, exactly $k$ of
the numbers $\sigma_{1},\sigma_{2},\ldots,\sigma_{n}$ are nonzero. Since
$\sigma_{1},\sigma_{2},\ldots,\sigma_{n}$ are nonnegative reals and satisfy
$\sigma_{1}\geq\sigma_{2}\geq\cdots\geq\sigma_{n}$, this entails that%
\begin{align}
\sigma_{1}  &  \geq\sigma_{2}\geq\cdots\geq\sigma_{k}%
>0\ \ \ \ \ \ \ \ \ \ \text{and}\label{pf.thm.svd.svd.pos}\\
\sigma_{k+1}  &  =\sigma_{k+2}=\cdots=\sigma_{n}=0.
\label{pf.thm.svd.svd.zero}%
\end{align}


Let $v_{1},v_{2},\ldots,v_{n}$ be the columns of the unitary matrix $V$. Thus,
$\left(  v_{1},v_{2},\ldots,v_{n}\right)  $ is an orthonormal basis of
$\mathbb{C}^{n}$ (since $V$ is unitary)\footnote{Here we are using the
implication $\mathcal{A}\Longrightarrow\mathcal{E}$ of Theorem
\ref{thm.unitary.unitary.eqs}.}. Since $V$ is the unitary matrix in a spectral
decomposition of $A^{\ast}A$, we can easily see that the columns of $V$ are
eigenvectors of $A^{\ast}A$ corresponding to the eigenvalues $\lambda
_{1},\lambda_{2},\ldots,\lambda_{n}$. In other words,%
\begin{equation}
A^{\ast}Av_{i}=\lambda_{i}v_{i}\ \ \ \ \ \ \ \ \ \ \text{for each }i\in\left[
n\right]  . \label{pf.thm.svd.svd.evec}%
\end{equation}


[\textit{Proof of (\ref{pf.thm.svd.svd.evec}):} Let $i\in\left[  n\right]  $.
Then, we have\footnote{Recall that the notation $M_{\bullet,i}$ denotes the
$i$-th column of a matrix $M$.} $V_{\bullet,i}=v_{i}$ (since $v_{1}%
,v_{2},\ldots,v_{n}$ are the columns of $V$). However, from
(\ref{pf.thm.svd.svd.A=}), we obtain $A^{\ast}AV=VD\underbrace{V^{\ast}%
V}_{\substack{=I_{n}\\\text{(since }V\text{ is unitary)}}}=VD$. Hence,%
\begin{align*}
\left(  A^{\ast}AV\right)  _{\bullet,i}  &  =\left(  VD\right)  _{\bullet,i}\\
&  =\left(  V\cdot\operatorname*{diag}\left(  \lambda_{1},\lambda_{2}%
,\ldots,\lambda_{n}\right)  \right)  _{\bullet,i}\ \ \ \ \ \ \ \ \ \ \left(
\text{since }D=\operatorname*{diag}\left(  \lambda_{1},\lambda_{2}%
,\ldots,\lambda_{n}\right)  \right) \\
&  =\lambda_{i}\underbrace{V_{\bullet,i}}_{=v_{i}}\ \ \ \ \ \ \ \ \ \ \left(
\begin{array}
[c]{c}%
\text{since multiplication by the diagonal}\\
\text{matrix }\operatorname*{diag}\left(  \lambda_{1},\lambda_{2}%
,\ldots,\lambda_{n}\right)  \text{ on the right}\\
\text{scales the }i\text{-th column of a matrix by }\lambda_{i}%
\end{array}
\right) \\
&  =\lambda_{i}v_{i}.
\end{align*}
Comparing this with%
\begin{align*}
\left(  A^{\ast}AV\right)  _{\bullet,i}  &  =A^{\ast}A\underbrace{V_{\bullet
,i}}_{=v_{i}}\ \ \ \ \ \ \ \ \ \ \left(  \text{by the rules for matrix
multiplication}\right) \\
&  =A^{\ast}Av_{i},
\end{align*}
we obtain $A^{\ast}Av_{i}=\lambda_{i}v_{i}$. This proves
(\ref{pf.thm.svd.svd.evec}).] \medskip

For each $j\in\left[  k\right]  $, we set%
\[
u_{j}:=\dfrac{1}{\sigma_{j}}Av_{j}.
\]
(The division by $\sigma_{j}$ in this definition is legitimate, since
(\ref{pf.thm.svd.svd.pos}) reveals that $\sigma_{j}\neq0$.) We claim that the
tuple $\left(  u_{1},u_{2},\ldots,u_{k}\right)  $ is orthonormal. Indeed:

\begin{itemize}
\item For any two distinct elements $i$ and $j$ of $\left[  k\right]  $, we
have%
\begin{align*}
\left\langle u_{i},u_{j}\right\rangle  &  =\left\langle \dfrac{1}{\sigma_{i}%
}Av_{i},\ \dfrac{1}{\sigma_{j}}Av_{j}\right\rangle \ \ \ \ \ \ \ \ \ \ \left(
\text{by the definitions of }u_{i}\text{ and }u_{j}\right) \\
&  =\dfrac{1}{\sigma_{i}\overline{\sigma_{j}}}\underbrace{\left\langle
Av_{i},\ Av_{j}\right\rangle }_{=\left(  Av_{j}\right)  ^{\ast}Av_{i}}%
=\dfrac{1}{\sigma_{i}\overline{\sigma_{j}}}\underbrace{\left(  Av_{j}\right)
^{\ast}}_{=v_{j}^{\ast}A^{\ast}}Av_{i}\\
&  =\dfrac{1}{\sigma_{i}\overline{\sigma_{j}}}v_{j}^{\ast}\underbrace{A^{\ast
}Av_{i}}_{\substack{=\lambda_{i}v_{i}\\\text{(by (\ref{pf.thm.svd.svd.evec})}%
}}=\dfrac{1}{\sigma_{i}\overline{\sigma_{j}}}v_{j}^{\ast}\lambda_{i}%
v_{i}=\dfrac{\lambda_{i}}{\sigma_{i}\overline{\sigma_{j}}}\underbrace{v_{j}%
^{\ast}v_{i}}_{\substack{=\left\langle v_{i},v_{j}\right\rangle
\\=0\\\text{(since }\left(  v_{1},v_{2},\ldots,v_{n}\right)  \\\text{is
orthonormal)}}}=0
\end{align*}
and thus $u_{i}\perp u_{j}$. Therefore, the tuple $\left(  u_{1},u_{2}%
,\ldots,u_{k}\right)  $ is orthogonal.

\item For any $i\in\left[  k\right]  $, we have%
\begin{align*}
\left\langle u_{i},u_{i}\right\rangle  &  =\left\langle \dfrac{1}{\sigma_{i}%
}Av_{i},\ \dfrac{1}{\sigma_{i}}Av_{i}\right\rangle \ \ \ \ \ \ \ \ \ \ \left(
\text{by the definition of }u_{i}\right) \\
&  =\underbrace{\dfrac{1}{\sigma_{i}\overline{\sigma_{i}}}}_{\substack{=\dfrac
{1}{\lambda_{i}}\\\text{(since }\sigma_{i}\in\mathbb{R}\text{ and}\\\text{thus
}\sigma_{i}\overline{\sigma_{i}}=\sigma_{i}\sigma_{i}=\sigma_{i}^{2}%
=\lambda_{i}\\\text{(because }\sigma_{i}=\sqrt{\lambda_{i}}\text{))}%
}}\underbrace{\left\langle Av_{i},\ Av_{i}\right\rangle }_{=\left(
Av_{i}\right)  ^{\ast}Av_{i}}=\dfrac{1}{\lambda_{i}}\underbrace{\left(
Av_{i}\right)  ^{\ast}}_{=v_{i}^{\ast}A^{\ast}}Av_{i}\\
&  =\dfrac{1}{\lambda_{i}}v_{i}^{\ast}\underbrace{A^{\ast}Av_{i}%
}_{\substack{=\lambda_{i}v_{i}\\\text{(by (\ref{pf.thm.svd.svd.evec})}%
}}=\dfrac{1}{\lambda_{i}}v_{i}^{\ast}\lambda_{i}v_{i}=v_{i}^{\ast}%
v_{i}=\left\langle v_{i},v_{i}\right\rangle =\left\vert \left\vert
v_{i}\right\vert \right\vert ^{2}=1
\end{align*}
(since $\left(  v_{1},v_{2},\ldots,v_{n}\right)  $ is orthonormal and thus
$\left\vert \left\vert v_{i}\right\vert \right\vert =1$), and thus $\left\vert
\left\vert u_{i}\right\vert \right\vert =1$. Hence, the orthogonal tuple
$\left(  u_{1},u_{2},\ldots,u_{k}\right)  $ is orthonormal.
\end{itemize}

So $\left(  u_{1},u_{2},\ldots,u_{k}\right)  $ is an orthonormal tuple of
vectors in $\mathbb{C}^{m}$. Hence, Corollary \ref{cor.unitary.orthon-extend}
shows that we can extend this tuple to an orthonormal basis $\left(
u_{1},u_{2},\ldots,u_{m}\right)  $ of $\mathbb{C}^{m}$ by appending $m-k$ new
(appropriately chosen) vectors $u_{k+1},u_{k+2},\ldots,u_{m}$. Let us consider
the orthonormal basis $\left(  u_{1},u_{2},\ldots,u_{m}\right)  $ of
$\mathbb{C}^{m}$ obtained in this way. Let $U\in\mathbb{C}^{m\times m}$ be the
$m\times m$-matrix with columns $u_{1},u_{2},\ldots,u_{m}$. Thus, the columns
of this matrix $U$ form an orthonormal basis of $\mathbb{C}^{m}$; hence, $U$
is a unitary matrix (by the implication $\mathcal{E}\Longrightarrow
\mathcal{A}$ of Theorem \ref{thm.unitary.unitary.eqs}).

\begin{noncompile}
Set $\sigma_{i}:=0$ for every $i>n$. Thus, $\sigma_{i}$ is defined for each
positive integer $i$ (not just for $i\in\left[  n\right]  $). Note that
$\sigma_{i}=0$ for every $i>k$ (since we already have $\sigma_{k+1}%
=\sigma_{k+2}=\cdots=\sigma_{n}=0$ and have furthermore set $\sigma_{i}=0$ for
every $i>n$).
\end{noncompile}

Let%
\[
\Sigma:=\left(
\begin{array}
[c]{ccccccc}%
\sigma_{1} & 0 & \cdots & 0 & 0 & \cdots & 0\\
0 & \sigma_{2} & \cdots & 0 & 0 & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & \sigma_{k} & 0 & \cdots & 0\\
0 & 0 & \cdots & 0 & 0 & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & 0 & 0 & \cdots & 0
\end{array}
\right)  \in\mathbb{C}^{m\times n}.
\]
(This is the $m\times n$-matrix whose $\left(  i,i\right)  $-th entries are
$\sigma_{i}$ for all $i\in\left[  k\right]  $, and whose all remaining entries
are $0$.) Clearly, this matrix $\Sigma$ is pseudodiagonal. Moreover, this
matrix $\Sigma$ has exactly $k$ nonzero diagonal entries (since
(\ref{pf.thm.svd.svd.pos}) shows that $\sigma_{1},\sigma_{2},\ldots,\sigma
_{k}$ are nonzero). All its diagonal entries are nonnegative reals.

Now, we claim that $A=U\Sigma V^{\ast}$. To prove this, we shall first show
that $AV=U\Sigma$.

It is sufficient to prove that\footnote{Recall that the notation
$M_{\bullet,j}$ means the $j$-th column of a matrix $M$.}%
\[
\left(  AV\right)  _{\bullet,j}=\left(  U\Sigma\right)  _{\bullet
,j}\ \ \ \ \ \ \ \ \ \ \text{for each }j\in\left[  n\right]  .
\]
So let us fix $j\in\left[  n\right]  $ and try to prove that $\left(
AV\right)  _{\bullet,j}=\left(  U\Sigma\right)  _{\bullet,j}$. We note that
$V_{\bullet,j}=v_{j}$ (since the columns of $V$ are $v_{1},v_{2},\ldots,v_{n}%
$) and $U_{\bullet,j}=u_{j}$ (since the columns of $U$ are $u_{1},u_{2}%
,\ldots,u_{m}$). We distinguish between the cases $j\leq k$ and $j>k$:

\begin{itemize}
\item Assume that $j\leq k$. Then, by the rules for multiplying matrices, we
have%
\begin{align}
\left(  AV\right)  _{\bullet,j}  &  =A\underbrace{V_{\bullet,j}}_{=v_{j}%
}=Av_{j}=\sigma_{j}\underbrace{u_{j}}_{\substack{=U_{\bullet,j}}%
}\ \ \ \ \ \ \ \ \ \ \left(  \text{since }u_{j}=\dfrac{1}{\sigma_{j}}%
Av_{j}\right) \nonumber\\
&  =\sigma_{j}U_{\bullet,j}. \label{pf.thm.svd.svd.AV=US.c1.1}%
\end{align}
On the other hand, $j\leq k$ shows that the $j$-th column of the matrix
$\Sigma$ has a $\sigma_{j}$ in its $j$-th position and zeroes in all other
positions. In other words, this column equals $\sigma_{j}e_{j}$ (where
$\left(  e_{1},e_{2},\ldots,e_{m}\right)  $ denotes the standard basis of
$\mathbb{C}^{m}$). In other words, $\Sigma_{\bullet,j}=\sigma_{j}e_{j}$ (since
$\Sigma_{\bullet,j}$ is the $j$-th column of $\Sigma$). Now, by the rules for
multiplying matrices, we have%
\[
\left(  U\Sigma\right)  _{\bullet,j}=U\underbrace{\Sigma_{\bullet,j}}%
_{=\sigma_{j}e_{j}}=U\cdot\sigma_{j}e_{j}=\sigma_{j}\underbrace{Ue_{j}%
}_{\substack{=U_{\bullet,j}\\\text{(since multiplying a matrix by }e_{j}\text{
always}\\\text{produces the }j\text{-th column of the matrix)}}}=\sigma
_{j}U_{\bullet,j}.
\]
Comparing this with (\ref{pf.thm.svd.svd.AV=US.c1.1}), we obtain $\left(
AV\right)  _{\bullet,j}=\left(  U\Sigma\right)  _{\bullet,j}$.

Thus, $\left(  AV\right)  _{\bullet,j}=\left(  U\Sigma\right)  _{\bullet,j}$
is proved in the case when $j\leq k$.

\item Now, assume that $j>k$. Then, $\sigma_{j}=0$ (by
(\ref{pf.thm.svd.svd.zero})), so that $\lambda_{j}=0$ (since the definition of
$\sigma_{j}$ yields $\sigma_{j}=\sqrt{\lambda_{j}}$, so that $\lambda
_{j}=\sigma_{j}^{2}$). However, applying (\ref{pf.thm.svd.svd.evec}) to $i=j$,
we obtain%
\[
A^{\ast}Av_{j}=\underbrace{\lambda_{j}}_{\substack{=0\\\text{(since
}j>k\text{)}}}v_{j}=0,
\]
so that $v_{j}\in\operatorname*{Ker}\left(  A^{\ast}A\right)
=\operatorname*{Ker}A$ (by Proposition \ref{prop.SVD.A*A} \textbf{(b)}). Now,
by the rules for multiplying matrices, we have%
\[
\left(  AV\right)  _{\bullet,j}=A\underbrace{V_{\bullet,j}}_{\substack{=v_{j}%
}}=Av_{j}=0\ \ \ \ \ \ \ \ \ \ \left(  \text{since }v_{j}\in
\operatorname*{Ker}A\right)  .
\]
Comparing this with%
\[
\left(  U\Sigma\right)  _{\bullet,j}=U\underbrace{\Sigma_{\bullet,j}%
}_{\substack{=0\\\text{(since }j>k\text{, so that all entries in}\\\text{the
}j\text{-th column of }\Sigma\text{ are }0\text{)}}}=0,
\]
we obtain $\left(  AV\right)  _{\bullet,j}=\left(  U\Sigma\right)
_{\bullet,j}$.

Thus, $\left(  AV\right)  _{\bullet,j}=\left(  U\Sigma\right)  _{\bullet,j}$
is proved in the case when $j>k$.
\end{itemize}

Thus, we have proved $\left(  AV\right)  _{\bullet,j}=\left(  U\Sigma\right)
_{\bullet,j}$ in both cases.

Forget that we fixed $j$. We thus have shown that $\left(  AV\right)
_{\bullet,j}=\left(  U\Sigma\right)  _{\bullet,j}$ for each $j\in\left[
n\right]  $. In other words, each column of the matrix $AV$ equals the
corresponding column of the matrix $U\Sigma$. Hence, $AV=U\Sigma$. Therefore,%
\[
\underbrace{U\Sigma}_{=AV}V^{\ast}=A\underbrace{VV^{\ast}}_{\substack{=I_{n}%
\\\text{(since }V\text{ is unitary)}}}=A,
\]
so that $A=U\Sigma V^{\ast}$, as desired.

This proves parts \textbf{(a)}, \textbf{(c)} and \textbf{(d)} of Theorem
\ref{thm.svd.svd}. \medskip

\textbf{(b)} We must show that if $P$ is a pseudodiagonal matrix such that all
diagonal entries of $P$ are nonnegative reals, and such that $A$ is unitarily
equivalent to $P$, then $P$ and $\Sigma$ have the same diagonal entries up to order.

Before we prove this, let us show two auxiliary results:

\begin{statement}
\textit{Claim 1:} Let $X\in\mathbb{C}^{m\times n}$ and $Y\in\mathbb{C}%
^{m\times n}$ be two unitarily equivalent matrices. Then, $X$ and $Y$ have the
same singular values\footnote{Recall that the \emph{singular values} of a
matrix $X$ are defined to be the square roots of the eigenvalues of $X^{\ast
}X$.}.
\end{statement}

[\textit{Proof of Claim 1:} Since $X$ and $Y$ are unitarily equivalent, there
exist two unitary matrices $U\in\operatorname*{U}\nolimits_{m}\left(
\mathbb{C}\right)  $ and $V\in\operatorname*{U}\nolimits_{n}\left(
\mathbb{C}\right)  $ such that $X=UYV^{\ast}$. (These $U$ and $V$ have nothing
to do with the $U$ and $V$ from Theorem \ref{thm.svd.svd}.) Consider these $U$
and $V$. From $X=UYV^{\ast}$, we obtain%
\[
X^{\ast}X=\underbrace{\left(  UYV^{\ast}\right)  ^{\ast}}_{=\left(  V^{\ast
}\right)  ^{\ast}Y^{\ast}U^{\ast}}\left(  UYV^{\ast}\right)
=\underbrace{\left(  V^{\ast}\right)  ^{\ast}}_{=V}Y^{\ast}\underbrace{U^{\ast
}U}_{\substack{=I_{m}\\\text{(since }U\text{ is unitary)}}}YV^{\ast}=VY^{\ast
}YV^{\ast}.
\]
This shows that $X^{\ast}X\overset{\operatorname*{us}}{\sim}Y^{\ast}Y$ (since
$V$ is unitary). Therefore, $X^{\ast}X\sim Y^{\ast}Y$ (by Proposition
\ref{prop.schurtri.unisim.sim}). Thus, the matrices $X^{\ast}X$ and $Y^{\ast
}Y$ have the same eigenvalues (by Proposition \ref{prop.schurtri.similar.same}
\textbf{(e)}). Therefore, $X$ and $Y$ have the same singular values (because
the singular values of $X$ are defined as the square roots of the eigenvalues
of $X^{\ast}X$, and likewise for $Y$). This proves Claim 1.] \medskip

\begin{statement}
\textit{Claim 2:} Let $D\in\mathbb{C}^{m\times n}$ be a pseudodiagonal matrix.
Then, the nonzero singular values of $D$ are the absolute values of the
nonzero diagonal entries of $D$.
\end{statement}

[\textit{Proof of Claim 2:} We WLOG assume that $m\leq n$, since the case
$m>n$ is similar but easier. Let $d_{1},d_{2},\ldots,d_{m}$ be the diagonal
entries of $D$. Then,%
\[
D=\left(
\begin{array}
[c]{ccccccc}%
d_{1} & 0 & \cdots & 0 & 0 & \cdots & 0\\
0 & d_{2} & \cdots & 0 & 0 & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & d_{m} & 0 & \cdots & 0
\end{array}
\right)  .
\]
Hence, it is easy to check that%
\begin{align*}
D^{\ast}D  &  =\left(
\begin{array}
[c]{ccccccc}%
\overline{d_{1}}d_{1} & 0 & \cdots & 0 & 0 & \cdots & 0\\
0 & \overline{d_{2}}d_{2} & \cdots & 0 & 0 & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & \overline{d_{m}}d_{m} & 0 & \cdots & 0\\
0 & 0 & \cdots & 0 & 0 & \cdots & 0\\
\vdots & \vdots & \ddots & \vdots & \vdots & \ddots & \vdots\\
0 & 0 & \cdots & 0 & 0 & \cdots & 0
\end{array}
\right) \\
&  =\operatorname*{diag}\left(  \overline{d_{1}}d_{1},\overline{d_{2}}%
d_{2},\ldots,\overline{d_{m}}d_{m},\underbrace{0,0,\ldots,0}_{n-m\text{
entries}}\right) \\
&  =\operatorname*{diag}\left(  \left\vert d_{1}\right\vert ^{2},\left\vert
d_{2}\right\vert ^{2},\ldots,\left\vert d_{m}\right\vert ^{2}%
,\underbrace{0,0,\ldots,0}_{n-m\text{ entries}}\right)  .
\end{align*}
Therefore, the eigenvalues of $D^{\ast}D$ are $\left\vert d_{1}\right\vert
^{2},\left\vert d_{2}\right\vert ^{2},\ldots,\left\vert d_{m}\right\vert
^{2},\underbrace{0,0,\ldots,0}_{n-m\text{ entries}}$ (since the eigenvalues of
a diagonal matrix are its diagonal entries). Hence, the singular values of $D$
are $\left\vert d_{1}\right\vert ,\left\vert d_{2}\right\vert ,\ldots
,\left\vert d_{m}\right\vert ,\underbrace{0,0,\ldots,0}_{n-m\text{ entries}}$
(since the singular values of $D$ are defined as the square roots of the
eigenvalues of $D^{\ast}D$). Thus, the nonzero singular values of $D$ are the
nonzero numbers among $\left\vert d_{1}\right\vert ,\left\vert d_{2}%
\right\vert ,\ldots,\left\vert d_{m}\right\vert $. In other words, they are
the absolute values of the nonzero diagonal entries of $D$. This proves Claim
2.] \medskip

Now, we can prove the claim we intended to prove. Let $P$ be a pseudodiagonal
matrix such that all diagonal entries of $P$ are nonnegative reals, and such
that $A$ is unitarily equivalent to $P$. As we recall, our goal is to prove
that $P$ and $\Sigma$ have the same diagonal entries up to order.

We know that the matrix $A$ is unitarily equivalent to $\Sigma$ and also to
$P$. Thus, $\Sigma$ is unitarily equivalent to $P$ (because unitary
equivalence is an equivalence relation). Therefore, Claim 1 (applied to
$X=\Sigma$ and $Y=P$) shows that the matrices $\Sigma$ and $P$ have the same
singular values. However, these two matrices are pseudodiagonal; thus, Claim 2
shows that their nonzero singular values are the absolute values of their
nonzero diagonal entries. Since their diagonal entries are nonnegative reals,
we can actually drop the \textquotedblleft absolute values\textquotedblright%
\ part from this sentence, and conclude that their nonzero singular values are
simply their nonzero diagonal entries. Thus, the matrices $\Sigma$ and $P$
have the same nonzero diagonal entries (because we have shown that they have
the same singular values). Therefore, the matrices $\Sigma$ and $P$ have the
same diagonal entries (since they have the same dimensions and therefore the
same number of diagonal entries). Thus, we have shown that the matrices $P$
and $\Sigma$ have the same diagonal entries up to order. This completes the
proof of Theorem \ref{thm.svd.svd} \textbf{(b)}.
\end{proof}

\begin{example}
Let $A=\left(
\begin{array}
[c]{ccc}%
3 & 2 & 2\\
2 & 3 & -2
\end{array}
\right)  \in\mathbb{C}^{2\times3}$. How do we find an SVD of $A$ ?

This is not the way SVDs are computed in practice, but we can try following
our above proof of Theorem \ref{thm.svd.svd}. Thus, we compute a spectral
decomposition of $A^{\ast}A$. (Since $A^{\ast}A$ is Hermitian, this is
equivalent to diagonalizing $A^{\ast}A$.) A simple computation yields that
\[
A^{\ast}A=\left(
\begin{array}
[c]{ccc}%
13 & 12 & 2\\
12 & 13 & -2\\
2 & -2 & 8
\end{array}
\right)
\]
and that $A^{\ast}A$ has eigenvalues $25,9,0$ and a spectral decomposition
$\left(  V,D\right)  $ with%
\[
V=\left(
\begin{array}
[c]{ccc}%
\sqrt{2}/2 & \sqrt{2}/6 & -2/3\\
\sqrt{2}/2 & -\sqrt{2}/6 & 2/3\\
0 & 2\sqrt{2}/3 & 1/3
\end{array}
\right)  \ \ \ \ \ \ \ \ \ \ \text{and}%
\ \ \ \ \ \ \ \ \ \ D=\operatorname*{diag}\left(  25,9,0\right)  .
\]
(Of course, we have handpicked $A$ to make the eigenvalues integers; a random
$A$ would give rise to irrational eigenvalues.) Thus, $\lambda_{1}=25$ and
$\lambda_{2}=9$ and $\lambda_{3}=0$ and $k=2$. Hence, $\sigma_{1}=\sqrt{25}=5$
and $\sigma_{2}=\sqrt{9}=3$, so that $\Sigma=\left(
\begin{array}
[c]{ccc}%
5 & 0 & 0\\
0 & 3 & 0
\end{array}
\right)  $. It remains to find $U$. To do so, we set $u_{j}:=\dfrac{1}%
{\sigma_{j}}Av_{j}$ for all $j\in\left[  k\right]  $; thus,%
\begin{align*}
u_{1}  &  =\dfrac{1}{\sigma_{1}}Av_{1}=\dfrac{1}{5}\left(
\begin{array}
[c]{ccc}%
3 & 2 & 2\\
2 & 3 & -2
\end{array}
\right)  \left(
\begin{array}
[c]{c}%
\sqrt{2}/2\\
\sqrt{2}/2\\
0
\end{array}
\right)  =\left(
\begin{array}
[c]{c}%
\sqrt{2}/2\\
\sqrt{2}/2
\end{array}
\right)  \ \ \ \ \ \ \ \ \ \ \text{and}\\
u_{2}  &  =\dfrac{1}{\sigma_{2}}Av_{2}=\dfrac{1}{3}\left(
\begin{array}
[c]{ccc}%
3 & 2 & 2\\
2 & 3 & -2
\end{array}
\right)  \left(
\begin{array}
[c]{c}%
\sqrt{2}/6\\
-\sqrt{2}/6\\
2\sqrt{2}/3
\end{array}
\right)  =\left(
\begin{array}
[c]{c}%
\sqrt{2}/2\\
-\sqrt{2}/2
\end{array}
\right)  .
\end{align*}
The proof of Theorem \ref{thm.svd.svd} tells us to extend this orthonormal
tuple $\left(  u_{1},u_{2},\ldots,u_{k}\right)  $ to an orthonormal basis
$\left(  u_{1},u_{2},\ldots,u_{m}\right)  $ of $\mathbb{C}^{m}$, but this is
unnecessary here, since it already is a basis (since $k=m$). Thus, we can now
compute $U$ as the matrix with columns $u_{1},u_{2},\ldots,u_{m}$; that is,
$U=\left(
\begin{array}
[c]{cc}%
\sqrt{2}/2 & \sqrt{2}/2\\
\sqrt{2}/2 & -\sqrt{2}/2
\end{array}
\right)  $. Hence, we obtain the SVD $\left(  U,V,\Sigma\right)  $ of $A$ with%
\begin{align*}
U  &  =\left(
\begin{array}
[c]{cc}%
\sqrt{2}/2 & \sqrt{2}/2\\
\sqrt{2}/2 & -\sqrt{2}/2
\end{array}
\right)  ,\ \ \ \ \ \ \ \ \ \ V=\left(
\begin{array}
[c]{ccc}%
\sqrt{2}/2 & \sqrt{2}/6 & -2/3\\
\sqrt{2}/2 & -\sqrt{2}/6 & 2/3\\
0 & 2\sqrt{2}/3 & 1/3
\end{array}
\right)  ,\\
\Sigma &  =\left(
\begin{array}
[c]{ccc}%
5 & 0 & 0\\
0 & 3 & 0
\end{array}
\right)  .
\end{align*}

\end{example}

\begin{exercise}
\fbox{3} Find an SVD of the matrix $A:=\left(
\begin{array}
[c]{ccc}%
1 & 0 & 1\\
0 & 1 & 0
\end{array}
\right)  $.
\end{exercise}

\begin{exercise}
\fbox{4} Find an SVD of the matrix $A:=\left(
\begin{array}
[c]{ccc}%
1 & 0 & 1\\
0 & 1 & 0\\
-1 & 0 & 1
\end{array}
\right)  $.
\end{exercise}

\begin{exercise}
\fbox{3} Let $\left(  U,V,\Sigma\right)  $ be an SVD of a matrix
$A\in\mathbb{C}^{m\times n}$. \medskip

\textbf{(a)} Construct an SVD of the matrix $A^{\ast}\in\mathbb{C}^{n\times
m}$. \medskip

\textbf{(b)} Now assume that $A$ is invertible (so that $m=n$). Construct an
SVD of the matrix $A^{-1}$.
\end{exercise}

\begin{exercise}
\fbox{3} Let $A$ and $B$ be two $m\times n$-matrices. Prove that $A$ and $B$
are unitarily equivalent if and only if the matrices $A^{\ast}A$ and $B^{\ast
}B$ are unitarily similar.
\end{exercise}

A variant of the SVD is the so-called \emph{compact SVD}, in which the unitary
matrices $U$ and $V$ are replaced by isometries and the pseudodiagonal matrix
$\Sigma$ is replaced by a diagonal $k\times k$-matrix for
$k=\operatorname*{rank}A$:

\begin{corollary}
\label{cor.svd.compact-svd}Let $A\in\mathbb{C}^{m\times n}$. Let
$k=\operatorname*{rank}A$. Then: \medskip

\textbf{(a)} There exist isometries $U\in\mathbb{C}^{m\times k}$ and
$V\in\mathbb{C}^{n\times k}$ and a diagonal matrix $\Sigma\in\mathbb{C}%
^{k\times k}$ such that all diagonal entries of $\Sigma$ are positive reals
and such that%
\[
A=U\Sigma V^{\ast}.
\]


\textbf{(b)} The matrix $\Sigma$ is unique up to permutation of its diagonal
entries. (The matrices $U$ and $V$ are usually not unique.) \medskip

\textbf{(c)} Let $\sigma_{1},\sigma_{2},\ldots,\sigma_{n}$ be the square roots
of the $n$ eigenvalues of the Hermitian matrix $A^{\ast}A$, listed in
decreasing order (so that $\sigma_{1}\geq\sigma_{2}\geq\cdots\geq\sigma_{n}$).
Then, we have $\sigma_{k+1}=\sigma_{k+2}=\cdots=\sigma_{n}=0$, and we can
take
\[
\Sigma=\operatorname*{diag}\left(  \sigma_{1},\sigma_{2},\ldots,\sigma
_{k}\right)  \in\mathbb{C}^{k\times k}%
\]
in part \textbf{(a)}.
\end{corollary}

\begin{exercise}
\fbox{4} Prove Corollary \ref{cor.svd.compact-svd}.
\end{exercise}

\begin{exercise}
\fbox{4} Give a simple algorithm (without using eigenvalues or spectral
decomposition) to compute a compact SVD of a given rank-$1$ matrix.
\end{exercise}

\section{Positive and nonnegative matrices (\cite[Chapter 8]{HorJoh13})}

\subsection{Basics}

Recall the triangle inequality:

\begin{proposition}
[triangle inequality]\label{prop.ineq.triangle-CC}Let $z_{1},z_{2}%
,\ldots,z_{n}$ be $n$ complex numbers. Then: \medskip

\textbf{(a)} We have the inequality%
\[
\left\vert z_{1}\right\vert +\left\vert z_{2}\right\vert +\cdots+\left\vert
z_{n}\right\vert \geq\left\vert z_{1}+z_{2}+\cdots+z_{n}\right\vert .
\]


\textbf{(b)} Equality holds in this inequality if and only if $z_{1}%
,z_{2},\ldots,z_{n}$ have the same argument (i.e., there exists some
$w\in\mathbb{C}$ such that $z_{1},z_{2},\ldots,z_{n}$ are nonnegative real
multiples of $w$).
\end{proposition}

\begin{definition}
\label{def.posmat.pos-nonneg}Let $A\in\mathbb{C}^{n\times m}$ be a matrix.
\medskip

\textbf{(a)} We say that $A$ is \emph{positive} (and write $A>0$) if all
entries of $A$ are positive reals. \medskip

\textbf{(b)} We say that $A$ is \emph{nonnegative} (and write $A\geq0$) if all
entries of $A$ are nonnegative reals. \medskip

\textbf{(c)} We let $\left\vert A\right\vert \in\mathbb{R}^{n\times m}$ be the
nonnegative matrix obtained by replacing each entry of $A$ by its absolute
value. In other words,%
\[
\left\vert A\right\vert :=\left(
\begin{array}
[c]{cccc}%
\left\vert A_{1,1}\right\vert  & \left\vert A_{1,2}\right\vert  & \cdots &
\left\vert A_{1,m}\right\vert \\
\left\vert A_{2,1}\right\vert  & \left\vert A_{2,2}\right\vert  & \cdots &
\left\vert A_{2,m}\right\vert \\
\vdots & \vdots & \ddots & \vdots\\
\left\vert A_{n,1}\right\vert  & \left\vert A_{n,2}\right\vert  & \cdots &
\left\vert A_{n,m}\right\vert
\end{array}
\right)  .
\]

\end{definition}

\begin{remark}
Recall that row vectors and column vectors are matrices. Thus, the statements
\textquotedblleft$v>0$\textquotedblright\ and \textquotedblleft$v\geq
0$\textquotedblright\ and the notation $\left\vert v\right\vert $ are defined
for them as well. If $v=\left(  v_{1},v_{2},\ldots,v_{k}\right)  ^{T}$, then
$\left\vert v\right\vert =\left(  \left\vert v_{1}\right\vert ,\left\vert
v_{2}\right\vert ,\ldots,\left\vert v_{k}\right\vert \right)  ^{T}$.
\end{remark}

\begin{warning}
Do not mistake $\left\vert v\right\vert $ (a vector) for $\left\vert
\left\vert v\right\vert \right\vert $ (a number). Also, when $A$ is a matrix,
do not mistake $\left\vert A\right\vert $ for (an old notation for) the
determinant of $A$. (We always write $\det A$ for the determinant of $A$, so
this confusion should not arise.)
\end{warning}

Let us stress once again that positive matrices and nonnegative matrices are
required to have real entries by definition.

\begin{exercise}
\fbox{1} Let $v\in\mathbb{C}^{m}$ be a column vector. Prove that $\left\vert
\left\vert \left\vert v\right\vert \right\vert \right\vert =\left\vert
\left\vert v\right\vert \right\vert $, where the left hand side means the
length of $\left\vert v\right\vert $.
\end{exercise}

\begin{exercise}
\label{exe.posmat.abs-lamA}\fbox{1} Let $\lambda\in\mathbb{C}$ and
$A\in\mathbb{C}^{n\times m}$. Prove that $\left\vert \lambda A\right\vert
=\left\vert \lambda\right\vert \cdot\left\vert A\right\vert $.
\end{exercise}

\begin{proposition}
A matrix $A\in\mathbb{C}^{n\times m}$ is nonnegative if and only if
$\left\vert A\right\vert =A$.
\end{proposition}

\begin{proof}
$\Longrightarrow:$ If $A$ is nonnegative, then each $i$ and $j$ satisfy
$A_{i,j}\geq0$ and thus $\left\vert A_{i,j}\right\vert =A_{i,j}$; therefore,
$\left\vert A\right\vert =A$.

$\Longleftarrow:$ If $\left\vert A\right\vert =A$, then $A$ is nonnegative
(since $\left\vert A\right\vert $ is always nonnegative).
\end{proof}

\begin{definition}
\label{def.posmat.ineqs}Let $A,B\in\mathbb{R}^{n\times m}$ be two matrices
with real entries. Then: \medskip

\textbf{(a)} We say that $A\geq B$ if and only if $A-B\geq0$ (or,
equivalently, $A_{i,j}\geq B_{i,j}$ for all $i\in\left[  n\right]  $ and
$j\in\left[  m\right]  $). \medskip

\textbf{(b)} We say that $A>B$ if and only if $A-B>0$ (or, equivalently,
$A_{i,j}>B_{i,j}$ for all $i\in\left[  n\right]  $ and $j\in\left[  m\right]
$). \medskip

\textbf{(c)} We say that $A\leq B$ if and only if $A-B\leq0$ (or,
equivalently, $A_{i,j}\leq B_{i,j}$ for all $i\in\left[  n\right]  $ and
$j\in\left[  m\right]  $). \medskip

\textbf{(d)} We say that $A<B$ if and only if $A-B<0$ (or, equivalently,
$A_{i,j}<B_{i,j}$ for all $i\in\left[  n\right]  $ and $j\in\left[  m\right]
$).
\end{definition}

\begin{example}
We have $\left(
\begin{array}
[c]{cc}%
1 & 2\\
3 & 4
\end{array}
\right)  \geq\left(
\begin{array}
[c]{cc}%
0 & 2\\
2 & 4
\end{array}
\right)  $.
\end{example}

The relations $\geq$, $>$, $\leq$ and $<$ are known as \emph{entrywise
inequalities} (specifically, \textquotedblleft entrywise greater or
equal\textquotedblright, \textquotedblleft entrywise greater\textquotedblright%
, etc.), since they are just saying that each entry of $A$ is $\geq$, $>$,
$\leq$ or $<$ to the corresponding entry of $B$.

\begin{remark}
Again, recall that row vectors and column vectors are matrices too; thus,
Definition \ref{def.posmat.ineqs} applies to them as well.
\end{remark}

\begin{proposition}
\textbf{(a)} The relations $\geq$, $>$, $\leq$ and $<$ on $\mathbb{R}^{n\times
m}$ (introduced in Definition \ref{def.posmat.ineqs}) are transitive. \medskip

\textbf{(b)} The relations $\geq$ and $\leq$ are reflexive and antisymmetric
(so they are weak partial orders on $\mathbb{R}^{n\times m}$). \medskip

\textbf{(c)} Let $A$ and $B$ be two matrices in $\mathbb{R}^{n\times m}$.
Then, the implications $\left(  A>B\right)  \ \Longrightarrow\ \left(  A\geq
B\right)  $ and $\left(  A<B\right)  \ \Longrightarrow\ \left(  A\leq
B\right)  $ as well as the equivalences $\left(  A>B\right)
\ \Longleftrightarrow\ \left(  B<A\right)  $ and $\left(  A\geq B\right)
\ \Longleftrightarrow\ \left(  B\leq A\right)  $ hold.
\end{proposition}

\begin{proof}
All of these are straightforward, since the relations $\geq$, $>$, $\leq$ and
$<$ are just entrywise inequalities.
\end{proof}

\begin{warning}
The relations $\geq$ and $\leq$ are not total orders (unless $n\leq1$). For
instance, the row vector $\left(  2,1\right)  $ is neither $\geq$ nor $\leq$
to $\left(  3,0\right)  $.
\end{warning}

\begin{warning}
Do not mistake the relation $\geq$ on column vectors for the relation
$\succcurlyeq$ (majorization).
\end{warning}

\begin{warning}
The trivial vector $v=\left(  {}\right)  \in\mathbb{R}^{0}$ (with no entries
at all) satisfies $v>v$ and $v<v$ and $v\geq v$ and $v\leq v$, because the
\textquotedblleft for all\textquotedblright\ statements in Definition
\ref{def.posmat.ineqs} are vacuously true. However, this is the only case in
which a vector $v$ satisfies both $v>v$ and $v\leq v$.
\end{warning}

\begin{warning}
Given two matrices $A$ and $B$, the relation $A\geq B$ is \textbf{not}
equivalent to \textquotedblleft$A>B$ or $A=B$\textquotedblright. For example,
$\left(  3,1\right)  \geq\left(  2,1\right)  $ is true, but we have neither
$\left(  2,1\right)  >\left(  3,1\right)  $ nor $\left(  2,1\right)  =\left(
3,1\right)  $.
\end{warning}

\begin{exercise}
\fbox{1} Let $A\in\mathbb{C}^{n\times n}$ be a doubly stochastic matrix (see
Definition \ref{def.major.doublestoch} for the meaning of this). Let $J$ be
the $n\times n$-matrix whose all entries equal $1$. Prove that $J\geq A\geq0$.
\end{exercise}

Two complex numbers $z$ and $w$ always satisfy $\left\vert z\right\vert
\cdot\left\vert w\right\vert =\left\vert zw\right\vert $. For two matrices,
however, this equality is not usually satisfied; however, it survives as an inequality:

\begin{proposition}
\label{prop.posmat.prod-geq}Let $A\in\mathbb{C}^{n\times m}$ and
$B\in\mathbb{C}^{m\times p}$ be two matrices. Then,%
\[
\left\vert A\right\vert \cdot\left\vert B\right\vert \geq\left\vert
AB\right\vert .
\]

\end{proposition}

\begin{proof}
We must prove that $\left(  \left\vert A\right\vert \cdot\left\vert
B\right\vert \right)  _{i,k}\geq\left\vert AB\right\vert _{i,k}$ for all
$i\in\left[  n\right]  $ and $k\in\left[  p\right]  $.

So let $i\in\left[  n\right]  $ and $k\in\left[  p\right]  $. Then, the
definition of the product of two matrices yields
\[
\left(  \left\vert A\right\vert \cdot\left\vert B\right\vert \right)
_{i,k}=\sum_{j=1}^{m}\underbrace{\left\vert A\right\vert _{i,j}}_{=\left\vert
A_{i,j}\right\vert }\cdot\underbrace{\left\vert B\right\vert _{j,k}%
}_{=\left\vert B_{j,k}\right\vert }=\sum_{j=1}^{m}\underbrace{\left\vert
A_{i,j}\right\vert \cdot\left\vert B_{j,k}\right\vert }_{=\left\vert
A_{i,j}B_{j,k}\right\vert }=\sum_{j=1}^{m}\left\vert A_{i,j}B_{j,k}\right\vert
\geq\left\vert \sum_{j=1}^{m}A_{i,j}B_{j,k}\right\vert
\]
(by the triangle inequality). In view of%
\[
\left\vert AB\right\vert _{i,k}=\left\vert \left(  AB\right)  _{i,k}%
\right\vert =\left\vert \sum_{j=1}^{m}A_{i,j}B_{j,k}\right\vert ,
\]
we can rewrite this as $\left(  \left\vert A\right\vert \cdot\left\vert
B\right\vert \right)  _{i,k}\geq\left\vert AB\right\vert _{i,k}$, qed.
\end{proof}

\begin{corollary}
\label{cor.posmat.pow-geq}Let $A\in\mathbb{C}^{n\times n}$ and $k\in
\mathbb{N}$. Then, $\left\vert A\right\vert ^{k}\geq\left\vert A^{k}%
\right\vert $.
\end{corollary}

\begin{proof}
Induction on $k$, using Proposition \ref{prop.posmat.prod-geq} (and the fact
that $\left\vert I_{n}\right\vert =I_{n}$).
\end{proof}

It is not easy to characterize when the inequality in Proposition
\ref{prop.posmat.prod-geq} becomes an equality. However, conclusions can be
drawn in some cases. The following proposition considers the case when the
matrix $B$ is a column vector (which we call $x$ to avoid unusual notations):

\begin{proposition}
\label{prop.posmat.prod-geq-Ax}Let $A\in\mathbb{C}^{n\times m}$ and
$x\in\mathbb{C}^{m}$. Then: \medskip

\textbf{(a)} We have $\left\vert A\right\vert \cdot\left\vert x\right\vert
\geq\left\vert Ax\right\vert $. \medskip

\textbf{(b)} If at least one row of $A$ is positive and we have $A\geq0$ and
$\left\vert Ax\right\vert =A\cdot\left\vert x\right\vert $, then $\left\vert
x\right\vert =\omega x$ for some $\omega\in\mathbb{C}$ satisfying $\left\vert
\omega\right\vert =1$. \medskip

\textbf{(c)} If $x>0$ and $Ax=\left\vert A\right\vert x$, then $A=\left\vert
A\right\vert $ (so that $A\geq0$).
\end{proposition}

\begin{proof}
\textbf{(a)} follows from Proposition \ref{prop.posmat.prod-geq}. \medskip

\textbf{(b)} Assume that at least one row of $A$ is positive and we have
$A\geq0$ and $\left\vert Ax\right\vert =A\cdot\left\vert x\right\vert $.

We have assumed that at least one row of $A$ is positive. Let the $i$-th row
of $A$ be positive. Thus, the numbers $A_{i,j}$ are positive reals for all
$j\in\left[  m\right]  $.

Write $x=\left(  x_{1},x_{2},\ldots,x_{m}\right)  ^{T}$. Thus, $\left\vert
x\right\vert =\left(  \left\vert x_{1}\right\vert ,\left\vert x_{2}\right\vert
,\ldots,\left\vert x_{m}\right\vert \right)  ^{T}$.

From $\left\vert Ax\right\vert =A\cdot\left\vert x\right\vert $, we obtain%
\begin{align*}
\left(  \text{the }i\text{-th entry of }\left\vert Ax\right\vert \right)   &
=\left(  \text{the }i\text{-th entry of }A\cdot\left\vert x\right\vert
\right)  =\sum_{j=1}^{m}\underbrace{A_{i,j}\cdot\left\vert x_{j}\right\vert
}_{\substack{=\left\vert A_{i,j}x_{j}\right\vert \\\text{(since }A\geq0\text{
and thus }A_{i,j}\geq0\text{)}}}\\
&  =\sum_{j=1}^{m}\left\vert A_{i,j}x_{j}\right\vert ,
\end{align*}
so that
\begin{align*}
\sum_{j=1}^{m}\left\vert A_{i,j}x_{j}\right\vert  &  =\left(  \text{the
}i\text{-th entry of }\left\vert Ax\right\vert \right)  =\left\vert \text{the
}i\text{-th entry of }Ax\right\vert \\
&  =\left\vert \sum_{j=1}^{m}A_{i,j}x_{j}\right\vert
\end{align*}
(since the $i$-th entry of $Ax$ is $\sum_{j=1}^{m}A_{i,j}x_{j}$). This is an
equality case of the triangle inequality. Thus, the complex numbers
$A_{i,j}x_{j}$ for all $j\in\left[  m\right]  $ have the same argument (by
Proposition \ref{prop.ineq.triangle-CC} \textbf{(b)}). In other words, the
numbers $x_{j}$ for all $j\in\left[  m\right]  $ have the same argument (since
all the $A_{i,j}$ are positive reals and thus we have $\arg\left(
A_{i,j}x_{j}\right)  =\arg x_{j}$). Let $\varphi$ be this argument, and let
$\omega:=e^{-i\varphi}$. Then, $\omega$ is a complex number satisfying
$\left\vert \omega\right\vert =1$, and the numbers $\omega x_{1},\omega
x_{2},\ldots,\omega x_{n}$ are nonnegative reals. This shows that $\omega
x\geq0$, so that $\left\vert \omega x\right\vert =\omega x$. However, Exercise
\ref{exe.posmat.abs-lamA} yields $\left\vert \omega x\right\vert
=\underbrace{\left\vert \omega\right\vert }_{=1}\cdot\left\vert x\right\vert
=\left\vert x\right\vert $. Comparing these two equalities, we obtain
$\left\vert x\right\vert =\omega x$. Theorem \ref{prop.posmat.prod-geq-Ax}
\textbf{(b)} is thus proven. \medskip

\textbf{(c)} Suppose $x>0$ and $Ax=\left\vert A\right\vert x$. We must show
that $A=\left\vert A\right\vert $ (so that $A\geq0$).

Write $x=\left(  x_{1},x_{2},\ldots,x_{m}\right)  ^{T}$. Thus, $x_{1}%
,x_{2},\ldots,x_{m}$ are positive reals (since $x>0$).

Fix $i\in\left[  n\right]  $. Then,%
\[
\left(  \text{the }i\text{-th entry of }Ax\right)  =\left(  \text{the
}i\text{-th entry of }\left\vert A\right\vert x\right)  .
\]
In other words,%
\[
\sum_{j=1}^{m}A_{i,j}x_{j}=\sum_{j=1}^{m}\underbrace{\left\vert A_{i,j}%
\right\vert x_{j}}_{\substack{=\left\vert A_{i,j}x_{j}\right\vert
\\\text{(since }x_{j}\text{ is a positive real)}}}=\sum_{j=1}^{m}\left\vert
A_{i,j}x_{j}\right\vert .
\]
This shows that $\sum_{j=1}^{m}A_{i,j}x_{j}$ is a nonnegative real.
Furthermore, we obtain
\begin{align*}
\sum_{j=1}^{m}A_{i,j}x_{j}  &  =\sum_{j=1}^{m}\left\vert A_{i,j}%
x_{j}\right\vert \geq\left\vert \sum_{j=1}^{m}A_{i,j}x_{j}\right\vert
\ \ \ \ \ \ \ \ \ \ \left(  \text{by the triangle inequality}\right) \\
&  \geq\sum_{j=1}^{m}A_{i,j}x_{j}.
\end{align*}
This is a chain of inequalities in which the first and the last side are
equal. Thus, all inequalities in it must be equalities. In particular, we thus
have equality in the triangle inequality $\sum_{j=1}^{m}\left\vert
A_{i,j}x_{j}\right\vert \geq\left\vert \sum_{j=1}^{m}A_{i,j}x_{j}\right\vert
$. Hence, the complex numbers $A_{i,j}x_{j}$ for all $j\in\left[  m\right]  $
have the same argument (by Proposition \ref{prop.ineq.triangle-CC}
\textbf{(b)}). Their sum $\sum_{j=1}^{m}A_{i,j}x_{j}$ therefore has the same
argument as them; but since we know that this sum $\sum_{j=1}^{m}A_{i,j}x_{j}$
is a nonnegative real, we thus conclude that this common argument is $0$. In
other words, the complex numbers $A_{i,j}x_{j}$ for all $j\in\left[  m\right]
$ are nonnegative reals. Since $x_{1},x_{2},\ldots,x_{m}$ are positive reals,
this means that the $A_{i,j}$ for all $j\in\left[  m\right]  $ are nonnegative
reals. Since we have proved this for all $i\in\left[  n\right]  $, we thus
conclude that all entries of $A$ are nonnegative reals. Hence, $A\geq0$, so
that $A=\left\vert A\right\vert $. This proves Theorem
\ref{prop.posmat.prod-geq-Ax} \textbf{(c)}.
\end{proof}

\begin{proposition}
\label{prop.posmat.ineq-calculus}\textbf{(a)} If $A,B,C,D\in\mathbb{C}%
^{n\times m}$ satisfy $A\leq B$ and $C\leq D$, then $A+C\leq B+D$. \medskip

\textbf{(b)} If $A,B\in\mathbb{C}^{n\times m}$ and $C\in\mathbb{C}^{m\times
p}$ satisfy $A\leq B$ and $0\leq C$, then $AC\leq BC$. \medskip

\textbf{(c)} If $A,B\in\mathbb{C}^{n\times m}$ and $C\in\mathbb{C}^{p\times
n}$ satisfy $A\leq B$ and $0\leq C$, then $CA\leq CB$. \medskip

\textbf{(d)} If $A,B\in\mathbb{C}^{n\times m}$ and $C,D\in\mathbb{C}^{m\times
p}$ satisfy $0\leq A\leq B$ and $0\leq C\leq D$, then $0\leq AC\leq BD$.
\medskip

\textbf{(e)} If $A,B\in\mathbb{C}^{n\times n}$ satisfy $0\leq A\leq B$, and if
$k\in\mathbb{N}$, then $0\leq A^{k}\leq B^{k}$.
\end{proposition}

\begin{proof}
\textbf{(a)} For all $i$ and $j$, we have $A_{i,j}\leq B_{i,j}$ and
$C_{i,j}\leq D_{i,j}$ and therefore $A_{i,j}+C_{i,j}\leq B_{i,j}+D_{i,j}$. But
this means $A+C\leq B+D$. \medskip

\textbf{(b)} Assume $A\leq B$ and $0\leq C$. Let $i\in\left[  n\right]  $ and
$k\in\left[  p\right]  $. The definition of the product of two matrices
yields
\[
\left(  AC\right)  _{i,k}=\sum_{j=1}^{m}A_{i,j}C_{j,k}%
\ \ \ \ \ \ \ \ \ \ \text{with}\ \ \ \ \ \ \ \ \ \ \left(  BC\right)
_{i,k}=\sum_{j=1}^{m}B_{i,j}C_{j,k}.
\]
The right hand side of the first equality is $\leq$ to the right hand side of
the second, because all $j\in\left[  m\right]  $ satisfy $A_{i,j}\leq B_{i,j}$
(since $A\leq B$) and $C_{j,k}\geq0$ (since $0\leq C$). Thus, we obtain
$\left(  AC\right)  _{i,k}\leq\left(  BC\right)  _{i,k}$. Since we have proved
this for all $i$ and $k$, we thus obtain $AC\leq BC$. This proves Proposition
\ref{prop.posmat.ineq-calculus} \textbf{(b)}. \medskip

\textbf{(c)} Similar to \textbf{(b)}. \medskip

\textbf{(d)} Part \textbf{(b)} yields $AC\leq BC$. Part \textbf{(c)} (applied
to $C$, $D$ and $B$ instead of $A$, $B$ and $C$) yields $BC\leq BD$. Since the
relation $\leq$ is transitive, we can conclude $AC\leq BD$ from these two
inequalities. \medskip

\textbf{(e)} Follows from \textbf{(d)} by induction on $k$.
\end{proof}

\begin{exercise}
\fbox{2} Let $n,m,p$ be three positive integers. \medskip

\textbf{(a)} Show that any two positive matrices $A\in\mathbb{R}^{n\times m}$
and $B\in\mathbb{R}^{m\times p}$ satisfy $AB>0$. \medskip

\textbf{(b)} Now, assume that $m>1$. Find an example of two nonzero
nonnegative matrices $A\in\mathbb{R}^{n\times m}$ and $B\in\mathbb{R}^{m\times
p}$ that nevertheless satisfy $AB=0$.
\end{exercise}

\subsection{The spectral radius}

\begin{definition}
\label{def.specrad.specrad}The \emph{spectral radius} $\rho\left(  A\right)  $
of a matrix $A\in\mathbb{C}^{n\times n}$ (with $n>0$) is defined to be the
largest absolute value of an eigenvalue of $A$. That is,%
\[
\rho\left(  A\right)  :=\max\left\{  \left\vert \lambda\right\vert
\ \mid\ \lambda\in\sigma\left(  A\right)  \right\}  .
\]


Note that $\rho\left(  A\right)  $ is always a nonnegative real.
\end{definition}

Some examples:

\begin{itemize}
\item If $A=\operatorname*{diag}\left(  \lambda_{1},\lambda_{2},\ldots
,\lambda_{n}\right)  $, then $\rho\left(  A\right)  =\max\left\{  \left\vert
\lambda_{1}\right\vert ,\left\vert \lambda_{2}\right\vert ,\ldots,\left\vert
\lambda_{n}\right\vert \right\}  $. More generally, this is true if $A$ is a
triangular matrix with diagonal entries $\lambda_{1},\lambda_{2}%
,\ldots,\lambda_{n}$.

\item By Exercise \ref{exe.jnf.step3.An=0} (equivalence $\mathcal{A}%
\Longleftrightarrow\mathcal{C}$), a square matrix $A$ satisfies $\rho\left(
A\right)  =0$ if and only if $A$ is nilpotent.
\end{itemize}

The following is obvious:

\begin{lemma}
\label{lem.posmat.specrad.scale}Let $\lambda\in\mathbb{C}$ and $A\in
\mathbb{C}^{n\times n}$ (where $n>0$). Then, $\rho\left(  \lambda A\right)
=\left\vert \lambda\right\vert \cdot\rho\left(  A\right)  $.
\end{lemma}

It is furthermore easy to see that each $n\times n$-matrix $A\in
\mathbb{C}^{n\times n}$ (with $n>0$) satisfies $\rho\left(  A^{T}\right)
=\rho\left(  A^{\ast}\right)  =\rho\left(  A\right)  $.

\begin{theorem}
\label{thm.posmat.specrad.leq1}Let $A\in\mathbb{C}^{n\times n}$ and
$B\in\mathbb{R}^{n\times n}$ be such that $B\geq\left\vert A\right\vert $ and
$n>0$. Then, $\rho\left(  A\right)  \leq\rho\left(  B\right)  $.
\end{theorem}

\begin{proof}
[Proof of Theorem \ref{thm.posmat.specrad.leq1}.]If $\rho\left(  A\right)
=0$, then this is obvious. So, WLOG assume that $\rho\left(  A\right)  >0$.

We can thus scale both matrices $A$ and $B$ by the positive real $\dfrac
{1}{\rho\left(  A\right)  }$. This does not break the inequality
$B\geq\left\vert A\right\vert $, and also does not break the claim
$\rho\left(  A\right)  \leq\rho\left(  B\right)  $ (by Lemma
\ref{lem.posmat.specrad.scale}).

Thus, we WLOG assume that $\rho\left(  A\right)  =1$. (This is achieved by the
scaling we just mentioned.)

This yields that $A$ has an eigenvalue $\lambda$ with $\left\vert
\lambda\right\vert =1$. Let $\lambda$ be such an eigenvalue, and let $v$ be a
nonzero $\lambda$-eigenvector. Thus, $Av=\lambda v$. Hence,%
\begin{equation}
A^{m}v=\lambda^{m}v\ \ \ \ \ \ \ \ \ \ \text{for any }m\in\mathbb{N}.
\label{pf.thm.posmat.specrad.leq1.Amv=}%
\end{equation}
(This follows easily by induction on $m$.)

Now, we must prove $\rho\left(  A\right)  \leq\rho\left(  B\right)  $. In
other words, we must prove that $1\leq\rho\left(  B\right)  $ (since
$\rho\left(  A\right)  =1$). Assume the contrary. Thus, $\rho\left(  B\right)
<1$. Hence, all eigenvalues of $B$ have absolute value $<1$. Therefore,
Corollary \ref{cor.jnf.powers.to0} (applied to $B$ instead of $A$) shows that
$\lim\limits_{m\rightarrow\infty}B^{m}=0$. Therefore, $\lim
\limits_{m\rightarrow\infty}B^{m}\cdot\left\vert v\right\vert =0$.

However, let $m\in\mathbb{N}$. Then, $B\geq\left\vert A\right\vert \geq0$
entails $B^{m}\geq\left\vert A\right\vert ^{m}$ (by Proposition
\ref{prop.posmat.ineq-calculus} \textbf{(e)}). Also, $\left\vert A\right\vert
^{m}\geq\left\vert A^{m}\right\vert $ (by Corollary \ref{cor.posmat.pow-geq}).
Thus, $B^{m}\geq\left\vert A\right\vert ^{m}\geq\left\vert A^{m}\right\vert $.
Hence, using Proposition \ref{prop.posmat.ineq-calculus} \textbf{(b)}, we
obtain%
\begin{align*}
B^{m}\cdot\left\vert v\right\vert  &  \geq\left\vert A^{m}\right\vert
\cdot\left\vert v\right\vert \ \ \ \ \ \ \ \ \ \ \left(  \text{since
}\left\vert v\right\vert \geq0\right) \\
&  \geq\left\vert A^{m}v\right\vert \ \ \ \ \ \ \ \ \ \ \left(  \text{by
Proposition \ref{prop.posmat.prod-geq}}\right) \\
&  =\left\vert \lambda^{m}v\right\vert \ \ \ \ \ \ \ \ \ \ \left(  \text{by
(\ref{pf.thm.posmat.specrad.leq1.Amv=})}\right) \\
&  =\underbrace{\left\vert \lambda^{m}\right\vert }_{\substack{=\left\vert
\lambda\right\vert ^{m}=1\\\text{(since }\left\vert \lambda\right\vert
=1\text{)}}}\cdot\left\vert v\right\vert \ \ \ \ \ \ \ \ \ \ \left(  \text{by
Exercise \ref{exe.posmat.abs-lamA}}\right) \\
&  =\left\vert v\right\vert .
\end{align*}
Taking limits as $m\rightarrow\infty$, we obtain $\lim\limits_{m\rightarrow
\infty}B^{m}\cdot\left\vert v\right\vert \geq\lim\limits_{m\rightarrow\infty
}\left\vert v\right\vert =\left\vert v\right\vert \neq0$ (since $v$ is
nonzero). This contradicts $\lim\limits_{m\rightarrow\infty}B^{m}%
\cdot\left\vert v\right\vert =0$. This contradiction shows that our assumption
was false, and the proof of Theorem \ref{thm.posmat.specrad.leq1} is complete.
\end{proof}

\begin{corollary}
\label{cor.posmat.specrad.leq1a}Let $A\in\mathbb{C}^{n\times n}$ and
$B\in\mathbb{R}^{n\times n}$ be such that $B\geq\left\vert A\right\vert $ and
$n>0$. Then, $\rho\left(  A\right)  \leq\rho\left(  \left\vert A\right\vert
\right)  \leq\rho\left(  B\right)  $.
\end{corollary}

\begin{proof}
Applying Theorem \ref{thm.posmat.specrad.leq1} to $\left\vert A\right\vert $
instead of $B$, we get $\rho\left(  A\right)  \leq\rho\left(  \left\vert
A\right\vert \right)  $.

Applying Theorem \ref{thm.posmat.specrad.leq1} to $\left\vert A\right\vert $
instead of $A$, we get $\rho\left(  \left\vert A\right\vert \right)  \leq
\rho\left(  B\right)  $ (since $\left\vert \ \left\vert A\right\vert
\ \right\vert =\left\vert A\right\vert $).

Hence, Corollary \ref{cor.posmat.specrad.leq1a} is proved.
\end{proof}

\begin{corollary}
\label{cor.posmat.specrad.leq1b}Let $A\in\mathbb{R}^{n\times n}$ and
$B\in\mathbb{R}^{n\times n}$ satisfy $B\geq A\geq0$ and $n>0$. Then,
$\rho\left(  A\right)  \leq\rho\left(  B\right)  $.
\end{corollary}

\begin{proof}
We have $\left\vert A\right\vert =A$. Thus, we can apply Theorem
\ref{thm.posmat.specrad.leq1}.
\end{proof}

\begin{corollary}
\label{cor.posmat.specrad.leq1c}Let $A\in\mathbb{R}^{n\times n}$ satisfy
$A\geq0$ and $n>0$. \medskip

\textbf{(a)} If $\widetilde{A}$ is a principal submatrix of $A$ (that is, a
matrix obtained from $A$ by removing a bunch of rows along with the
corresponding columns), then $\rho\left(  \widetilde{A}\right)  \leq
\rho\left(  A\right)  $. \medskip

\textbf{(b)} We have $\max\left\{  A_{i,i}\ \mid\ i\in\left[  n\right]
\right\}  \leq\rho\left(  A\right)  $. \medskip

\textbf{(c)} If $A_{i,i}>0$ for some $i\in\left[  n\right]  $, then
$\rho\left(  A\right)  >0$.
\end{corollary}

\begin{proof}
\textbf{(a)} Let $\widetilde{A}$ be a principal submatrix of $A$. For
simplicity, I assume that $\widetilde{A}$ is $A$ with the $n$-th row and the
$n$-th column removed\footnote{The proof in the general case is similar; it
just requires more notational work.}. Thus,%
\begin{equation}
A=\left(
\begin{array}
[c]{cc}%
\widetilde{A} & y\\
x & \lambda
\end{array}
\right)  \ \ \ \ \ \ \ \ \ \ \left(  \text{in block-matrix notation}\right)
\label{pf.cor.posmat.specrad.leq1c.a.A=}%
\end{equation}
for some nonnegative $x\in\mathbb{R}^{1\times\left(  n-1\right)  }$,
$y\in\mathbb{R}^{\left(  n-1\right)  \times1}$ and $\lambda\in\mathbb{R}$.
Let
\begin{equation}
B:=\left(
\begin{array}
[c]{cc}%
\widetilde{A} & 0\\
0 & 0
\end{array}
\right)  \ \ \ \ \ \ \ \ \ \ \left(  \text{in block-matrix notation}\right)  ,
\label{pf.cor.posmat.specrad.leq1c.a.B=}%
\end{equation}
where the three $0$s have the same dimensions as the $x$, $y$ and $\lambda$
above. Comparing (\ref{pf.cor.posmat.specrad.leq1c.a.A=}) with
(\ref{pf.cor.posmat.specrad.leq1c.a.B=}), we see that $A\geq B$ (since
$x\geq0$ and $y\geq0$ and $\lambda\geq0$). Also, $\widetilde{A}\geq0$ (since
$A\geq0$) and thus $B\geq0$. Thus, $A\geq B\geq0$. Hence, Corollary
\ref{cor.posmat.specrad.leq1b} (applied to $B$ and $A$ instead of $A$ and $B$)
yields $\rho\left(  B\right)  \leq\rho\left(  A\right)  $.

However, it is easy to see from (\ref{pf.cor.posmat.specrad.leq1c.a.B=}) that
$\sigma\left(  B\right)  =\sigma\left(  \widetilde{A}\right)  \cup\left\{
0\right\}  $ (for example, because we can pick any Schur triangularization
$\left(  U,T\right)  $ of $\widetilde{A}$, and then obtain a Schur
triangularization $\left(  U^{\prime},T^{\prime}\right)  $ of $B$ by setting
$U^{\prime}=\left(
\begin{array}
[c]{cc}%
U & 0\\
0 & 1
\end{array}
\right)  $ and $T^{\prime}=\left(
\begin{array}
[c]{cc}%
T & 0\\
0 & 0
\end{array}
\right)  $). Hence, $\rho\left(  B\right)  =\rho\left(  \widetilde{A}\right)
$ (because inserting $0$ into a set of nonnegative reals cannot change the
maximum of this set). Hence, $\rho\left(  B\right)  \leq\rho\left(  A\right)
$ rewrites as $\rho\left(  \widetilde{A}\right)  \leq\rho\left(  A\right)  $.
This proves Corollary \ref{cor.posmat.specrad.leq1a} \textbf{(a)}. \medskip

\textbf{(b)} We must show that $A_{i,i}\leq\rho\left(  A\right)  $ for all
$i\in\left[  n\right]  $.

So let $i\in\left[  n\right]  $. Then, the $1\times1$-matrix $\left(
\begin{array}
[c]{c}%
A_{i,i}%
\end{array}
\right)  $ is a principal submatrix of $A$ (obtained by removing all rows of
$A$ other than the $i$-th one, and all columns of $A$ other than the $i$-th
one). Hence, part \textbf{(a)} yields $\rho\left(  \left(
\begin{array}
[c]{c}%
A_{i,i}%
\end{array}
\right)  \right)  \leq\rho\left(  A\right)  $. However, since the only
eigenvalue of the $1\times1$-matrix $\left(
\begin{array}
[c]{c}%
A_{i,i}%
\end{array}
\right)  $ is $A_{i,i}$, we have $\rho\left(  \left(
\begin{array}
[c]{c}%
A_{i,i}%
\end{array}
\right)  \right)  =\left\vert A_{i,i}\right\vert =A_{i,i}$ (since $A\geq0$).
Therefore, $A_{i,i}=\rho\left(  \left(
\begin{array}
[c]{c}%
A_{i,i}%
\end{array}
\right)  \right)  \leq\rho\left(  A\right)  $. This proves Corollary
\ref{cor.posmat.specrad.leq1a} \textbf{(b)}. \medskip

\textbf{(c)} Follows from \textbf{(b)}.
\end{proof}

\begin{exercise}
\fbox{1} Let $A\in\mathbb{R}^{n\times n}$ satisfy $A>0$ and $n>0$. Prove that
$\rho\left(  A\right)  >0$.
\end{exercise}

\begin{exercise}
\fbox{2} Let $A\in\mathbb{C}^{n\times n}$ and $B\in\mathbb{R}^{n\times n}$ be
such that $B>\left\vert A\right\vert $ and $n>0$. Prove that $\rho\left(
A\right)  <\rho\left(  B\right)  $. \medskip

[\textbf{Hint:} Show that there exists some real $\lambda>1$ such that
$B\geq\lambda\cdot\left\vert A\right\vert $.]
\end{exercise}

Let us next prove some more bounds for $\rho\left(  A\right)  $ when $A$ is a
nonnegative matrix. We will use the following notions:\footnote{Recall that
$\max\limits_{i\in I}a_{i}$ is a shorthand notation for $\max\left\{
a_{i}\ \mid\ i\in I\right\}  $ (when $I$ is a set and $a_{i}$ is a real number
for each $i\in I$).}

\begin{definition}
\label{def.posmat.infty1-norms}Let $\mathbb{F}$ be a field. Let $A\in
\mathbb{F}^{n\times m}$. \medskip

\textbf{(a)} The \emph{column sums} of $A$ are the $m$ sums
\[
\sum_{i=1}^{n}A_{i,j}=\left(  \text{the sum of all entries of the }j\text{-th
column of }A\right)
\]
for $j\in\left[  m\right]  $. \medskip

\textbf{(b)} The \emph{row sums} of $A$ are%
\[
\sum_{j=1}^{m}A_{i,j}=\left(  \text{the sum of all entries of the }i\text{-th
row of }A\right)
\]
for $i\in\left[  n\right]  $. \medskip

\textbf{(c)} Now, assume that $\mathbb{F}=\mathbb{C}$ and $n>0$ and $m>0$.
Then, we set%
\[
\left\vert \left\vert A\right\vert \right\vert _{\infty}:=\left(  \text{the
largest row sum of }\left\vert A\right\vert \right)  =\max\limits_{i\in\left[
n\right]  }\sum_{j=1}^{m}\left\vert A_{i,j}\right\vert
\]
and%
\[
\left\vert \left\vert A\right\vert \right\vert _{1}:=\left(  \text{the largest
column sum of }\left\vert A\right\vert \right)  =\max\limits_{j\in\left[
m\right]  }\sum_{i=1}^{n}\left\vert A_{i,j}\right\vert .
\]
These two numbers $\left\vert \left\vert A\right\vert \right\vert _{\infty}$
and $\left\vert \left\vert A\right\vert \right\vert _{1}$ are called the
$\infty$\emph{-norm} and the $1$\emph{-norm} of $A$ (for reasons that will be
explained in a later chapter).
\end{definition}

\begin{example}
The column sums of a $2\times2$-matrix $\left(
\begin{array}
[c]{cc}%
a & b\\
c & d
\end{array}
\right)  $ are $a+c$ and $b+d$, whereas its row sums are $a+b$ and $c+d$.
\end{example}

\begin{warning}
Let $A$ be a matrix. Then, the sum of all rows of $A$ is a row vector whose
entries are the column sums (not the row sums!) of $A$. Likewise, the sum of
all columns of $A$ is a column vector whose entries are the row sums (not the
column sums!) of $A$.
\end{warning}

The following is obvious:

\begin{remark}
\label{rmk.posmat.rowsums-AT}Let $A\in\mathbb{F}^{n\times m}$ be a matrix over
a field $\mathbb{F}$. \medskip

\textbf{(a)} The row sums of $A$ are the column sums of $A^{T}$, and vice
versa. \medskip

\textbf{(b)} If $\mathbb{F}=\mathbb{C}$ and $n>0$ and $m>0$, then $\left\vert
\left\vert A\right\vert \right\vert _{\infty}=\left\vert \left\vert
A^{T}\right\vert \right\vert _{1}$ and $\left\vert \left\vert A\right\vert
\right\vert _{1}=\left\vert \left\vert A^{T}\right\vert \right\vert _{\infty}$.
\end{remark}

We can now bound the spectral radius of a matrix in terms of its $1$-norm and
its $\infty$-norm:

\begin{lemma}
\label{lem.posmat.rho-in-terms-of-norms}Let $A\in\mathbb{C}^{n\times n}$ with
$n>0$. Then: \medskip

\textbf{(a)} We have $\rho\left(  A\right)  \leq\left\vert \left\vert
A\right\vert \right\vert _{\infty}$. \medskip

\textbf{(b)} If $A\geq0$ and if all row sums of $A$ are equal, then
$\rho\left(  A\right)  =\left\vert \left\vert A\right\vert \right\vert
_{\infty}$. \medskip

\textbf{(c)} We have $\rho\left(  A\right)  \leq\left\vert \left\vert
A\right\vert \right\vert _{1}$. \medskip

\textbf{(d)} If $A\geq0$ and if all column sums of $A$ are equal, then
$\rho\left(  A\right)  =\left\vert \left\vert A\right\vert \right\vert _{1}$.
\end{lemma}

\begin{proof}
\textbf{(a)} We have $\rho\left(  A\right)  =\left\vert \lambda\right\vert $
for some eigenvalue $\lambda$ of $A$ (by the definition of $\rho\left(
A\right)  $). Consider this $\lambda$, and let $v=\left(  v_{1},v_{2}%
,\ldots,v_{n}\right)  ^{T}\in\mathbb{C}^{n}$ be a nonzero $\lambda
$-eigenvector of $A$. Then, $Av=\lambda v$.

Choose an $i\in\left[  n\right]  $ such that $\left\vert v_{i}\right\vert
=\max\left\{  \left\vert v_{1}\right\vert ,\left\vert v_{2}\right\vert
,\ldots,\left\vert v_{n}\right\vert \right\}  $. Then, $\left\vert
v_{i}\right\vert >0$ (since $v$ is nonzero). Furthermore,%
\begin{equation}
\left\vert v_{j}\right\vert \leq\left\vert v_{i}\right\vert
\ \ \ \ \ \ \ \ \ \ \text{for each }j\in\left[  n\right]
\label{pf.lem.posmat.rho-in-terms-of-norms.max}%
\end{equation}
(since $\left\vert v_{i}\right\vert =\max\left\{  \left\vert v_{1}\right\vert
,\left\vert v_{2}\right\vert ,\ldots,\left\vert v_{n}\right\vert \right\}  $).

Now, the $i$-th entry of the column vector $Av$ is $\sum\limits_{j=1}%
^{n}A_{i,j}v_{j}$ (by the definition of the product $Av$); however, the same
entry is $\lambda v_{i}$ (since $Av=\lambda v$). Comparing these two facts, we
obtain%
\[
\lambda v_{i}=\sum\limits_{j=1}^{n}A_{i,j}v_{j}.
\]
Taking absolute values on both sides of this equality, we obtain%
\begin{align*}
\left\vert \lambda v_{i}\right\vert  &  =\left\vert \sum\limits_{j=1}%
^{n}A_{i,j}v_{j}\right\vert \leq\sum\limits_{j=1}^{n}\underbrace{\left\vert
A_{i,j}v_{j}\right\vert }_{\substack{=\left\vert A_{i,j}\right\vert
\cdot\left\vert v_{j}\right\vert \leq\left\vert A_{i,j}\right\vert
\cdot\left\vert v_{i}\right\vert \\\text{(by
(\ref{pf.lem.posmat.rho-in-terms-of-norms.max}))}}}\ \ \ \ \ \ \ \ \ \ \left(
\text{by the triangle inequality}\right) \\
&  \leq\sum\limits_{j=1}^{n}\left\vert A_{i,j}\right\vert \cdot\left\vert
v_{i}\right\vert .
\end{align*}
Since $\left\vert \lambda v_{i}\right\vert =\left\vert \lambda\right\vert
\cdot\left\vert v_{i}\right\vert $, we can rewrite this as
\[
\left\vert \lambda\right\vert \cdot\left\vert v_{i}\right\vert \leq
\sum\limits_{j=1}^{n}\left\vert A_{i,j}\right\vert \cdot\left\vert
v_{i}\right\vert .
\]
Since $\left\vert v_{i}\right\vert >0$, we can cancel $\left\vert
v_{i}\right\vert $ from this inequality, and thus we obtain%
\begin{align*}
\left\vert \lambda\right\vert  &  \leq\sum\limits_{j=1}^{n}\left\vert
A_{i,j}\right\vert =\left(  \text{the }i\text{-th row sum of }\left\vert
A\right\vert \right) \\
&  \leq\left(  \text{the largest row sum of }\left\vert A\right\vert \right)
=\left\vert \left\vert A\right\vert \right\vert _{\infty}.
\end{align*}
Since $\rho\left(  A\right)  =\left\vert \lambda\right\vert $, this rewrites
as $\rho\left(  A\right)  \leq\left\vert \left\vert A\right\vert \right\vert
_{\infty}$. This proves Lemma \ref{lem.posmat.rho-in-terms-of-norms}
\textbf{(a)}. \medskip

\textbf{(b)} Assume that $A\geq0$ and that all row sums of $A$ are equal. Let
$e=\left(  1,1,\ldots,1\right)  ^{T}\in\mathbb{R}^{n}$, and let $\kappa$ be
the common value of the row sums of $A$. Then, all row sums of $A$ equal
$\kappa$; in other words, we have $Ae=\kappa e$ (since $Ae$ is the column
vector whose entries are the row sums of $A$, whereas $\kappa e$ is the column
vector whose entries are $\underbrace{\kappa,\kappa,\ldots,\kappa}_{n\text{
times}}$). Hence, $\kappa$ is an eigenvalue of $A$ (since $e\neq0$), so that
$\rho\left(  A\right)  \geq\left\vert \kappa\right\vert =\kappa$ (since
$A\geq0$ entails $\kappa\geq0$).

On the other hand, $\left\vert A\right\vert =A$ (since $A\geq0$). Now, Lemma
\ref{lem.posmat.rho-in-terms-of-norms} \textbf{(a)} yields
\[
\rho\left(  A\right)  \leq\left\vert \left\vert A\right\vert \right\vert
_{\infty}=\left(  \text{the largest row sum of }\underbrace{\left\vert
A\right\vert }_{=A}\right)  =\left(  \text{the largest row sum of }A\right)
=\kappa
\]
(since all row sums of $A$ are $\kappa$). Combining this with $\rho\left(
A\right)  \geq\kappa$, we obtain $\rho\left(  A\right)  =\kappa=\left\vert
\left\vert A\right\vert \right\vert _{\infty}$. This proves Lemma
\ref{lem.posmat.rho-in-terms-of-norms} \textbf{(b)}. \medskip

\textbf{(c)} This follows by applying Lemma
\ref{lem.posmat.rho-in-terms-of-norms} \textbf{(a)} of the lemma to $A^{T}$
instead of $A$, and recalling that $\left\vert \left\vert A^{T}\right\vert
\right\vert _{\infty}=\left\vert \left\vert A\right\vert \right\vert _{1}$ and
$\rho\left(  A^{T}\right)  =\rho\left(  A\right)  $. \medskip

\textbf{(d)} This follows by applying Lemma
\ref{lem.posmat.rho-in-terms-of-norms} \textbf{(b)} of the lemma to $A^{T}$
instead of $A$, and recalling that $\left\vert \left\vert A^{T}\right\vert
\right\vert _{\infty}=\left\vert \left\vert A\right\vert \right\vert _{1}$ and
$\rho\left(  A^{T}\right)  =\rho\left(  A\right)  $ and the row sums of
$A^{T}$ are the column sums of $A$.
\end{proof}

\begin{remark}
We note that the converse of Lemma \ref{lem.posmat.rho-in-terms-of-norms}
\textbf{(b)} is false: For example, the $3\times3$-matrix $A:=\left(
\begin{array}
[c]{ccc}%
1 & 0 & 0\\
0 & 0 & 1\\
0 & 0 & 0
\end{array}
\right)  $ satisfies $A\geq0$ and $\rho\left(  A\right)  =\left\vert
\left\vert A\right\vert \right\vert _{\infty}$, but the row sums of $A$ are
not all equal.
\end{remark}

Next, let us bound the spectral radius $\rho\left(  A\right)  $ of a matrix
$A$ from both sides when $A\geq0$:

\begin{theorem}
\label{thm.posmat.rho-in-terms-of-rowsums}Let $A\in\mathbb{R}^{n\times n}$
satisfy $A\geq0$ and $n>0$. Then,%
\[
\left(  \text{the smallest row sum of }A\right)  \leq\rho\left(  A\right)
\leq\left(  \text{the largest row sum of }A\right)  .
\]

\end{theorem}

\begin{proof}
We have $\left\vert A\right\vert =A$ (since $A\geq0$). Now, Lemma
\ref{lem.posmat.rho-in-terms-of-norms} \textbf{(a)} yields%
\[
\rho\left(  A\right)  \leq\left\vert \left\vert A\right\vert \right\vert
_{\infty}=\left(  \text{the largest row sum of }\underbrace{\left\vert
A\right\vert }_{=A}\right)  =\left(  \text{the largest row sum of }A\right)
.
\]
Hence, it remains to prove that $\left(  \text{the smallest row sum of
}A\right)  \leq\rho\left(  A\right)  $.

Let $r_{1},r_{2},\ldots,r_{n}$ be the row sums of $A$. Let $r_{i}$ be the
smallest among them. We must thus prove that $r_{i}\leq\rho\left(  A\right)
$. If $r_{i}=0$, then this is obvious. So let WLOG assume that $r_{i}>0$.
Hence, all $n$ numbers $r_{1},r_{2},\ldots,r_{n}$ are positive (since the
smallest among them is $r_{i}>0$) and thus nonzero.

Let $B$ the $n\times n$-matrix whose $\left(  u,v\right)  $-th entry is
$\dfrac{r_{i}}{r_{u}}A_{u,v}$ for all $u,v\in\left[  n\right]  $%
.\ \ \ \ \footnote{This is well-defined, since $r_{u}\neq0$ (because all $n$
numbers $r_{1},r_{2},\ldots,r_{n}$ are nonzero).} Thus, $B$ is obtained from
the matrix $A$ by scaling each row by a certain positive real factor (namely,
$\dfrac{r_{i}}{r_{u}}$ for the $u$-th row) chosen in such a way that the row
sums all become $r_{i}$. Hence, the matrix $B$ is $\geq0$ (since $A\geq0$, and
since all $n$ numbers $r_{1},r_{2},\ldots,r_{n}$ are positive), and its row
sums are all equal to $r_{i}$. Hence, Lemma
\ref{lem.posmat.rho-in-terms-of-norms} \textbf{(b)} (applied to $B$ instead of
$A$) yields
\begin{align}
\rho\left(  B\right)   &  =\left\vert \left\vert B\right\vert \right\vert
_{\infty}=\left(  \text{the largest row sum of }\underbrace{\left\vert
B\right\vert }_{\substack{=B\\\text{(since }B\geq0\text{)}}}\right)
\nonumber\\
&  =\left(  \text{the largest row sum of }B\right)  =r_{i}
\label{pf.thm.posmat.rho-in-terms-of-rowsums.4}%
\end{align}
(since all row sums of $B$ are $r_{i}$). However, for each $u,v\in\left[
n\right]  $, we have $\dfrac{r_{i}}{r_{u}}A_{u,v}\leq A_{u,v}$ (since
$r_{i}\leq r_{u}$ (because $r_{i}$ is the smallest among the numbers
$r_{1},r_{2},\ldots,r_{n}$)). In other words, $B\leq A$ (since the entries of
$B$ are the numbers $\dfrac{r_{i}}{r_{u}}A_{u,v}$, whereas the corresponding
entries of $A$ are $A_{u,v}$). Hence, Corollary \ref{cor.posmat.specrad.leq1b}
(applied to $B$ and $A$ instead of $A$ and $B$) yields $\rho\left(  B\right)
\leq\rho\left(  A\right)  $. Thus,
(\ref{pf.thm.posmat.rho-in-terms-of-rowsums.4}) becomes $r_{i}=\rho\left(
B\right)  \leq\rho\left(  A\right)  $. This proves Theorem
\ref{thm.posmat.rho-in-terms-of-rowsums}.
\end{proof}

\begin{corollary}
\label{cor.posmat.rho-in-terms-of-rowsums-x}Let $A\in\mathbb{R}^{n\times n}$
satisfy $A\geq0$ and $n>0$. Let $x_{1},x_{2},\ldots,x_{n}$ be any $n$ positive
reals. Then,%
\[
\min\limits_{i\in\left[  n\right]  }\sum_{j=1}^{n}\dfrac{x_{i}}{x_{j}}%
A_{i,j}\leq\rho\left(  A\right)  \leq\max\limits_{i\in\left[  n\right]  }%
\sum_{j=1}^{n}\dfrac{x_{i}}{x_{j}}A_{i,j}.
\]

\end{corollary}

\begin{proof}
Let $D=\operatorname*{diag}\left(  x_{1},x_{2},\ldots,x_{n}\right)  $. Then,
$DAD^{-1}$ is the $n\times n$-matrix whose $\left(  i,j\right)  $-th entry is
$x_{i}A_{i,j}x_{j}^{-1}=\dfrac{x_{i}}{x_{j}}A_{i,j}$ for all $i,j\in\left[
n\right]  $. Thus, $DAD^{-1}\geq0$ (since $A\geq0$ and since $x_{1}%
,x_{2},\ldots,x_{n}$ are positive). Hence, Theorem
\ref{thm.posmat.rho-in-terms-of-rowsums} (applied to $DAD^{-1}$ instead of
$A$) yields%
\[
\left(  \text{the smallest row sum of }DAD^{-1}\right)  \leq\rho\left(
DAD^{-1}\right)  \leq\left(  \text{the largest row sum of }DAD^{-1}\right)  .
\]
In view of $\rho\left(  DAD^{-1}\right)  =\rho\left(  A\right)  $ (which is a
consequence of the fact that the matrices $DAD^{-1}$ and $A$ are similar and
thus have the same spectrum), we can rewrite this as
\[
\left(  \text{the smallest row sum of }DAD^{-1}\right)  \leq\rho\left(
A\right)  \leq\left(  \text{the largest row sum of }DAD^{-1}\right)  .
\]
Now, it remains only to notice that the row sums of $DAD^{-1}$ are exactly the
sums $\sum_{j=1}^{n}\dfrac{x_{i}}{x_{j}}A_{i,j}$ for $i\in\left[  n\right]  $.
\end{proof}

\begin{remark}
If the matrix $A$ in Corollary \ref{cor.posmat.rho-in-terms-of-rowsums-x} is
positive, then there is a choice of $x_{1},x_{2},\ldots,x_{n}>0$ such that
both of the inequalities become equalities. (This follows from Theorem
\ref{thm.posmat.perron} \textbf{(c)} further below.)
\end{remark}

\begin{corollary}
\label{cor.posmat.supereigenvectors}Let $A\in\mathbb{R}^{n\times n}$ satisfy
$A\geq0$ and $n>0$.

Let $x\in\mathbb{R}^{n}$ satisfy $x>0$.

Let $\alpha$ be a nonnegative real. Then: \medskip

\textbf{(a)} If $Ax\geq\alpha x$, then $\rho\left(  A\right)  \geq\alpha$.
\medskip

\textbf{(b)} If $Ax>\alpha x$, then $\rho\left(  A\right)  >\alpha$. \medskip

\textbf{(c)} If $Ax\leq\alpha x$, then $\rho\left(  A\right)  \leq\alpha$.
\medskip

\textbf{(d)} If $Ax<\alpha x$, then $\rho\left(  A\right)  <\alpha$.
\end{corollary}

\begin{proof}
Write $x$ as $x=\left(  x_{1},x_{2},\ldots,x_{n}\right)  ^{T}$. Then, the $n$
numbers $x_{1},x_{2},\ldots,x_{n}$ are positive reals (since $x>0$); hence,
their reciprocals $1/x_{1},\ 1/x_{2},\ \ldots,\ 1/x_{n}$ are well-defined
positive reals as well. \medskip

\textbf{(a)} Assume that $Ax\geq\alpha x$. Then, for each $i\in\left[
n\right]  $, we have%
\begin{align}
\sum_{j=1}^{n}A_{i,j}x_{j}  &  =\left(  \text{the }i\text{-th entry of
}Ax\right) \nonumber\\
&  \geq\left(  \text{the }i\text{-th entry of }\alpha x\right)
\ \ \ \ \ \ \ \ \ \ \left(  \text{since }Ax\geq\alpha x\right) \nonumber\\
&  =\alpha x_{i}. \label{pf.cor.posmat.supereigenvectors.a.1}%
\end{align}


However, Corollary \ref{cor.posmat.rho-in-terms-of-rowsums-x} (applied to
$1/x_{k}$ instead of $x_{k}$) yields%
\[
\min\limits_{i\in\left[  n\right]  }\sum_{j=1}^{n}\dfrac{1/x_{i}}{1/x_{j}%
}A_{i,j}\leq\rho\left(  A\right)  \leq\max\limits_{i\in\left[  n\right]  }%
\sum_{j=1}^{n}\dfrac{1/x_{i}}{1/x_{j}}A_{i,j}.
\]
The first of these two inequalities yields%
\begin{align*}
\rho\left(  A\right)   &  \geq\min\limits_{i\in\left[  n\right]  }\sum
_{j=1}^{n}\underbrace{\dfrac{1/x_{i}}{1/x_{j}}A_{i,j}}_{=\dfrac{1}{x_{i}%
}A_{i,j}x_{j}}=\min\limits_{i\in\left[  n\right]  }\sum_{j=1}^{n}\dfrac
{1}{x_{i}}A_{i,j}x_{j}=\min\limits_{i\in\left[  n\right]  }\dfrac{1}{x_{i}%
}\underbrace{\sum_{j=1}^{n}A_{i,j}x_{j}}_{\substack{\geq\alpha x_{i}%
\\\text{(by (\ref{pf.cor.posmat.supereigenvectors.a.1}))}}}\\
&  \geq\min\limits_{i\in\left[  n\right]  }\underbrace{\dfrac{1}{x_{i}}%
\cdot\alpha x_{i}}_{=\alpha}=\min\limits_{i\in\left[  n\right]  }\alpha
=\alpha.
\end{align*}
This proves Corollary \ref{cor.posmat.supereigenvectors} \textbf{(a)}.
\medskip

\textbf{(b)} The proof is analogous to the proof of Corollary
\ref{cor.posmat.supereigenvectors} \textbf{(a)}, but uses $>$ signs instead of
$\geq$ signs. \medskip

\textbf{(c)} The proof is similar to the proof of Corollary
\ref{cor.posmat.supereigenvectors} \textbf{(a)}, but uses $>$ signs instead of
$\geq$ signs and uses $\max$ instead of $\min$. \medskip

\textbf{(d)} The proof is analogous to the proof of Corollary
\ref{cor.posmat.supereigenvectors} \textbf{(c)}.
\end{proof}

\begin{corollary}
\label{cor.posmat.supereig-cor}Let $A\in\mathbb{R}^{n\times n}$ satisfy $A>0$
and $n>0$ and $\rho\left(  A\right)  =1$. Let $w\in\mathbb{R}^{n}$ satisfy
$w\geq0$ and $w\neq0$. Then: \medskip

\textbf{(a)} We always have $Aw>0$. \medskip

\textbf{(b)} If $Aw\geq w$, then $Aw=w>0$.
\end{corollary}

\begin{proof}
Write the vector $w$ as $w=\left(  w_{1},w_{2},\ldots,w_{n}\right)  ^{T}$.
Then, the numbers $w_{1},w_{2},\ldots,w_{n}$ are nonnegative reals (since
$w\geq0$). Moreover, at least one $k\in\left[  n\right]  $ satisfies
$w_{k}\neq0$ (since $w\neq0$). Consider this $k$. Thus, $w_{k}>0$ (since
$w\geq0$). \medskip

\textbf{(a)} For each $i\in\left[  n\right]  $, the $i$-th entry of $Aw$ is
$\sum_{j=1}^{n}A_{i,j}w_{j}$. This is a sum of nonnegative addends (since
$A>0$ and $w\geq0$), and at least one of these addends is actually positive
(indeed, $A>0$ entails $A_{i,k}>0$ and thus $\underbrace{A_{i,k}}%
_{>0}\underbrace{w_{k}}_{>0}>0$). Hence, this sum is positive. We have thus
shown that for each $i\in\left[  n\right]  $, the $i$-th entry of $Aw$ is
positive. In other words, $Aw>0$. This proves Corollary
\ref{cor.posmat.supereig-cor} \textbf{(a)}. \medskip

\textbf{(b)} Assume that $Aw\geq w$. Let $z:=Aw-w$. Then, $z=Aw-w\geq0$.
However, Corollary \ref{cor.posmat.supereig-cor} \textbf{(a)} also yields
$Aw>0$.

We claim that $z=0$. Indeed, assume the contrary. Thus, $z\neq0$. Hence,
Corollary \ref{cor.posmat.supereig-cor} \textbf{(a)} (applied to $z$ instead
of $w$) yields $Az>0$. Therefore, $AAw>Aw$ (since $AAw-Aw=A\underbrace{\left(
Aw-w\right)  }_{=z}=Az>0$). Also, $A\geq0$ (since $A>0$). Hence, Corollary
\ref{cor.posmat.supereigenvectors} \textbf{(b)} (applied to $x=Aw$ and
$\alpha=1$) yields $\rho\left(  A\right)  >1$, which contradicts $\rho\left(
A\right)  =1$. This contradiction shows that our assumption was wrong. Hence,
$z=0$ is proved. Thus, $Aw=w$ (since $Aw-w=z=0$). Hence, $w=Aw>0$. This proves
Corollary \ref{cor.posmat.supereig-cor} \textbf{(b)}.
\end{proof}

\subsection{Perron--Frobenius theorems}

We now come to the most important results about nonnegative matrices: the
Perron--Frobenius theorems.

\subsubsection{Motivation}

Let us first motivate the theorems using a less general (but more intuitive) setting.

Recall a standard situation in probability theory: Consider a system (e.g., a
slot machine) that can be in one of $n$ possible \emph{states} $s_{1}%
,s_{2},\ldots,s_{n}$. Every minute, the system randomly changes states
according to the following rule: If the system is in state $s_{i}$, then it
changes to state $s_{j}$ with probability $P_{i,j}$, where $P$ is a (fixed,
pre-determined) nonnegative $n\times n$-matrix whose row sums all equal $1$
(such a matrix is called \emph{row-stochastic}). This is commonly known as a
\emph{Markov chain}.

Given such a Markov chain, one often wonders about its \textquotedblleft
steady state\textquotedblright: If you wait long enough, how likely is the
system to be in a given state?

\begin{example}
\label{exa.posmat.markov-chain.1}Let $P=\left(
\begin{array}
[c]{cc}%
0.9 & 0.1\\
0.5 & 0.5
\end{array}
\right)  $. This corresponds to a system that has two states $s_{1}$ and
$s_{2}$, and the probability of going from state $s_{1}$ to state $s_{2}$ (any
given minute) is $0.1$, whereas the probability of going from state $s_{1}$ to
state $s_{1}$ (that is, staying at state $s_{1}$) is $0.9$, and the
probability of going from state $s_{2}$ to either state is $0.5$.

We encode the two states $s_{1}$ and $s_{2}$ as the basis vectors
$e_{1}=\left(  1,0\right)  $ and $e_{2}=\left(  0,1\right)  $ of the vector
space $\mathbb{R}^{1\times2}$ (we work with row vectors here for convenience).
Thus, a probability distribution on the set of states (i.e., a distribution of
the form \textquotedblleft state $s_{1}$ with probability $a_{1}$ and state
$s_{2}$ with probability $a_{2}$\textquotedblright) corresponds to a row
vector $\left(  a_{1},a_{2}\right)  \in\mathbb{R}^{1\times2}$ satisfying
$a_{1}\geq0$ and $a_{2}\geq0$ and $a_{1}+a_{2}=1$.

If we start at state $s_{1}$ and let $k$ minutes pass, then the probability
distribution for the resulting state is $s_{1}P^{k}$ (why?). More generally,
if we start with a probability distribution $d\in\mathbb{R}^{1\times2}$ and
let $k$ minutes pass, then the resulting state will be distributed according
to $dP^{k}$ (why?). So our question about the steady state can be rewritten as
follows: What is $\lim\limits_{k\rightarrow\infty}dP^{k}$ ? Does this limit
even exist?

We can notice one thing right away: If the limit $\lim\limits_{k\rightarrow
\infty}dP^{k}$ exists, then this limit is a left $1$-eigenvector of $P$, in
the sense that it is a row vector $y$ such that $yP=y$ (because if we set
$y=\lim\limits_{k\rightarrow\infty}dP^{k}$, then we have $y=\lim
\limits_{k\rightarrow\infty}dP^{k}=\lim\limits_{k\rightarrow\infty
}d\underbrace{P^{k+1}}_{=P^{k}P}=\left(  \lim\limits_{k\rightarrow\infty
}dP^{k}\right)  P=yP$). Since it is furthermore a probability distribution
(because it is a limit of probability distributions), we can easily compute it
(indeed, our matrix $P$ has only one left $1$-eigenvector up to scaling, and
the scaling factor is uniquely determined by the requirement that it be a
probability distribution). We obtain%
\[
\lim\limits_{k\rightarrow\infty}dP^{k}=\left(  \dfrac{5}{6},\dfrac{1}%
{6}\right)  .
\]


But does this limit actually exist? Yes: In our specific example, it does; but
this isn't quite that obvious. Note that this limit (known as the \emph{steady
state} of the Markov chain) actually does not depend on the starting
distribution $d$.
\end{example}

Does this generalize? Not always. Here are two examples where things go bad:

\begin{itemize}
\item If $P=I_{2}=\left(
\begin{array}
[c]{cc}%
1 & 0\\
0 & 1
\end{array}
\right)  $, then $\lim\limits_{k\rightarrow\infty}dP^{k}=d$ for each $d$, so
the limits do depend on $d$.

\item If $P=\left(
\begin{array}
[c]{cc}%
0 & 1\\
1 & 0
\end{array}
\right)  $, then $\lim\limits_{k\rightarrow\infty}dP^{k}$ does not exist
unless $d=\left(  0.5,\ 0.5\right)  $, since in all other cases the sequence
$\left(  dP^{k}\right)  _{k\geq0}$ oscillates between $\left(  a_{1}%
,a_{2}\right)  $ and $\left(  a_{2},a_{1}\right)  $.
\end{itemize}

Perhaps surprisingly, such bad cases are an exception. For \textbf{most}
row-stochastic matrices $P$ (that is, nonnegative matrices whose row sums all
equal $1$), there is a \textbf{unique} steady state (i.e., left $1$%
-eigenvector whose entries sum up to $1$), and it can be obtained as
$\lim\limits_{k\rightarrow\infty}dP^{k}$ for any starting distribution $d$. To
be more precise, this holds whenever $P$ is positive (i.e., all $P_{i,j}>0$).
Some weaker assumptions also suffice.

More general versions of these facts hold even if we don't assume $P$ to be
row-stochastic, but merely require $P>0$ (or $P\geq0$ with some extra
conditions). These will be the Perron and Perron--Frobenius theorems.

\subsubsection{The theorems}

We can now state the Perron and Perron--Frobenius theorems; we will prove them later:

\begin{theorem}
[Perron theorem]\label{thm.posmat.perron}Let $A\in\mathbb{R}^{n\times n}$
satisfy $A>0$ and $n>0$. Then: \medskip

\textbf{(a)} We have $\rho\left(  A\right)  >0$. \medskip

\textbf{(b)} The number $\rho\left(  A\right)  $ is an eigenvalue of $A$ and
has algebraic multiplicity $1$ (and therefore geometric multiplicity $1$ as
well). \medskip

\textbf{(c)} There is a unique $\rho\left(  A\right)  $-eigenvector $x=\left(
x_{1},x_{2},\ldots,x_{n}\right)  ^{T}\in\mathbb{C}^{n}$ of $A$ with
$x_{1}+x_{2}+\cdots+x_{n}=1$. This eigenvector $x$ is furthermore positive.
(It is called the \emph{Perron vector} of $A$.) \medskip

\textbf{(d)} There is a unique vector $y=\left(  y_{1},y_{2},\ldots
,y_{n}\right)  ^{T}\in\mathbb{C}^{n}$ such that $y^{T}A=\rho\left(  A\right)
y^{T}$ and $x_{1}y_{1}+x_{2}y_{2}+\cdots+x_{n}y_{n}=1$. This vector $y$ is
also positive. \medskip

\textbf{(e)} We have%
\[
\left(  \dfrac{1}{\rho\left(  A\right)  }A\right)  ^{m}\rightarrow
xy^{T}\ \ \ \ \ \ \ \ \ \ \text{as }m\rightarrow\infty.
\]


\textbf{(f)} The only eigenvalue of $A$ that has absolute value $\rho\left(
A\right)  $ is $\rho\left(  A\right)  $ itself.
\end{theorem}

We will prove this soon, but first we have two more theorems to state. The
Perron theorem applies to positive matrices; but some parts of it can be
adapted to the more general situation of a nonnegative matrix. If we require
nothing other than nonnegativity, then only two statements hold:

\begin{theorem}
[Perron--Frobenius theorem 1]\label{thm.posmat.pf1}Let $A\in\mathbb{R}%
^{n\times n}$ satisfy $A\geq0$ and $n>0$. Then: \medskip

\textbf{(a)} The number $\rho\left(  A\right)  $ is an eigenvalue of $A$.
\medskip

\textbf{(b)} The matrix $A$ has a nonzero nonnegative $\rho\left(  A\right)  $-eigenvector.
\end{theorem}

To get stronger statements without requiring $A>0$, we need two further
properties of $A$.

\begin{definition}
Let $A\in\mathbb{R}^{n\times n}$ be an $n\times n$-matrix with $n>0$. \medskip

\textbf{(a)} We say that $A$ is \emph{reducible} if there exist two disjoint
nonempty subsets $I$ and $J$ of $\left[  n\right]  $ such that $I\cup
J=\left[  n\right]  $ and such that%
\[
A_{i,j}=0\ \ \ \ \ \ \ \ \ \ \text{for all }i\in I\text{ and }j\in J.
\]
Equivalently, $A$ is reducible if and only if there exists a permutation
matrix $P\in\mathbb{R}^{n\times n}$ such that%
\begin{align*}
P^{-1}AP  &  =\left(
\begin{array}
[c]{cc}%
B & C\\
0_{\left(  n-r\right)  \times r} & D
\end{array}
\right)  \ \ \ \ \ \ \ \ \ \ \text{for some }0<r<n\\
&
\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \text{and
some matrices }B,C,D.
\end{align*}
(Note that $P^{-1}AP$ is the matrix obtained from $A$ by permuting the rows
and then permuting the columns using the same permutation.) \medskip

\textbf{(b)} We say that $A$ is \emph{irreducible} if $A$ is not reducible.
\medskip

\textbf{(c)} We say that $A$ is \emph{primitive} if there exists some $m>0$
such that $A^{m}>0$.
\end{definition}

\begin{theorem}
[Perron--Frobenius theorem 2]\label{thm.posmat.pf2}Let $A\in\mathbb{R}%
^{n\times n}$ be nonnegative and irreducible and satisfy $n>0$. Then: \medskip

\textbf{(a)} We have $\rho\left(  A\right)  >0$. \medskip

\textbf{(b)} The number $\rho\left(  A\right)  $ is an eigenvalue of $A$ and
has algebraic multiplicity $1$ (and therefore geometric multiplicity $1$ as
well). \medskip

\textbf{(c)} There is a unique $\rho\left(  A\right)  $-eigenvector $x=\left(
x_{1},x_{2},\ldots,x_{n}\right)  ^{T}\in\mathbb{C}^{n}$ of $A$ with
$x_{1}+x_{2}+\cdots+x_{n}=1$. This eigenvector $x$ is furthermore positive.
(It is called the \emph{Perron vector} of $A$.) \medskip

\textbf{(d)} There is a unique vector $y=\left(  y_{1},y_{2},\ldots
,y_{n}\right)  ^{T}\in\mathbb{C}^{n}$ such that $y^{T}A=\rho\left(  A\right)
y^{T}$ and $x_{1}y_{1}+x_{2}y_{2}+\cdots+x_{n}y_{n}=1$. This vector $y$ is
also positive. \medskip

\textbf{(e)} Assume furthermore that $A$ is primitive. We have%
\[
\left(  \dfrac{1}{\rho\left(  A\right)  }A\right)  ^{m}\rightarrow
xy^{T}\ \ \ \ \ \ \ \ \ \ \text{as }m\rightarrow\infty.
\]


\textbf{(f)} Assume again that $A$ is primitive. The only eigenvalue of $A$
that has absolute value $\rho\left(  A\right)  $ is $\rho\left(  A\right)  $ itself.
\end{theorem}

\begin{remark}
If $A$ is the row-stochastic matrix $P$ corresponding to a Markov chain, then:

\begin{itemize}
\item $A$ is irreducible if and only if there is no set of states from which
you cannot escape (except for the empty set and for the set of all states);

\item $A$ is primitive if and only if there is an $m>0$ such that we can get
from any state to any state in exactly $m$ minutes (this technical condition
rules out the kind of \textquotedblleft oscillation\textquotedblright\ that
prevented us from finding a steady state for $P=\left(
\begin{array}
[c]{cc}%
0 & 1\\
1 & 0
\end{array}
\right)  $).
\end{itemize}
\end{remark}

\subsubsection{Proof of Perron}

We shall now approach the proof of the Perron theorem (Theorem
\ref{thm.posmat.perron}). We begin with some notations:

\begin{definition}
For the rest of this subsection, we shall use the following notations:

Let $n$ be a fixed positive integer. Let $e:=\left(  1,1,\ldots,1\right)
^{T}\in\mathbb{R}^{n}$.
\end{definition}

\begin{remark}
\label{rmk.posmat.e-rowsums}\textbf{(a)} An $n\times n$-matrix $A$ satisfies
$Ae=e$ if and only if all row sums of $A$ equal $1$. \medskip

\textbf{(b)} An $n\times n$-matrix $A$ satisfies $e^{T}A=e^{T}$ if and only if
all column sums of $A$ equal $1$.
\end{remark}

\begin{proof}
[Proof of Remark \ref{rmk.posmat.e-rowsums}.]\textbf{(a)} This is because the
entries of the column vector $Ae$ are the row sums of $A$, while the entries
of the column vector $e$ are all $1$. \medskip

\textbf{(b)} This is because the entries of the row vector $e^{T}A$ are the
column sums of $A$, while the entries of the row vector $e^{T}$ are all $1$.
\end{proof}

Next, we state a few lemmas. The first one is an instance of the obvious idea
that when some addends in a sum have different signs, they interfere
destructively with each other:

\begin{lemma}
\label{lem.posmat.proper-subset}Let $y\in\mathbb{R}^{n}$ and $v=\left(
v_{1},v_{2},\ldots,v_{n}\right)  ^{T}\in\mathbb{R}^{n}$ be two column vectors
such that $y\geq0$ and $y\neq0$ and $y^{T}v=0$. Let $a_{1},a_{2},\ldots,a_{n}$
be nonnegative reals. Then, there exists some \textbf{proper} subset $K$ of
$\left[  n\right]  $ such that%
\[
\left\vert \sum_{k=1}^{n}a_{k}v_{k}\right\vert \leq\sum_{k\in K}%
a_{k}\left\vert v_{k}\right\vert .
\]

\end{lemma}

\begin{proof}
Let
\[
P:=\left\{  k\in\left[  n\right]  \ \mid\ v_{k}>0\right\}
\ \ \ \ \ \ \ \ \ \ \text{and}\ \ \ \ \ \ \ \ \ \ N:=\left\{  k\in\left[
n\right]  \ \mid\ v_{k}<0\right\}  .
\]
Then, $P$ and $N$ are two disjoint subsets of $\left[  n\right]  $, and every
$k\in\left[  n\right]  \setminus\left(  P\cup N\right)  $ satisfies $v_{k}=0$.
Therefore, we can break up the sum $\sum_{k=1}^{n}a_{k}v_{k}$ as follows:%
\begin{align}
\sum_{k=1}^{n}a_{k}v_{k}  &  =\sum_{k\in P}a_{k}\underbrace{v_{k}%
}_{\substack{=\left\vert v_{k}\right\vert \\\text{(since }k\in P\text{ and
thus }v_{k}>0\text{)}}}+\sum_{k\in N}a_{k}\underbrace{v_{k}}%
_{\substack{=-\left\vert v_{k}\right\vert \\\text{(since }k\in N\text{ and
thus }v_{k}<0\text{)}}}\nonumber\\
&  =\sum_{k\in P}a_{k}\left\vert v_{k}\right\vert -\sum_{k\in N}%
a_{k}\left\vert v_{k}\right\vert . \label{pf.lem.posmat.proper-subset.1}%
\end{align}
Note that both sums $\sum_{k\in P}a_{k}\left\vert v_{k}\right\vert $ and
$\sum_{k\in N}a_{k}\left\vert v_{k}\right\vert $ are nonnegative (since
$a_{1},a_{2},\ldots,a_{n}$ are nonnegative). However, it is easy to see that
$\left\vert x-y\right\vert \leq\max\left\{  x,y\right\}  $ for any two
nonnegative reals $x$ and $y$. Applying this to $x=\sum_{k\in P}%
a_{k}\left\vert v_{k}\right\vert $ and $y=\sum_{k\in N}a_{k}\left\vert
v_{k}\right\vert $, we thus obtain%
\[
\left\vert \sum_{k\in P}a_{k}\left\vert v_{k}\right\vert -\sum_{k\in N}%
a_{k}\left\vert v_{k}\right\vert \right\vert \leq\max\left\{  \sum_{k\in
P}a_{k}\left\vert v_{k}\right\vert ,\ \sum_{k\in N}a_{k}\left\vert
v_{k}\right\vert \right\}  .
\]
Therefore,%
\begin{equation}
\left\vert \sum_{k\in P}a_{k}\left\vert v_{k}\right\vert -\sum_{k\in N}%
a_{k}\left\vert v_{k}\right\vert \right\vert \leq\sum_{k\in K}a_{k}\left\vert
v_{k}\right\vert , \label{pf.lem.posmat.proper-subset.2}%
\end{equation}
where $K$ is either $P$ or $N$ depending on whether $\sum_{k\in P}%
a_{k}\left\vert v_{k}\right\vert \geq\sum_{k\in N}a_{k}\left\vert
v_{k}\right\vert $ or not. Consider this $K$. Clearly, $K$ is a subset of
$\left[  n\right]  $.

We shall now show that $K$ is a \textbf{proper} subset of $\left[  n\right]
$. Indeed, assume the contrary. Thus, $K=\left[  n\right]  $. However, $K$ is
either $P$ or $N$ (by definition). We WLOG assume that $K=P$ (since the case
$K=N$ is analogous). Thus, $P=K=\left[  n\right]  $. In other words, $v_{k}>0$
for each $k\in\left[  n\right]  $ (by the definition of $P$). In other words,
$v>0$.

Write the vector $y$ as $y=\left(  y_{1},y_{2},\ldots,y_{n}\right)  ^{T}$.
Then, at least one $k\in\left[  n\right]  $ satisfies $y_{k}\neq0$ (since
$y\neq0$). Consider this $k$. Then, $y_{k}>0$ (since $y\geq0$).

However, $y^{T}v=\sum_{j=1}^{n}y_{j}v_{j}$. This is a sum of nonnegative reals
(since $y\geq0$ and $v>0$), and at least one of its addends is actually
positive (indeed, $\underbrace{y_{k}}_{>0}\underbrace{v_{k}}%
_{\substack{>0\\\text{(since }v>0\text{)}}}>0$). Thus, the entire sum is
positive. Therefore, $y^{T}v>0$. But this contradicts $y^{T}v=0$. This
contradiction shows that our assumption was false. Hence, we have shown that
$K$ is a proper subset of $\left[  n\right]  $. Furthermore, from
(\ref{pf.lem.posmat.proper-subset.1}), we obtain%
\[
\left\vert \sum_{k=1}^{n}a_{k}v_{k}\right\vert =\left\vert \sum_{k\in P}%
a_{k}\left\vert v_{k}\right\vert -\sum_{k\in N}a_{k}\left\vert v_{k}%
\right\vert \right\vert \leq\sum_{k\in K}a_{k}\left\vert v_{k}\right\vert
\]
(by (\ref{pf.lem.posmat.proper-subset.2})). This proves Lemma
\ref{lem.posmat.proper-subset}.
\end{proof}

The next three lemmas will help us derive some parts of Theorem
\ref{thm.posmat.perron} from others:

\begin{lemma}
[crucifix lemma, stochastic case]\label{lem.posmat.crucifix-e}Let
$A\in\mathbb{R}^{n\times n}$ satisfy $A>0$ and $Ae=e$. Let $y\in\mathbb{R}%
^{n}$ satisfy $y^{T}A=y^{T}$ and $y\geq0$ and $y^{T}e=1$ (that is, the sum of
all entries of $y$ is $1$). Then,%
\[
A^{m}\rightarrow ey^{T}\ \ \ \ \ \ \ \ \ \ \text{as }m\rightarrow\infty.
\]

\end{lemma}

\begin{proof}
Write the vector $y=\left(  y_{1},y_{2},\ldots,y_{n}\right)  ^{T}$. Then, the
entries $y_{1},y_{2},\ldots,y_{n}$ are nonnegative reals (since $y\geq0$), and
we have $y^{T}e=y_{1}+y_{2}+\cdots+y_{n}$ (since $e=\left(  1,1,\ldots
,1\right)  ^{T}$). Hence, $y_{1}+y_{2}+\cdots+y_{n}=y^{T}e=1$.

Thus, the $n$ numbers $y_{1},y_{2},\ldots,y_{n}$ are nonnegative reals whose
sum is $1$ (since $y_{1}+y_{2}+\cdots+y_{n}=1$). Hence, all these $n$ numbers
$y_{1},y_{2},\ldots,y_{n}$ lie in the interval $\left[  0,1\right]  $. In
other words,%
\begin{equation}
y_{j}\in\left[  0,1\right]  \ \ \ \ \ \ \ \ \ \ \text{for each }j\in\left[
n\right]  . \label{pf.lem.posmat.crucifix-e.yj-in}%
\end{equation}


From $Ae=e$, we conclude that all row sums of $A$ equal $1$ (by Remark
\ref{rmk.posmat.e-rowsums} \textbf{(a)}). Since $A>0$, this implies that all
entries $A_{u,v}$ of $A$ satisfy%
\begin{equation}
0<A_{u,v}\leq1. \label{pf.lem.posmat.crucifix-e.0<A}%
\end{equation}


Let
\[
\mu:=1-\min\left\{  A_{u,v}\ \mid\ u,v\in\left[  n\right]  \right\}  .
\]
Then, $0\leq\mu<1$ (by (\ref{pf.lem.posmat.crucifix-e.0<A})). We claim the following:

\begin{statement}
\textit{Claim 1:} For each $i\in\left[  n\right]  $ and each \textbf{proper}
subset $K$ of $\left[  n\right]  $, we have%
\[
\sum_{k\in K}A_{i,k}\leq\mu.
\]

\end{statement}

[\textit{Proof of Claim 1:} Let $i\in\left[  n\right]  $. Let $K$ be a proper
subset of $\left[  n\right]  $. Then, there exists at least one element
$j\in\left[  n\right]  $ satisfying $j\notin K$. Consider this $j$. Then,
$\sum_{\substack{k\in\left[  n\right]  ;\\k\notin K}}A_{i,k}\geq A_{i,j}$
(since $A\geq0$ entails that all $A_{i,k}$ are nonnegative). However,%
\begin{align*}
\sum_{k\in K}A_{i,k}  &  =\underbrace{\sum_{k\in\left[  n\right]  }A_{i,k}%
}_{\substack{=\left(  \text{the }i\text{-th row sum of }A\right)
\\=1\\\text{(since all row sums of }A\text{ equal }1\text{)}}%
}-\underbrace{\sum_{\substack{k\in\left[  n\right]  ;\\k\notin K}}A_{i,k}%
}_{\geq A_{i,j}}\leq1-\underbrace{A_{i,j}}_{\geq\min\left\{  A_{u,v}%
\ \mid\ u,v\in\left[  n\right]  \right\}  }\\
&  \leq1-v=\mu\ \ \ \ \ \ \ \ \ \ \left(  \text{by the definition of }%
\mu\right)  .
\end{align*}
This proves Claim 1.] \medskip

Next, we claim:

\begin{statement}
\textit{Claim 2:} For any $i,j\in\left[  n\right]  $ and any $m\in\mathbb{N}$,
we have%
\[
\left\vert \left(  A^{m}-ey^{T}\right)  _{i,j}\right\vert \leq\mu^{m}.
\]

\end{statement}

Once Claim 2 is proved, it will follow easily that $\left(  A^{m}%
-ey^{T}\right)  _{i,j}\rightarrow0$ as $m\rightarrow\infty$ (because $0\leq
\mu<1$), so that $A^{m}\rightarrow ey^{T}$, and the lemma will thus follow.

[\textit{Proof of Claim 2:} We induct on $m$:

\textit{Base case:} For any $i,j\in\left[  n\right]  $, we have%
\[
\left(  A^{0}-ey^{T}\right)  _{i,j}=\underbrace{\left(  A^{0}\right)  _{i,j}%
}_{\substack{=\left(  I_{n}\right)  _{i,j}\\=\delta_{i,j}}}-y_{j}%
=\underbrace{\delta_{i,j}}_{\in\left\{  0,1\right\}  }-\underbrace{y_{j}%
}_{\substack{\in\left[  0,1\right]  \\\text{(by
(\ref{pf.lem.posmat.crucifix-e.yj-in}))}}}\in\left[  -1,1\right]
\]
and therefore $\left\vert \left(  A^{0}-ey^{T}\right)  _{i,j}\right\vert
\leq1=\mu^{0}$. In other words, Claim 2 holds for $m=0$.

\textit{Induction step:} Let $p\in\mathbb{N}$. Assume (as the induction
hypothesis) that Claim 2 holds for $m=p$. We must now show that Claim 2 also
holds for $m=p+1$.

Let
\[
B:=A^{p}-ey^{T}\ \ \ \ \ \ \ \ \ \ \text{and}\ \ \ \ \ \ \ \ \ \ C:=A^{p+1}%
-ey^{T}.
\]
Our induction hypothesis says that Claim 2 holds for $m=p$. In other words,
for all $i,j\in\left[  n\right]  $, we have $\left\vert \left(  A^{p}%
-ey^{T}\right)  _{i,j}\right\vert \leq\mu^{p}$. In other words, for all
$i,j\in\left[  n\right]  $, we have%
\begin{equation}
\left\vert B_{i,j}\right\vert \leq\mu^{p}
\label{pf.lem.posmat.crucifix-e.c2.IH}%
\end{equation}
(since $B=A^{p}-ey^{T}$).

Our goal is to show that Claim 2 holds for $m=p+1$. In other words, our goal
is to show that for all $i,j\in\left[  n\right]  $, we have $\left\vert
\left(  A^{p+1}-ey^{T}\right)  _{i,j}\right\vert \leq\mu^{p+1}$. In other
words, our goal is to show that for all $i,j\in\left[  n\right]  $, we have
$\left\vert C_{i,j}\right\vert \leq\mu^{p+1}$ (since $C=A^{p+1}-ey^{T}$).

Fix $i,j\in\left[  n\right]  $. Thus we must prove that $\left\vert
C_{i,j}\right\vert \leq\mu^{p+1}$.

From $B=A^{p}-ey^{T}$, we obtain%
\[
AB=A\left(  A^{p}-ey^{T}\right)  =\underbrace{AA^{p}}_{=A^{p+1}}%
-\underbrace{Ae}_{=e}y^{T}=A^{p+1}-ey^{T}=C.
\]
Hence, $C=AB$, so that (by the definition of the product of two matrices we
have)%
\begin{equation}
C_{i,j}=\sum_{k=1}^{n}A_{i,k}B_{k,j}. \label{pf.lem.posmat.crucifix-e.c2.Cij=}%
\end{equation}


For each $m\in\mathbb{N}$, we have
\[
y^{T}A^{m}=y^{T}%
\]
(indeed, this is easily proved by induction on $m$, using the fact that
$y^{T}A=y^{T}$). Applying this to $m=p$, we obtain $y^{T}A^{p}=y^{T}$.

Recall that $B_{\bullet,j}$ denotes the $j$-th column of the matrix $B$. We
have%
\[
y^{T}\underbrace{B}_{=A^{p}-ey^{T}}=y^{T}\left(  A^{p}-ey^{T}\right)
=\underbrace{y^{T}A^{p}}_{=y^{T}}-\underbrace{y^{T}e}_{=1}y^{T}=y^{T}%
-y^{T}=0,
\]
and thus $y^{T}B_{\bullet,j}=0$ (since $y^{T}B_{\bullet,j}$ is the $j$-th
entry of the row vector $y^{T}B$). Also, from $y^{T}e=1\neq0$, we obtain
$y\neq0$. Hence, Lemma \ref{lem.posmat.proper-subset} (applied to
$B_{\bullet,j}$ and $B_{k,j}$ and $A_{i,k}$ instead of $v$ and $v_{k}$ and
$a_{i}$) yields that there exists some \textbf{proper} subset $K$ of $\left[
n\right]  $ such that%
\begin{equation}
\left\vert \sum_{k=1}^{n}A_{i,k}B_{k,j}\right\vert \leq\sum_{k\in K}%
A_{i,k}\left\vert B_{k,j}\right\vert .
\label{pf.lem.posmat.crucifix-e.c2.Kieq}%
\end{equation}
Consider this $K$. Now, from (\ref{pf.lem.posmat.crucifix-e.c2.Cij=}), we
obtain%
\begin{align*}
\left\vert C_{i,j}\right\vert  &  =\left\vert \sum_{k=1}^{n}A_{i,k}%
B_{k,j}\right\vert \leq\sum_{k\in K}A_{i,k}\underbrace{\left\vert
B_{k,j}\right\vert }_{\substack{\leq\mu^{p}\\\text{(by
(\ref{pf.lem.posmat.crucifix-e.c2.IH}),}\\\text{applied to }k\text{ instead of
}i\text{)}}}\ \ \ \ \ \ \ \ \ \ \left(  \text{by
(\ref{pf.lem.posmat.crucifix-e.c2.Kieq})}\right) \\
&  \leq\underbrace{\sum_{k\in K}A_{i,k}}_{\substack{\leq\mu\\\text{(by Claim
1)}}}\mu^{p}\ \ \ \ \ \ \ \ \ \ \left(  \text{since }A_{i,k}\geq0\text{ for
all }k\right) \\
&  \leq\mu\mu^{p}=\mu^{p+1}.
\end{align*}
As we explained above, this completes the induction step. Thus, Claim 2 is
proved.] \medskip

Finishing the proof of Lemma \ref{lem.posmat.crucifix-e} is now easy: We have
$0\leq\mu<1$ and therefore $\mu^{m}\rightarrow0$ as $m\rightarrow\infty$.
Thus, Claim 2 shows that for any $i,j\in\left[  n\right]  $, we have%
\[
\left(  A^{m}-ey^{T}\right)  _{i,j}\rightarrow0\ \ \ \ \ \ \ \ \ \ \text{as
}m\rightarrow\infty.
\]
In other words, $A^{m}-ey^{T}\rightarrow0$ as $m\rightarrow\infty$. In other
words, $A^{m}\rightarrow ey^{T}$ as $m\rightarrow\infty$. This proves Lemma
\ref{lem.posmat.crucifix-e}.
\end{proof}

Next, we generalize Lemma \ref{lem.posmat.crucifix-e} by replacing $e$ by an
arbitrary positive vector $x$:

\begin{lemma}
[crucifix lemma, general case]\label{lem.posmat.crucifix-x}Let $A\in
\mathbb{R}^{n\times n}$ satisfy $A>0$. Let $x\in\mathbb{R}^{n}$ satisfy $Ax=x$
and $x>0$. Let $y\in\mathbb{R}^{n}$ satisfy $y^{T}A=y^{T}$ and $y\geq0$ and
$y^{T}x=1$. Then,%
\[
A^{m}\rightarrow xy^{T}\ \ \ \ \ \ \ \ \ \ \text{as }m\rightarrow\infty.
\]

\end{lemma}

\begin{proof}
We can easily reduce this to Lemma \ref{lem.posmat.crucifix-e}.

Indeed, write the vector $x$ as $x=\left(  x_{1},x_{2},\ldots,x_{n}\right)
^{T}$. Thus, $x_{1},x_{2},\ldots,x_{n}$ are positive reals (since $x>0$). Let%
\[
D:=\operatorname*{diag}\left(  x_{1},x_{2},\ldots,x_{n}\right)  .
\]
Thus, we easily obtain $De=x$. Moreover, the matrix $D$ is diagonal, so that
$D^{T}=D$. Furthermore, we have
\[
D^{-1}AD>0
\]
(since the $\left(  u,v\right)  $-th entry of the matrix $D^{-1}AD$ is
$\underbrace{x_{u}^{-1}}_{>0}\ \ \underbrace{A_{u,v}}_{>0}%
\ \ \underbrace{x_{v}}_{>0}>0$ for each $u,v\in\left[  n\right]  $) and%
\[
D^{-1}ADe=e
\]
(since $A\underbrace{De}_{=x}=Ax=x=De$) and
\[
\underbrace{\left(  Dy\right)  ^{T}}_{\substack{=y^{T}D^{T}=y^{T}%
D\\\text{(since }D^{T}=D\text{)}}}D^{-1}AD=y^{T}\underbrace{DD^{-1}}_{=I_{n}%
}AD=\underbrace{y^{T}A}_{=y^{T}}D=y^{T}D
\]
and%
\[
\left(  Dy\right)  ^{T}=y^{T}\underbrace{D^{T}}_{=D}=y^{T}D\geq0
\]
(since $y\geq0$ and $D\geq0$) and%
\[
\underbrace{\left(  Dy\right)  ^{T}}_{=y^{T}D}e=y^{T}\underbrace{De}%
_{=x}=y^{T}x=1.
\]
Hence, we can apply Lemma \ref{lem.posmat.crucifix-e} to $D^{-1}AD$ and $Dy$
instead of $A$ and $y$. We thus obtain that%
\[
\left(  D^{-1}AD\right)  ^{m}\rightarrow e\left(  Dy\right)  ^{T}%
\ \ \ \ \ \ \ \ \ \ \text{as }m\rightarrow\infty.
\]
Since we have $\left(  D^{-1}AD\right)  ^{m}=D^{-1}A^{m}D$ for each
$m\in\mathbb{N}$ (in fact, this is essentially the equality
(\ref{pf.prop.schurtri.similar.same.f.1}) we proved long ago), we can rewrite
this as%
\[
D^{-1}A^{m}D\rightarrow e\left(  Dy\right)  ^{T}\ \ \ \ \ \ \ \ \ \ \text{as
}m\rightarrow\infty
\]
Multiplying both sides by $D$ from the left and by $D^{-1}$ from the right, we
can transform this into%
\[
A^{m}\rightarrow De\left(  Dy\right)  ^{T}D^{-1}\ \ \ \ \ \ \ \ \ \ \text{as
}m\rightarrow\infty.
\]
In other words,%
\[
A^{m}\rightarrow xy^{T}\ \ \ \ \ \ \ \ \ \ \text{as }m\rightarrow\infty
\]
(since $\underbrace{De}_{=x}\underbrace{\left(  Dy\right)  ^{T}}_{=y^{T}%
D}D^{-1}=xy^{T}\underbrace{DD^{-1}}_{=I_{n}}=xy^{T}$). This proves Lemma
\ref{lem.posmat.crucifix-x}.
\end{proof}

\begin{lemma}
\label{lem.posmat.uniques}Let $A\in\mathbb{C}^{n\times n}$ and $B\in
\mathbb{C}^{n\times n}$ be two matrices such that $\rho\left(  A\right)  =1$
and $\operatorname*{rank}B\leq1$. Assume that%
\begin{equation}
A^{m}\rightarrow B\ \ \ \ \ \ \ \ \ \ \text{as }m\rightarrow\infty.
\label{eq.lem.posmat.uniques.ass}%
\end{equation}


Then: \medskip

\textbf{(a)} The number $1$ is an eigenvalue of $A$ and has algebraic
multiplicity $1$ (and therefore geometric multiplicity $1$ as well). \medskip

\textbf{(b)} There is at most one $1$-eigenvector $x=\left(  x_{1}%
,x_{2},\ldots,x_{n}\right)  ^{T}\in\mathbb{C}^{n}$ of $A$ with $x_{1}%
+x_{2}+\cdots+x_{n}=1$. \medskip

\textbf{(c)} There is at most one vector $y=\left(  y_{1},y_{2},\ldots
,y_{n}\right)  ^{T}\in\mathbb{C}^{n}$ such that $y^{T}A=y^{T}$ and $x_{1}%
y_{1}+x_{2}y_{2}+\cdots+x_{n}y_{n}=1$. \medskip

\textbf{(d)} The only eigenvalue of $A$ that has absolute value $1$ is $1$.
\end{lemma}

\begin{proof}
We know that $A$ has a Schur triangularization. Let $\left(  U,T\right)  $ be
a Schur triangularization of $A$. Then, $U\in\mathbb{C}^{n\times n}$ is
unitary and $T\in\mathbb{C}^{n\times n}$ is upper-triangular and $A=UTU^{\ast
}$. Since $U$ is unitary, we have $U^{\ast}=U^{-1}$, so that
$A=UT\underbrace{U^{\ast}}_{=U^{-1}}=UTU^{-1}$. Moreover, Proposition
\ref{prop.schurtri.schurtri.T-diag} shows that the diagonal entries of $T$ are
the eigenvalues of $A$ (with their algebraic multiplicities). Let $\lambda
_{1},\lambda_{2},\ldots,\lambda_{n}$ be these diagonal entries, in the order
in which they appear on the diagonal of $T$. Thus, $\lambda_{1},\lambda
_{2},\ldots,\lambda_{n}$ are the eigenvalues of $A$.

For each $m\in\mathbb{N}$, we have%
\begin{align*}
A^{m}  &  =\left(  UTU^{-1}\right)  ^{m}\ \ \ \ \ \ \ \ \ \ \left(
\text{since }A=UTU^{-1}\right) \\
&  =UT^{m}U^{-1}%
\end{align*}
(this is essentially the equality (\ref{pf.prop.schurtri.similar.same.f.1}) we
proved long ago). Hence, our assumption (\ref{eq.lem.posmat.uniques.ass}) can
be rewritten as follows:
\begin{equation}
UT^{m}U^{-1}\rightarrow B\ \ \ \ \ \ \ \ \ \ \text{as }m\rightarrow\infty.
\label{pf.lem.posmat.uniques.1}%
\end{equation}
Thus, $T^{m}$ converges to a limit (namely, to $U^{-1}BU$) as $m\rightarrow
\infty$. Therefore, each entry of $T^{m}$ converges to a limit as
$m\rightarrow\infty$. In particular, $\lim\limits_{m\rightarrow\infty}\left(
T^{m}\right)  _{i,i}$ is well-defined for each $i\in\left[  n\right]  $.
However, since $T$ is an upper-triangular matrix with diagonal entries
$\lambda_{1},\lambda_{2},\ldots,\lambda_{n}$, its $m$-th power $T^{m}$ (for
each $m\in\mathbb{N}$) is an upper-triangular matrix with diagonal entries
$\lambda_{1}^{m},\lambda_{2}^{m},\ldots,\lambda_{n}^{m}$. In particular, we
have $\left(  T^{m}\right)  _{i,i}=\lambda_{i}^{m}$ for each $i\in\left[
n\right]  $. Therefore, $\lim\limits_{m\rightarrow\infty}\lambda_{i}^{m}$ is
well-defined for each $i\in\left[  n\right]  $ (since we have shown that
$\lim\limits_{m\rightarrow\infty}\left(  T^{m}\right)  _{i,i}$ is well-defined
for each $i\in\left[  n\right]  $). This shows that
\begin{equation}
\left\vert \lambda_{i}\right\vert <1\text{ or }\lambda_{i}%
=1\ \ \ \ \ \ \ \ \ \ \text{for each }i\in\left[  n\right]  .
\label{pf.lem.posmat.uniques.2}%
\end{equation}


If we had $\left\vert \lambda_{i}\right\vert <1$ for each $i\in\left[
n\right]  $, then we would have $\rho\left(  A\right)  <1$ (since $\lambda
_{1},\lambda_{2},\ldots,\lambda_{n}$ are the eigenvalues of $A$); but this
would contradict $\rho\left(  A\right)  =1$. Hence, we cannot have $\left\vert
\lambda_{i}\right\vert <1$ for each $i\in\left[  n\right]  $. According to
(\ref{pf.lem.posmat.uniques.2}), this shows that at least one $i\in\left[
n\right]  $ satisfies $\lambda_{i}=1$. In other words, $1$ is an eigenvalue of
$A$ (since $\lambda_{1},\lambda_{2},\ldots,\lambda_{n}$ are the eigenvalues of
$A$). Moreover, (\ref{pf.lem.posmat.uniques.2}) shows that the only eigenvalue
of $A$ that has absolute value $1$ is $1$. This proves Lemma
\ref{lem.posmat.uniques} \textbf{(d)}.

Let $k$ be the number of all $i\in\left[  n\right]  $ that satisfy
$\lambda_{i}=1$. The matrix $\lim\limits_{m\rightarrow\infty}T^{m}$ is an
upper-triangular matrix whose diagonal entries are $\lim\limits_{m\rightarrow
\infty}\lambda_{1}^{m},\ \lim\limits_{m\rightarrow\infty}\lambda_{2}%
^{m},\ \ldots,\ \lim\limits_{m\rightarrow\infty}\lambda_{n}^{m}$ (since each
$T^{m}$ is an upper-triangular matrix whose diagonal entries are $\lambda
_{1}^{m},\lambda_{2}^{m},\ldots,\lambda_{n}^{m}$). In view of
(\ref{pf.lem.posmat.uniques.2}), we conclude that exactly $k$ of these
diagonal entries are nonzero (since $\left\vert \lambda_{i}\right\vert <1$
entails $\lim\limits_{m\rightarrow\infty}\lambda_{i}^{m}=0$, whereas
$\lambda_{i}=1$ entails $\lim\limits_{m\rightarrow\infty}\lambda_{i}^{m}%
=\lim\limits_{m\rightarrow\infty}1^{m}=1$). Therefore, the matrix
$\lim\limits_{m\rightarrow\infty}T^{m}$ has rank $\geq k$ (since the rank of
an upper-triangular matrix is always $\geq$ to the number of its diagonal
entries that are nonzero\footnote{This can be proved in several ways; for
example, nonzero diagonal entries can be used to create a nonzero principal
minor.}). Therefore, the matrix $U\left(  \lim\limits_{m\rightarrow\infty
}T^{m}\right)  U^{-1}$ has rank $\geq k$ as well (since $U$ is invertible, so
that $\operatorname*{rank}\left(  U\left(  \lim\limits_{m\rightarrow\infty
}T^{m}\right)  U^{-1}\right)  =\operatorname*{rank}\left(  \lim
\limits_{m\rightarrow\infty}T^{m}\right)  $). In other words, the matrix $B$
has rank $\geq k$ (since (\ref{pf.lem.posmat.uniques.1}) yields $U\left(
\lim\limits_{m\rightarrow\infty}T^{m}\right)  U^{-1}=B$). In view of the
assumption $\operatorname*{rank}B\leq1$, this entails $k\leq1$. In other
words, at most one $i\in\left[  n\right]  $ satisfies $\lambda_{i}=1$ (since
$k$ is the number of all $i\in\left[  n\right]  $ that satisfy $\lambda_{i}%
=1$). In other words, the algebraic multiplicity of the eigenvalue $1$ of $A$
is at most $1$ (since $\lambda_{1},\lambda_{2},\ldots,\lambda_{n}$ are the
eigenvalues of $A$). Since this multiplicity is at least $1$ (because we know
that $1$ is an eigenvalue of $A$), we thus conclude that this multiplicity is
$1$. This proves Lemma \ref{lem.posmat.uniques} \textbf{(a)}.

The geometric multiplicity of the eigenvalue $1$ of $A$ is therefore also $1$.
Hence, the $1$-eigenvectors $x=\left(  x_{1},x_{2},\ldots,x_{n}\right)
^{T}\in\mathbb{C}^{n}$ of $A$ form a $1$-dimensional vector subspace of
$\mathbb{C}^{n}$. Thus, at most one of these $1$-eigenvectors $x$ satisfies
$x_{1}+x_{2}+\cdots+x_{n}=1$. This proves Lemma \ref{lem.posmat.uniques}
\textbf{(b)}.

The matrices $A$ and $A^{T}$ have the same characteristic polynomial, and thus
have the same eigenvalues with the same algebraic multiplicities. Hence, the
algebraic multiplicity of the eigenvalue $1$ of $A^{T}$ equals the algebraic
multiplicity of the eigenvalue $1$ of $A$. Since the latter multiplicity is
$1$, we thus conclude that the former is $1$ as well. Hence, the geometric
multiplicity of the eigenvalue $1$ of $A^{T}$ must also equal $1$.

The vectors $y=\left(  y_{1},y_{2},\ldots,y_{n}\right)  ^{T}\in\mathbb{C}^{n}$
satisfying $y^{T}A=y^{T}$ are the $1$-eigenvectors of $A^{T}$ (since the
equality $y^{T}A=y^{T}$ is equivalent to $A^{T}y=y$ (because $y^{T}A=\left(
A^{T}y\right)  ^{T}$)). Hence, they form a $1$-dimensional vector subspace of
$\mathbb{C}^{n}$ (since the geometric multiplicity of the eigenvalue $1$ of
$A^{T}$ is $1$). Thus, at most one of these vectors $y$ satisfies $x_{1}%
y_{1}+x_{2}y_{2}+\cdots+x_{n}y_{n}=1$. This proves Lemma
\ref{lem.posmat.uniques} \textbf{(c)}.
\end{proof}

Proving Theorem \ref{thm.posmat.perron} is now an easy matter of combining lemmas:

\begin{proof}
[Proof of Theorem \ref{thm.posmat.perron}.]\textbf{(a)} We have $A>0$. Thus,
all the more, we have $A_{i,i}>0$ for some $i\in\left[  n\right]  $. Hence,
Corollary \ref{cor.posmat.specrad.leq1c} \textbf{(c)} yields $\rho\left(
A\right)  >0$. This proves Theorem \ref{thm.posmat.perron} \textbf{(a)}.

This also shows that the matrix $\dfrac{1}{\rho\left(  A\right)  }A$ is
well-defined. Moreover, $\dfrac{1}{\rho\left(  A\right)  }A>0$ (since $A>0$).
The matrix $\dfrac{1}{\rho\left(  A\right)  }A$ has the same eigenvectors as
$A$ (with the same algebraic multiplicities), while the corresponding
eigenvalues are those of $A$ multiplied by $\dfrac{1}{\rho\left(  A\right)  }%
$. Hence, if we replace $A$ by $\dfrac{1}{\rho\left(  A\right)  }A$, then the
claim of Theorem \ref{thm.posmat.perron} does not substantially change (i.e.,
it gets replaced by an equivalent claim). Thus, let us replace $A$ by
$\dfrac{1}{\rho\left(  A\right)  }A$. This replacement causes $\rho\left(
A\right)  $ to become $1$ (since $\rho\left(  \dfrac{1}{\rho\left(  A\right)
}A\right)  =\dfrac{1}{\rho\left(  A\right)  }\rho\left(  A\right)  =1$). Thus,
we have $\rho\left(  A\right)  =1$ now.

Next, we shall show that $A$ has a positive $1$-eigenvector. Indeed, from
$\rho\left(  A\right)  =1$, we see that $A$ has an eigenvalue $\lambda
\in\mathbb{C}$ with $\left\vert \lambda\right\vert =1$. Consider this
$\lambda$. Pick any nonzero $\lambda$-eigenvector $z=\left(  z_{1}%
,z_{2},\ldots,z_{n}\right)  ^{T}\in\mathbb{C}^{n}$ of $A$. Thus, $Az=\lambda
z$. Moreover, $z\neq0$ and thus $\left\vert z\right\vert \neq0$. Also,
clearly, $\left\vert z\right\vert \geq0$.

From $A>0$, we obtain $A=\left\vert A\right\vert $. Hence,%
\begin{align*}
\underbrace{A}_{=\left\vert A\right\vert }\left\vert z\right\vert  &
=\left\vert A\right\vert \cdot\left\vert z\right\vert \geq\left\vert
\underbrace{Az}_{=\lambda z}\right\vert \ \ \ \ \ \ \ \ \ \ \left(  \text{by
Proposition \ref{prop.posmat.prod-geq-Ax} \textbf{(a)}}\right) \\
&  =\left\vert \lambda z\right\vert =\underbrace{\left\vert \lambda\right\vert
}_{=1}\cdot\left\vert z\right\vert =\left\vert z\right\vert .
\end{align*}
Thus, Corollary \ref{cor.posmat.supereig-cor} \textbf{(b)} (applied to
$w=\left\vert z\right\vert $) yields $A\left\vert z\right\vert =\left\vert
z\right\vert >0$. In other words, $\left\vert z\right\vert $ is a positive
$1$-eigenvector of $A$.

We have thus constructed a positive $1$-eigenvector of $A$. The same argument
(applied to $A^{T}$ instead of $A$) lets us construct a positive
$1$-eigenvector of $A^{T}$ (since $A>0$ entails $A^{T}>0$, and since
$\rho\left(  A^{T}\right)  =\rho\left(  A\right)  =1$). Let these two
eigenvectors be $x$ and $y$ (with $x$ being the $1$-eigenvector of $A$ and $y$
being the one of $A^{T}$). Thus, $x\in\mathbb{R}^{n}$ satisfies $Ax=x$ and
$x>0$, whereas $y\in\mathbb{R}^{n}$ satisfies $A^{T}y=y$ and $y>0$.

Write the vectors $x$ and $y$ as $x=\left(  x_{1},x_{2},\ldots,x_{n}\right)
^{T}$ and $y=\left(  y_{1},y_{2},\ldots,y_{n}\right)  ^{T}$.

By scaling $x$ by an appropriately chosen real scalar\footnote{namely, by the
scalar $\dfrac{1}{x_{1}+x_{2}+\cdots+x_{n}}$}, we can achieve $x_{1}%
+x_{2}+\cdots+x_{n}=1$. So we WLOG assume that $x_{1}+x_{2}+\cdots+x_{n}=1$.

Moreover, by scaling $y$ by an appropriately chosen positive real scalar
\footnote{Specifically, we must scale $y$ by $\dfrac{1}{y^{T}x}$ (which is a
well-defined positive real scalar, since $x>0$ and $y>0$ entail $y^{T}x>0$).},
we can achieve $y^{T}x=1$ (without disturbing the properties $A^{T}y=y$ and
$y>0$). So we WLOG assume that $y^{T}x=1$. In other words, $x_{1}y_{1}%
+x_{2}y_{2}+\cdots+x_{n}y_{n}=1$ (since $y^{T}x=x_{1}y_{1}+x_{2}y_{2}%
+\cdots+x_{n}y_{n}$).

By taking transposes on both sides of the equality $A^{T}y=y$, we obtain
$y^{T}A=y^{T}$ (since $\left(  A^{T}y\right)  ^{T}=y^{T}A$). Thus, Lemma
\ref{lem.posmat.crucifix-x} yields
\begin{equation}
A^{m}\rightarrow xy^{T}\ \ \ \ \ \ \ \ \ \ \text{as }m\rightarrow\infty.
\label{pf.thm.posmat.perron.e.at}%
\end{equation}
This proves Theorem \ref{thm.posmat.perron} \textbf{(e)} (since $\dfrac
{1}{\rho\left(  A\right)  }A=A$ (because $\rho\left(  A\right)  =1$)).

The matrix $xy^{T}$ has rank%
\[
\operatorname*{rank}\left(  xy^{T}\right)  \leq\operatorname*{rank}x\leq1
\]
(since $x$ is a column vector). Hence, we can apply Lemma
\ref{lem.posmat.uniques} to $B=xy^{T}$ (because of
(\ref{pf.thm.posmat.perron.e.at})).

Lemma \ref{lem.posmat.uniques} \textbf{(a)} tells us that the number $1$ is an
eigenvalue of $A$ and has algebraic multiplicity $1$ (and therefore geometric
multiplicity $1$ as well). In view of $\rho\left(  A\right)  =1$, this proves
Theorem \ref{thm.posmat.perron} \textbf{(b)}.

Lemma \ref{lem.posmat.uniques} \textbf{(d)} tells us that the only eigenvalue
of $A$ that has absolute value $1$ is $1$. In view of $\rho\left(  A\right)
=1$, this proves Theorem \ref{thm.posmat.perron} \textbf{(f)}.

We already know that $x=\left(  x_{1},x_{2},\ldots,x_{n}\right)  ^{T}%
\in\mathbb{C}^{n}$ is an $1$-eigenvector of $A$ with $x_{1}+x_{2}+\cdots
+x_{n}=1$. Moreover, Lemma \ref{lem.posmat.uniques} \textbf{(b)} tells us that
there is at most one such $1$-eigenvector; therefore, $x$ is the only such
$1$-eigenvector. This proves Theorem \ref{thm.posmat.perron} \textbf{(c)}
(since $\rho\left(  A\right)  =1$, and since we also know that $x$ is positive).

We already know that $y=\left(  y_{1},y_{2},\ldots,y_{n}\right)  ^{T}%
\in\mathbb{C}^{n}$ is a vector such that $y^{T}A=y^{T}$ and $x_{1}y_{1}%
+x_{2}y_{2}+\cdots+x_{n}y_{n}=1$. Moreover, Lemma \ref{lem.posmat.uniques}
\textbf{(c)} tells us that there is at most one such vector; therefore, $y$ is
the only such vector. This proves Theorem \ref{thm.posmat.perron} \textbf{(c)}
(since $\rho\left(  A\right)  =1$, and since we know that $y$ is positive).
\end{proof}

We shall now prove the Perron--Frobenius theorems.

[...]

\newpage

\begin{thebibliography}{99999999}                                                                                         %


\bibitem[AigZie14]{AigZie}%
\href{https://doi.org/10.1007/978-3-662-57265-8}{Martin Aigner, G\"{u}nter M.
Ziegler, \textit{Proofs from the Book}, 6th edition, Springer 2018.}

\bibitem[AndDos10]{AndDos}\href{https://bookstore.ams.org/xyz-13}{Titu
Andreescu, Gabriel Dospinescu, \textit{Problems from the Book}, 2nd edition,
XYZ Press 2010}.

\bibitem[AndDos12]{AndDosS}\href{https://bookstore.ams.org/xyz-6}{Titu
Andreescu, Gabriel Dospinescu, \textit{Straight from the Book}, XYZ Press
2012}.

\bibitem[Bartle14]{Bartle14}Padraic Bartlett, \textit{Math 108b: Advanced
Linear Algebra, Winter 2014}, 2014.\newline\url{http://web.math.ucsb.edu/~padraic/ucsb_2013_14/math108b_w2014/math108b_w2014.html}

\bibitem[Bourba74]{Bourba74}%
\href{http://libgen.rs/book/index.php?md5=3270565F6D0052635A1550883588204C}{Nicolas
Bourbaki, \textit{Algebra I: Chapters 1--3}, Addison-Wesley 1974}.

\bibitem[Bourba03]{Bourba03}%
\href{http://libgen.rs/book/index.php?md5=F0E3884F2EFB5C1F0B08054605192F6A}{Nicolas
Bourbaki, \textit{Algebra II: Chapters 4--7}, Springer 2003}.

\bibitem[BoyDip12]{BoyDip12}William E. Boyce, Richard C. DiPrima,
\textit{Elementary Differential Equations}, 10th edition, Wiley 2012.

\bibitem[ChaSed97]{ChaSed97}%
\href{https://www.jstor.org/stable/10.4169/j.ctt19b9mbq}{Gengzhe Chang, Thomas
W. Sederberg, \textit{Over and Over Again}, Anneli Lax New Mathematical
Library \textbf{39}, The Mathematical Association of America 1997}.

\bibitem[Conrad]{Conrad}Keith Conrad, \textit{Expository notes
(\textquotedblleft blurbs\textquotedblright)}.\newline\url{https://kconrad.math.uconn.edu/blurbs/}

\bibitem[Edward05]{Edwards-Essays}%
\href{https://doi.org/10.1007/b138656}{Harold M. Edwards, \textit{Essays in
Constructive Mathematics}, Springer 2005}.\newline See
\url{https://www.math.nyu.edu/faculty/edwardsh/eserrata.pdf} for errata.

\bibitem[Edward95]{Edward95}%
\href{https://doi.org/10.1007/978-0-8176-4446-8}{Harold M. Edwards,
\textit{Linear Algebra}, Springer 1995}.

\bibitem[Elman20]{Elman20}Richard Elman, \textit{Lectures on Abstract
Algebra}, 28 September 2020.\newline\url{https://www.math.ucla.edu/~rse/algebra_book.pdf}

\bibitem[GalQua20]{GalQua20}Jean Gallier and Jocelyn Quaintance,
\textit{Algebra, Topology, Differential Calculus, and Optimization Theory For
Computer Science and Engineering}, 11 November 2020.\newline\url{https://www.cis.upenn.edu/~jean/gbooks/geomath.html}

\bibitem[Geck20]{Geck20}\href{https://doi.org/10.13001/ela.2020.5055}{Meinolf
Geck, \textit{On Jacob's construction of the rational canonical form of a
matrix}, Electronic Journal of Linear Algebra \textbf{36} (2020), pp.
177--182}.

\bibitem[GelAnd17]{GelAnd}%
\href{https://doi.org/10.1007/978-3-319-58988-6}{R\u{a}zvan Gelca, Titu
Andreescu, \textit{Putnam and Beyond}, 2nd edition, Springer 2017}.

\bibitem[Goodma15]{Goodman}Frederick M. Goodman, \textit{Algebra: Abstract and
Concrete}, edition 2.6, 1 May 2015.\newline%
\url{http://homepage.math.uiowa.edu/~goodman/algebrabook.dir/book.2.6.pdf} .

\bibitem[Grinbe15]{detnotes}Darij Grinberg, \textit{Notes on the combinatorial
fundamentals of algebra}, 10 January 2019.\newline%
\url{http://www.cip.ifi.lmu.de/~grinberg/primes2015/sols.pdf} \newline The
numbering of theorems and formulas in this link might shift when the project
gets updated; for a \textquotedblleft frozen\textquotedblright\ version whose
numbering is guaranteed to match that in the citations above, see
\url{https://github.com/darijgr/detnotes/releases/tag/2019-01-10} .

\bibitem[Grinbe19]{lina}Darij Grinberg, \textit{Notes on linear algebra}, 4th
December 2019.\newline\url{http://www.cip.ifi.lmu.de/~grinberg/t/16f/lina.pdf}

\bibitem[Grinbe19]{trach}Darij Grinberg, \textit{The trace Cayley-Hamilton
theorem}, 14 July 2019.\newline\url{https://www.cip.ifi.lmu.de/~grinberg/algebra/trach.pdf}

\bibitem[Grinbe21]{21s}Darij Grinberg, \textit{An Introduction to Algebraic
Combinatorics [Math 701, Spring 2021 lecture notes]}, 10 September
2021.\newline\url{https://www.cip.ifi.lmu.de/~grinberg/t/21s/lecs.pdf}

\bibitem[Heffer20]{Heffer20}Jim Hefferon, \textit{Linear Algebra}, 4th edition
2020.\newline\url{http://joshua.smcvt.edu/linearalgebra}

\bibitem[Ho14]{Ho-rear2}%
\href{https://www.math.hkust.edu.hk/excalibur/v19_n3.pdf}{Law Ka Ho,
\textit{Variations and Generalisations to the Rearrangement Inequality},
Mathematical Excalibur \textbf{19}, Number 3, pp. 1--2, 4}.

\bibitem[HorJoh13]{HorJoh13}%
\href{http://www.cse.zju.edu.cn/eclass/attachments/2015-10/01-1446086008-145421.pdf}{Roger
A. Horn, Charles R. Johnson, \textit{Matrix analysis}, Cambridge University
Press, 2nd edition 2013}.\newline See
\url{https://www.cambridge.org/us/files/7413/7180/9643/Errata_HJ_Matrix_Analysis_2nd_ed.pdf}
and \url{https://www.kth.se/social/files/5707c3bef2765428eba786d3/errata.pdf}
for errata.

\bibitem[Hung07]{Hung07}%
\href{http://refkol.ro/matek/mathbooks/!Books!/Secrets in Inequalities (volume 1) Pham Kim Hung.pdf}{Pham
Kim Hung, \textit{Secrets in Inequalities, volume 1}, GIL 2007}.

\bibitem[Ivanov08]{Ivanov08}Nikolai V. Ivanov, \textit{Linear Recurrences}, 17
January 2008.\newline\url{https://nikolaivivanov.files.wordpress.com/2014/02/ivanov2008arecurrence.pdf}

\bibitem[KDLM05]{KDLM05}%
\href{http://elib.mi.sanu.ac.rs/files/journals/tm/14/tm813.pdf}{Zoran
Kadelburg, Dusan Dukic, Milivoje Lukic and Ivan Matic, \textit{Inequalities of
Karamata, Schur and Muirhead, and some applications}, The teaching of
mathematics \textbf{VIII} (2005), issue 1, pp. 31--45}.

\bibitem[Knapp16]{Knapp1}Anthony W. Knapp, \textit{Basic Algebra}, digital
second edition 2016.\newline\url{http://www.math.stonybrook.edu/~aknapp/download.html}

\bibitem[Korner20]{Korner20}T. W. K\"{o}rner, \textit{Where Do Numbers Come
From?}, Cambridge University Press 2020.\newline See
\url{https://web.archive.org/web/20190813160507/https://www.dpmms.cam.ac.uk/~twk/Number.pdf}
for a preprint.\newline See \url{https://www.dpmms.cam.ac.uk/~twk/} for errata
and solutions.

\bibitem[LaNaSc16]{LaNaSc16}Isaiah Lankham, Bruno Nachtergaele, Anne
Schilling, \textit{Linear Algebra As an Introduction to Abstract Mathematics},
2016.\newline\url{https://www.math.ucdavis.edu/~anne/linear_algebra/mat67_course_notes.pdf}

\bibitem[Li99]{Li-rear1}%
\href{https://www.math.hkust.edu.hk/excalibur/v4_n3.pdf}{Kin-Yin Li,
\textit{Rearrangement Inequality}, Mathematical Excalibur \textbf{4}, Number
3, pp. 1--2, 4}.

\bibitem[LibLav15]{LibLav15}Leo Liberti, Carlile Lavor, \textit{Six
mathematical gems from the history of distance geometry}, International
Transactions in Operational Research 2015.\newline\url{https://doi.org/10.1111/itor.12170}

\bibitem[Loehr14]{Loehr14}%
\href{https://elblogdecontar.files.wordpress.com/2017/01/ebookdaraz-advanced-linear-algebra.pdf}{Nicholas
Loehr, \textit{Advanced Linear Algebra}, CRC Press 2014}.

\bibitem[MaOlAr11]{MaOlAr11}%
\href{https://doi.org/10.1007/978-0-387-68276-1}{Albert W. Marshall, Ingram
Olkin, Barry C. Arnold, \textit{Inequalities: Theory of Majorization and Its
Applications}, 2nd Edition, Springer 2011}.

\bibitem[Markus83]{Markus83}%
\href{https://archive.org/details/recursion-sequences}{Aleksei Ivanovich
Markushevich, \textit{Recursion sequences}, Mir Publishers, Moscow, 2nd
printing 1983}.

\bibitem[Mate16]{Mate16}Attila M\'{a}t\'{e}, \textit{The Cayley-Hamilton
Theorem}, version 28 March 2016.\newline\url{http://www.sci.brooklyn.cuny.edu/~mate/misc/cayley_hamilton.pdf}

\bibitem[Melian01]{Melian01}Mar\'{\i}a Victoria Meli\'{a}n, \textit{Linear
recurrence relations with constant coefficients}, 9 April 2001.\newline\url{http://matematicas.uam.es/~mavi.melian/CURSO_15_16/web_Discreta/recurrence.pdf}

\bibitem[Nathan21]{Nathan21}\href{https://arxiv.org/abs/2109.01746v1}{Melvyn
B. Nathanson, \textit{The Muirhead-Rado inequality, 1 Vector majorization and
the permutohedron}, arXiv:2109.01746v1}.

\bibitem[OmClVi11]{OmClVi11}Kevin C. O'Meara, John Clark, Charles I.
Vinsonhaler, \textit{Advanced Topics in Linear Algebra: Weaving Matrix
Problems through the Weyr Form}, Oxford University Press 2011.

\bibitem[PolSze78]{PolSze78}%
\href{https://doi.org/10.1007/978-3-642-61983-0}{George P\'{o}lya, Gabor
Szeg\H{o}, \textit{Problems and Theorems in Analysis I}, Springer 1978
(reprinted 1998)}.

\bibitem[Prasol94]{Prasolov}Viktor V. Prasolov,
\textit{\href{http://www2.math.su.se/~mleites/books/prasolov-1994-problems.pdf}{\textit{Problems
and Theorems in Linear Algebra}}}, Translations of Mathematical Monographs,
vol. \#134, AMS 1994.

\bibitem[Shapir15]{Shapir15}Helene Shapiro, \textit{Linear Algebra and
Matrices: Topics for a Second Course}, Pure and applied undergraduate texts
\textbf{24}, AMS 2015.

\bibitem[Shurma15]{Shurma15}Jerry Shurman, \textit{The Cayley-Hamilton theorem
via multilinear algebra}, \url{http://people.reed.edu/~jerry/332/28ch.pdf} .
Part of the collection \textit{Course Materials for Mathematics 332: Algebra},
available at \url{http://people.reed.edu/~jerry/332/mat.html}

\bibitem[Silves00]{Silvest}%
\href{https://web.archive.org/web/20140505161153/http://www.mth.kcl.ac.uk/~jrs/gazette/blocks.pdf}{John
R. Silvester, \textit{Determinants of Block Matrices}, The Mathematical
Gazette, Vol. 84, No. 501 (Nov., 2000), pp. 460--467.}

\bibitem[Steele04]{Steele04}%
\href{http://www.ma.huji.ac.il/~ehudf/courses/Ineq09/The Cauchy-Schwarz Master Class .pdf}{J.
Michael Steele, \textit{The Cauchy--Schwarz Master Class: An Introduction to
the Art of Mathematical Inequalities}, Cambridge University Press
2004}.\newline See
\url{http://www-stat.wharton.upenn.edu/~steele/Publications/Books/CSMC/CSMC_Errata.pdf}
for errata.

\bibitem[Steinb06]{Steinb06}Mark Steinberger, \textit{Algebra}, 31 August
2006.\newline\url{https://web.archive.org/web/20180821125315/https://www.albany.edu/~mark/algebra.pdf}

\bibitem[Straub83]{Straub83}Howard Straubing, \textit{A combinatorial proof of
the Cayley-Hamilton theorem}, Discrete Mathematics, Volume 43, Issues 2--3,
1983, pp. 273--279.\newline\url{https://doi.org/10.1016/0012-365X(83)90164-4}

\bibitem[Strick20]{Strick20}Neil Strickland, \textit{Linear mathematics for
applications}, 11 February 2020.\newline\url{https://neilstrickland.github.io/linear_maths/notes/linear_maths.pdf}

\bibitem[Swanso20]{Swanso20}Irena Swanson, \textit{Introduction with Analysis
with Complex Numbers}, 2020.\newline\url{https://web.archive.org/web/20201012174324/https://people.reed.edu/~iswanson/analysisconstructR.pdf}

\bibitem[Tao07]{Tao07}Terence Tao, \textit{The Jordan normal form and the
Euclidean algorithm}, 12 October 2007.\newline\url{https://terrytao.wordpress.com/2007/10/12/the-jordan-normal-form-and-the-euclidean-algorithm/}

\bibitem[Taylor20]{Taylor20}%
\href{https://bookstore.ams.org/amstext-45/}{Michael Taylor, \textit{Linear
Algebra}, AMS 2020}.\newline See
\url{https://mtaylor.web.unc.edu/wp-content/uploads/sites/16915/2018/04/linalg.pdf}
for a preprint.

\bibitem[TreBau97]{TreBau97}Lloyd Nicholas Trefethen, David Bau III,
\textit{Numerical linear algebra}, SIAM 1997.\newline See
\url{https://people.maths.ox.ac.uk/trefethen/text.html} for the first five
sections (\textquotedblleft lectures\textquotedblright).

\bibitem[Treil15]{Treil15}Sergei Treil, \textit{Linear Algebra Done Wrong},
2017.\newline\url{https://www.math.brown.edu/~treil/papers/LADW/LADW.html}

\bibitem[Walker87]{Walker87}%
\href{https://web.archive.org/web/20170809055317/https://www.math.nmsu.edu/~elbert/AbsAlgeb.pdf}{Elbert
A. Walker, \textit{Introduction to Abstract Algebra}, Random House/Birkhauser,
New York, 1987.}

\bibitem[Woerde16]{Woerde16}Hugo J. Woerdeman, \textit{Advanced Linear
Algebra}, CRC Press 2016.

\bibitem[Zeilbe85]{Zeilbe}%
\href{http://www.math.rutgers.edu/~zeilberg/mamarimY/DM85.pdf}{Doron
Zeilberger, \textit{A combinatorial approach to matrix algebra}, Discrete
Mathematics 56 (1985), pp. 61--72.}

\bibitem[Zill17]{Zill17}Dennis G. Zill, \textit{A First Course in Differential
Equations with Modeling Applications}, Cengage 2017.
\end{thebibliography}


\end{document}