\documentclass[numbers=enddot,12pt,final,onecolumn,notitlepage]{scrartcl}%
\usepackage[headsepline,footsepline,manualmark]{scrlayer-scrpage}
\usepackage[all,cmtip]{xy}
\usepackage{amssymb}
\usepackage{amsmath}
\usepackage{amsthm}
\usepackage{framed}
\usepackage{comment}
\usepackage{color}
\usepackage{hyperref}
\usepackage{datetime}
\usepackage[sc]{mathpazo}
\usepackage[T1]{fontenc}
\usepackage{needspace}
\usepackage{tabls}
%TCIDATA{OutputFilter=latex2.dll}
%TCIDATA{Version=5.50.0.2960}
%TCIDATA{LastRevised=Wednesday, April 22, 2026 07:51:31}
%TCIDATA{SuppressPackageManagement}
%TCIDATA{<META NAME="GraphicsSave" CONTENT="32">}
%TCIDATA{<META NAME="SaveForMode" CONTENT="1">}
%TCIDATA{BibliographyScheme=Manual}
%TCIDATA{Language=American English}
%BeginMSIPreambleData
\providecommand{\U}[1]{\protect\rule{.1in}{.1in}}
%EndMSIPreambleData
\theoremstyle{definition}
\newtheorem{theo}{Theorem}[section]
\newenvironment{theorem}[1][]
{\begin{theo}[#1]\begin{leftbar}}
{\end{leftbar}\end{theo}}
\newtheorem{lem}[theo]{Lemma}
\newenvironment{lemma}[1][]
{\begin{lem}[#1]\begin{leftbar}}
{\end{leftbar}\end{lem}}
\newtheorem{prop}[theo]{Proposition}
\newenvironment{proposition}[1][]
{\begin{prop}[#1]\begin{leftbar}}
{\end{leftbar}\end{prop}}
\newtheorem{defi}[theo]{Definition}
\newenvironment{definition}[1][]
{\begin{defi}[#1]\begin{leftbar}}
{\end{leftbar}\end{defi}}
\newtheorem{remk}[theo]{Remark}
\newenvironment{remark}[1][]
{\begin{remk}[#1]\begin{leftbar}}
{\end{leftbar}\end{remk}}
\newtheorem{coro}[theo]{Corollary}
\newenvironment{corollary}[1][]
{\begin{coro}[#1]\begin{leftbar}}
{\end{leftbar}\end{coro}}
\newtheorem{conv}[theo]{Convention}
\newenvironment{condition}[1][]
{\begin{conv}[#1]\begin{leftbar}}
{\end{leftbar}\end{conv}}
\newtheorem{warn}[theo]{Warning}
\newenvironment{conclusion}[1][]
{\begin{warn}[#1]\begin{leftbar}}
{\end{leftbar}\end{warn}}
\newtheorem{conj}[theo]{Conjecture}
\newenvironment{conjecture}[1][]
{\begin{conj}[#1]\begin{leftbar}}
{\end{leftbar}\end{conj}}
\newtheorem{exam}[theo]{Example}
\newenvironment{example}[1][]
{\begin{exam}[#1]\begin{leftbar}}
{\end{leftbar}\end{exam}}
\newtheorem{exmp}[theo]{Exercise}
\newenvironment{exercise}[1][]
{\begin{exmp}[#1]\begin{leftbar}}
{\end{leftbar}\end{exmp}}
\newenvironment{statement}{\begin{quote}}{\end{quote}}
\newtheorem{quest}[theo]{TODO}
\newenvironment{todo}[1][]
{\begin{quest}[#1]\begin{leftbar}}
{\end{leftbar}\end{quest}}
\iffalse
\newenvironment{proof}[1][Proof]{\noindent\textbf{#1.} }{\ \rule{0.5em}{0.5em}}
\fi
\let\sumnonlimits\sum
\let\prodnonlimits\prod
\let\cupnonlimits\bigcup
\let\capnonlimits\bigcap
\renewcommand{\sum}{\sumnonlimits\limits}
\renewcommand{\prod}{\prodnonlimits\limits}
\renewcommand{\bigcup}{\cupnonlimits\limits}
\renewcommand{\bigcap}{\capnonlimits\limits}
\setlength\tablinesep{3pt}
\setlength\arraylinesep{3pt}
\setlength\extrarulesep{3pt}
\voffset=0cm
\hoffset=-0.7cm
\setlength\textheight{22.5cm}
\setlength\textwidth{15.5cm}
\newenvironment{verlong}{}{}
\newenvironment{vershort}{}{}
\newenvironment{noncompile}{}{}
\excludecomment{verlong}
\includecomment{vershort}
\excludecomment{noncompile}
\newcommand{\id}{\operatorname{id}}
\newcommand{\NN}{\mathbb{N}}
\ihead{Notes on linear algebra (\today, \currenttime)}
\ohead{page \thepage}
\cfoot{}
\begin{document}

\title{Notes on linear algebra}
\author{Darij Grinberg}
\date{
%TCIMACRO{\TeXButton{today}{\today} }%
%BeginExpansion
\today
%EndExpansion
\ at \currenttime\\
These notes are \textbf{frozen} in a (very) unfinished state. Currently, only
the basics of matrix algebra have been completed (products, triangularity, row
operations etc.).}
\maketitle
\tableofcontents

\section{Preface}

These notes are accompanying
\href{http://www.cip.ifi.lmu.de/~grinberg/t/16f/}{a class on applied linear
algebra (Math 4242) I am giving at the University of Minneapolis in Fall 2016}
(the website of the class is \url{http://www.cip.ifi.lmu.de/~grinberg/t/16f/}
). They contain both the material of the class (although with no promise of
timeliness!) and the homework exercises (and possibly some additional exercises).

There will (probably) be no actual applications in these notes, but only the
mathematical material used in these applications. If time allows, the notes
will contain tutorials on the use of \href{http://www.sagemath.org/}{SageMath}
(a computer algebra system suited both for numerical and for algebraic computations).

Sections marked with an asterisk (*) are not a required part of the Math 4242 course.

Several good books have been written on linear algebra; these notes are not
supposed to replace any of them. Let me just mention four sources I can
recommend\footnote{I have \textbf{not} read any of the books myself (apart
from fragments). My recommendations are based on cursory skimming and random
appraisal of specific points; I therefore cannot guarantee anything.}:

\begin{itemize}
\item Olver's and Shakiban's \cite{OlvSha06} is the traditional text for Math
4242 at UMN. It might be the best place to learn about the applications of
linear algebra.

\item Hefferon's \cite{Heffer16} is a free text that does things slowly but
rigorously (at least for the standards of an introductory linear-algebra
text). It has plenty of examples (and exercises with solutions), fairly
detailed proofs, and occasional applications. (Which is why it is over 500
pages long; I hope you can easily decide what to skip based on your
preferences.) Altogether, I think it does a lot very well. The main drawback
is its lack of the theory of bilinear forms (but I don't know if we will even
have time for that).

\item Lankham's, Nachtergaele's and Schilling's \cite{LaNaSc16} is a set of
notes for introductory linear algebra, doing the abstract side (vector spaces,
linear maps) early on and in some detail.

\item Treil's \cite{Treil15} is another free text; this is written for a more
mathematically mature reader, and has a slight bias towards the linear algebra
useful for functional analysis.\footnote{The title of the book is a play on
Axler's \textquotedblleft Linear Algebra Done Right\textquotedblright, which
is biased towards analysis (or, rather, against algebra) to a ridiculous
extent. Axler seems to write really well, but the usefulness of this book is
severely limited by its obstinate avoidance of anything that looks too
explicit and algebraic.}
\end{itemize}

{\small [Please let the authors know if you find any errors or unclarities.
Feel free to ask me if you want your doubts resolved beforehand.]}

Also, some previous iterations of Math 4242 have left behind interesting notes:

\begin{itemize}
\item Stephen Lewis, Fall 2014, \url{http://www.stephen-lewis.net/4242/}
(enable javascript!).

\item Natalie Sheils, Fall 2015,
\url{http://math.umn.edu/~nesheils/F15_M4242/LectureNotes.html} (yes, those
are on dropbox).
\end{itemize}

There are countless other sets of lecture notes on the internet\footnote{Let
me mention some: Two good-looking advanced texts for a mathematically prepared
reader are Cameron's \cite{Camero08} and Kowalski's \cite{Kowals16}; a reader
bored with the present notes might want to take a look at them. On the other
side, Wildon's notes \cite{Wildon16} include a lot of examples and geometric
illustrations (but are probably too brief to peruse as a standalone text),
whereas Chen's notes \cite{Chen08} boast numerous applications (and seem quite
readable, though I have not looked at them in depth).}, books in the library,
and even books on the internet if you know where to look. You can find an
overview of (published, paper) books in \cite{Drucker12} (but usually without
assessing their quality), and another (with reviews) on the MAA website
\url{http://www.maa.org/tags/linear-algebra} . (Reviews on Amazon and
goodreads are usually just good for a laugh.)

The notes you are reading are under construction, and will remain so for at
least the whole Fall term 2016. Please let me know of any errors and
unclarities you encounter (my email address is
\texttt{darijgrinberg@gmail.com})\footnote{The sourcecode of the notes is also
publicly available at \url{https://github.com/darijgr/lina} .}. Thank you!

\subsection{Acknowledgments}

I would like to thank Mark Richard for correcting a typo in the notes.

\section{\label{chp.intro}Introduction to matrices}

In this chapter, we shall introduce matrices, define the basic operations with
matrices (addition, scaling, multiplication and powers) and two fundamental
families of matrices (zero matrices and identity matrices), and state the most
fundamental of their properties (and even prove some of them). We shall not go
very deep here (most of this chapter corresponds to a part of \cite[\S A.2]%
{LaNaSc16}), but we will give plenty of examples and some detailed proofs that
will (hopefully) help you get some experience with the material.

\subsection{Matrices and entries}

In the following, we shall study matrices filled with numbers. This is not the
most general thing to study (we could also fill matrices with other things,
such as polynomials -- and in fact, such matrices are highly useful); nor will
we be very precise about it. In fact, for most of this chapter, we shall not
even specify what we mean by \textquotedblleft numbers\textquotedblright, even
though the word \textquotedblleft number\textquotedblright\ is far from being
a well-defined notion. However, as soon as we start caring about (say)
computer calculations, we will have to spend some words specifying our
\textquotedblleft numbers\textquotedblright\ more precisely. See Section
\ref{sect.intro.numbers} for what the word \textquotedblleft
number\textquotedblright\ means.

We shall use the symbol $\mathbb{N}$ for the set $\left\{  0,1,2,\ldots
\right\}  $. This is the set of all nonnegative integers.\footnote{Some
authors use the symbol $\mathbb{N}$ for the set $\left\{  1,2,3,\ldots
\right\}  $ (the set of all \textbf{positive} integers) instead.
Unfortunately, there is no consensus here. If you want to avoid notational
conflicts, use the notation $\mathbb{Z}_{\geq0}$ for $\left\{  0,1,2,\ldots
\right\}  $ and the notation $\mathbb{Z}_{>0}$ for $\left\{  1,2,3,\ldots
\right\}  $. These notations, at least, are self-explanatory (once you know
that $\mathbb{Z}$ denotes the set of all integers, i.e., $\left\{
\ldots,-2,-1,0,1,2,\ldots\right\}  $).}

\begin{definition}
If $n\in\mathbb{N}$ and $m\in\mathbb{N}$, then an $n\times m$\textit{-matrix}
simply means a rectangular table with $n$ rows and $m$ columns, such that each
cell is filled with a number.
\end{definition}

For example, $\left(
\begin{array}
[c]{ccc}%
1 & 7 & 2\\
-\sqrt{2} & 6 & 1/3
\end{array}
\right)  $ is a $2\times3$-matrix, whereas $\left(
\begin{array}
[c]{c}%
1\\
-2\\
0
\end{array}
\right)  $ is a $3\times1$-matrix.\footnote{For the friends of stupid examples
(me?), here are two more: $\left(
\vphantom{\begin{array}{c} a \\ a \\ a \end{array}}\right)  $ is a $3\times
0$-matrix (it contains no cells, and thus no numbers), and $\left(
\phantom{\begin{array}{ccc} a & a & a \end{array}}\right)  $ is a $0\times
3$-matrix (again, with no numbers because it has no cells). For various
technical reasons (\cite{deBoor}), it is helpful to regard such empty matrices
as different.}

The word \textquotedblleft$n\times m$-matrix\textquotedblright\ is pronounced
\textquotedblleft$n$-by-$m$-matrix\textquotedblright\ (and sometimes even
written in this way). Notice that the sign \textquotedblleft$\times
$\textquotedblright\ in \textquotedblleft$n\times m$-matrix\textquotedblright%
\ is a symbol whose purpose is to separate $n$ from $m$; it does \textbf{not}
mean multiplication (although, of course, an $n\times m$-matrix does have $nm$
entries). Do not rewrite \textquotedblleft$2\times3$-matrix\textquotedblright%
\ as \textquotedblleft$6$-matrix\textquotedblright\ (after all, a $2\times
3$-matrix is not the same as a $6\times1$-matrix, although $2\cdot3=6\cdot1$).

\begin{definition}
The word \textquotedblleft\textit{matrix}\textquotedblright\ will encompass
all $n\times m$-matrices for all possible values of $n$ and $m$.
\end{definition}

\begin{definition}
The \textit{dimensions} of an $n\times m$-matrix $A$ are the two integers $n$
and $m$. When they are equal (that is, $n=m$), we say that $A$ is a
\textit{square matrix}, and call $n$ its \textit{size}.

For a rectangular matrix $A$, we also will sometimes say that
\textquotedblleft$A$ has size $n\times m$\textquotedblright\ when we mean that
\textquotedblleft$A$ is an $n\times m$-matrix\textquotedblright.
\end{definition}

\begin{definition}
If $A$ is an $n\times m$-matrix, and if $i\in\left\{  1,2,\ldots,n\right\}  $
and $j\in\left\{  1,2,\ldots,m\right\}  $, then $A_{i,j}$ will denote the
entry of $A$ in row $i$ and column $j$. This entry is also called the $\left(
i,j\right)  $\textit{-th entry of }$A$ (or simply the $\left(  i,j\right)
$\textit{-entry of }$A$).
\end{definition}

For example, $\left(
\begin{array}
[c]{ccc}%
1 & 7 & 2\\
-\sqrt{2} & 6 & 1/3
\end{array}
\right)  _{1,3}=2$. Note that this notation $A_{i,j}$ (for the $\left(
i,j\right)  $-th entry of a matrix $A$) is not standard in literature! Some
authors instead use $A_{j}^{i}$ (the $i$ on top is not an exponent, but just a
superscript) or $a_{i,j}$ (where $a$ is the lowercase letter corresponding to
the uppercase letter $A$ denoting the matrix\footnote{This notation is bad for
two reasons: First, it forces you to always denote matrices by uppercase
letters; second, it doesn't let you write things like $\left(
\begin{array}
[c]{ccc}%
1 & 7 & 2\\
-\sqrt{2} & 6 & 1/3
\end{array}
\right)  _{1,3}$.}). Many authors often drop the comma between $i$ and $j$ (so
they call it $A_{ij}$ or $a_{ij}$); this notation is slightly ambiguous (does
$A_{132}$ mean $A_{13,2}$ or $A_{1,32}$ ?). Unfortunately, some authors use
the notation $A_{i,j}$ for something else called a \textit{cofactor} of $A$
(which is, in a sense, quite the opposite of the $\left(  i,j\right)  $-th
entry of $A$); but we will never do this here (and probably we will not really
get into cofactors anyway).

\subsection{The matrix builder notation}

I would like to do something interesting, but I am forced to introduce more
notations. Please have patience with me. Let me introduce a notation for
building a matrix out of a bunch of entries:

\begin{definition}
Let $n\in\mathbb{N}$ and $m\in\mathbb{N}$. Assume that you are given a number
$a_{i,j}$ for each pair $\left(  i,j\right)  $ of an integer $i\in\left\{
1,2,\ldots,n\right\}  $ and $j\in\left\{  1,2,\ldots,m\right\}  $. Then,
$\left(  a_{i,j}\right)  _{1\leq i\leq n,\ 1\leq j\leq m}$ shall denote the
$n\times m$-matrix whose $\left(  i,j\right)  $-th entry is $a_{i,j}$ for all
$i\in\left\{  1,2,\ldots,n\right\}  $ and $j\in\left\{  1,2,\ldots,m\right\}
$. (To say it differently: $\left(  a_{i,j}\right)  _{1\leq i\leq n,\ 1\leq
j\leq m}$ shall denote the $n\times m$-matrix $A$ such that $A_{i,j}=a_{i,j}$
for all $i\in\left\{  1,2,\ldots,n\right\}  $ and $j\in\left\{  1,2,\ldots
,m\right\}  $. In other words,%
\[
\left(  a_{i,j}\right)  _{1\leq i\leq n,\ 1\leq j\leq m}=\left(
\begin{array}
[c]{cccc}%
a_{1,1} & a_{1,2} & \cdots & a_{1,m}\\
a_{2,1} & a_{2,2} & \cdots & a_{2,m}\\
\vdots & \vdots & \ddots & \vdots\\
a_{n,1} & a_{n,2} & \cdots & a_{n,m}%
\end{array}
\right)  .
\]
)
\end{definition}

Some examples:

\begin{itemize}
\item What is the matrix $\left(  i-j\right)  _{1\leq i\leq2,\ 1\leq j\leq3}$
? By definition, it is the $2\times3$-matrix whose $\left(  i,j\right)  $-th
entry is $i-j$ for all $i\in\left\{  1,2\right\}  $ and $j\in\left\{
1,2,3\right\}  $. Thus, $\left(  i-j\right)  _{1\leq i\leq2,\ 1\leq j\leq
3}=\left(
\begin{array}
[c]{ccc}%
1-1 & 1-2 & 1-3\\
2-1 & 2-2 & 2-3
\end{array}
\right)  =\left(
\begin{array}
[c]{ccc}%
0 & -1 & -2\\
1 & 0 & -1
\end{array}
\right)  $.

\item We have $\left(  j-i\right)  _{1\leq i\leq2,\ 1\leq j\leq3}=\left(
\begin{array}
[c]{ccc}%
1-1 & 2-1 & 3-1\\
1-2 & 2-2 & 3-2
\end{array}
\right)  =\left(
\begin{array}
[c]{ccc}%
0 & 1 & 2\\
-1 & 0 & 1
\end{array}
\right)  $.

\item We have $\left(  i+j\right)  _{1\leq i\leq3,\ 1\leq j\leq2}=\left(
\begin{array}
[c]{cc}%
1+1 & 1+2\\
2+1 & 2+2\\
3+1 & 3+2
\end{array}
\right)  =\left(
\begin{array}
[c]{cc}%
2 & 3\\
3 & 4\\
4 & 5
\end{array}
\right)  $.

\item We have $\left(  \dfrac{i+1}{j}\right)  _{1\leq i\leq3,\ 1\leq j\leq
3}=\left(
\begin{array}
[c]{ccc}%
\dfrac{1+1}{1} & \dfrac{1+1}{2} & \dfrac{1+1}{3}\\
\dfrac{2+1}{1} & \dfrac{2+1}{2} & \dfrac{2+1}{3}\\
\dfrac{3+1}{1} & \dfrac{3+1}{2} & \dfrac{3+1}{3}%
\end{array}
\right)  =\left(
\begin{array}
[c]{ccc}%
2 & 1 & \dfrac{2}{3}\\
3 & \dfrac{3}{2} & 1\\
4 & 2 & \dfrac{4}{3}%
\end{array}
\right)  $.

\item We have $\left(  \dfrac{i-j}{i+j}\right)  _{1\leq i\leq3,\ 1\leq j\leq
2}=\left(
\begin{array}
[c]{cc}%
\dfrac{1-1}{1+1} & \dfrac{1-2}{1+2}\\
\dfrac{2-1}{2+1} & \dfrac{2-2}{2+2}\\
\dfrac{3-1}{3+1} & \dfrac{3-2}{3+2}%
\end{array}
\right)  =\left(
\begin{array}
[c]{cc}%
0 & -\dfrac{1}{3}\\
\dfrac{1}{3} & 0\\
\dfrac{1}{2} & \dfrac{1}{5}%
\end{array}
\right)  $.
\end{itemize}

The notation $\left(  a_{i,j}\right)  _{1\leq i\leq n,\ 1\leq j\leq m}$ is
fairly standard (you will be understood if you use it), though again there are
variations in the literature.

We used the two letters $i$ and $j$ in the notation $\left(  a_{i,j}\right)
_{1\leq i\leq n,\ 1\leq j\leq m}$, but we could just as well have picked any
other two letters (as long as they aren't already taken for something else).
For example, $\left(  xy\right)  _{1\leq x\leq2,\ 1\leq y\leq2}=\left(
\begin{array}
[c]{cc}%
1\cdot1 & 1\cdot2\\
2\cdot1 & 2\cdot2
\end{array}
\right)  =\left(
\begin{array}
[c]{cc}%
1 & 2\\
2 & 4
\end{array}
\right)  $. For a more confusing example, $\left(  i-j\right)  _{1\leq
i\leq2,\ 1\leq j\leq1}=\left(
\begin{array}
[c]{c}%
1-1\\
2-1
\end{array}
\right)  =\left(
\begin{array}
[c]{c}%
0\\
1
\end{array}
\right)  $ can be rewritten as $\left(  j-i\right)  _{1\leq j\leq2,\ 1\leq
i\leq1}$ (we just renamed the letters $i$ and $j$ as $j$ and $i$ here). Do not
confuse this with the $1\times2$-matrix $\left(  j-i\right)  _{1\leq
i\leq1,\ 1\leq j\leq2}=\left(
\begin{array}
[c]{cc}%
1-1 & 2-1
\end{array}
\right)  =\left(
\begin{array}
[c]{cc}%
0 & 1
\end{array}
\right)  $. The difference between the two matrices $\left(  j-i\right)
_{1\leq j\leq2,\ 1\leq i\leq1}$ and $\left(  j-i\right)  _{1\leq
i\leq1,\ 1\leq j\leq2}$ is the order in which $j$ and $i$ appear in the
subscript (\textquotedblleft$1\leq j\leq2,\ 1\leq i\leq1$\textquotedblright%
\ versus \textquotedblleft$1\leq i\leq1,\ 1\leq j\leq2$\textquotedblright). If
$j$ comes first, then $j$ is the number of the row and $i$ the number of the
column; but if $i$ comes first, then it's the other way round!

Of course, if you decompose an $n\times m$-matrix $A$ into its entries, and
then assemble these entries back into an $n\times m$-matrix (arranged in the
same way as in $A$), then you get back $A$. In other words: For every $n\times
m$-matrix $A$, we have%
\begin{equation}
\left(  A_{i,j}\right)  _{1\leq i\leq n,\ 1\leq j\leq m}=A.
\label{eq.matrix.A-through-Aij}%
\end{equation}


\subsection{Row and column vectors}

Here is some more terminology:

\begin{definition}
Let $n\in\mathbb{N}$. A \textit{row vector of size }$n$ means a $1\times
n$-matrix. A \textit{column vector of size }$n$ means an $n\times1$-matrix.
\end{definition}

For example, $\left(
\begin{array}
[c]{cc}%
a & b
\end{array}
\right)  $ is a row vector of size $2$, while $\left(
\begin{array}
[c]{c}%
a\\
b
\end{array}
\right)  $ is a column vector of size $2$.

The following definition is common-sense:

\begin{definition}
Let $n\in\mathbb{N}$. If $v$ is a row vector of size $n$, then the $\left(
1,j\right)  $-th entry of $v$ (for $j\in\left\{  1,2,\ldots,n\right\}  $) will
also be called the $j$\textit{-th entry} of $v$ (because $v$ has only one row,
so that we don't have to say which row an entry lies in). If $v$ is a column
vector of size $n$, then the $\left(  i,1\right)  $-th entry of $v$ (for
$i\in\left\{  1,2,\ldots,n\right\}  $) will also be called the $i$\textit{-th
entry} of $v$.
\end{definition}

\subsection{Transposes}

\begin{definition}
The \textit{transpose} of an $n\times m$-matrix $A$ is defined to be the
$m\times n$-matrix $\left(  A_{j,i}\right)  _{1\leq i\leq m,\ 1\leq j\leq n}$.
It is denoted by $A^{T}$.
\end{definition}

Let us unravel this confusing-looking definition! It says that the transpose
of an $n\times m$-matrix $A$ is the $m\times n$-matrix whose $\left(
i,j\right)  $-th entry (for $i\in\left\{  1,2,\ldots,m\right\}  $ and
$j\in\left\{  1,2,\ldots,n\right\}  $) is the $\left(  j,i\right)  $-th entry
of $A$. So the transpose of $A$ has the very same entries as $A$, but in
different position: namely, the entry in position $\left(  i,j\right)  $ gets
moved into position $\left(  j,i\right)  $. In other words, the entry that was
in row $i$ and column $j$ gets moved into column $i$ and row $j$. So, visually
speaking, the transpose of the matrix $A$ is obtained by \textquotedblleft
reflecting $A$ around the diagonal\textquotedblright. Some examples should
help clarify this:%
\begin{align*}
\left(
\begin{array}
[c]{ccc}%
a & b & c\\
a^{\prime} & b^{\prime} & c^{\prime}%
\end{array}
\right)  ^{T}  &  =\left(
\begin{array}
[c]{cc}%
a & a^{\prime}\\
b & b^{\prime}\\
c & c^{\prime}%
\end{array}
\right)  ;\\
\left(
\begin{array}
[c]{cc}%
a & b\\
a^{\prime} & b^{\prime}%
\end{array}
\right)  ^{T}  &  =\left(
\begin{array}
[c]{cc}%
a & a^{\prime}\\
b & b^{\prime}%
\end{array}
\right)  ;\\
\left(
\begin{array}
[c]{c}%
a\\
b\\
c
\end{array}
\right)  ^{T}  &  =\left(
\begin{array}
[c]{ccc}%
a & b & c
\end{array}
\right)  .
\end{align*}


Transposes have many uses, but for now we stress one particular use: as a
space-saving device. Namely, if you work with column vectors, you quickly
notice that they take up a lot of vertical space in writing: just see by how
much the column vector $\left(
\begin{array}
[c]{c}%
4\\
-1\\
2\\
0
\end{array}
\right)  $ has stretched the spacing between its line and the lines above and
below\footnote{Additionally, column vectors of size $2$ have the annoying
property that they can get confused for binomial coefficients. To wit,
$\left(
\begin{array}
[c]{c}%
4\\
2
\end{array}
\right)  $ denotes a column vector, whereas $\dbinom{4}{2}$ denotes a binomial
coefficient (which equals the number $6$). The only way to tell them apart is
by the amount of empty space between the parentheses and the entries; this is
not a very reliable way to keep different notations apart.}! It is much more
economical to rewrite it as the transpose of a row vector: $\left(
\begin{array}
[c]{cccc}%
4 & -1 & 2 & 0
\end{array}
\right)  ^{T}$. It is furthermore common to write row vectors as tuples (i.e.,
put commas between their entries instead of leaving empty space); thus, the
row vector $\left(
\begin{array}
[c]{cccc}%
4 & -1 & 2 & 0
\end{array}
\right)  $ becomes $\left(  4,-1,2,0\right)  $ (which takes up less space),
and our column vector above becomes $\left(  4,-1,2,0\right)  ^{T}$.

The transpose of a matrix $A$ is also denoted by $A^{t}$ or $^{T}A$ or $^{t}A$
by various authors (not me).

Here is a very simple fact about transposes: The transpose of the transpose of
a matrix $A$ is the matrix $A$ itself. In other words:

\begin{proposition}
\label{prop.transpose.invol}Let $n\in\mathbb{N}$ and $m\in\mathbb{N}$. Let $A$
be an $n\times m$-matrix. Then, $\left(  A^{T}\right)  ^{T}=A$.
\end{proposition}

\begin{proof}
[Proof of Proposition \ref{prop.transpose.invol}.]This is fairly clear, but
let me give a formal proof just to get you used to the notations.

We have $A^{T}=\left(  A_{j,i}\right)  _{1\leq i\leq m,\ 1\leq j\leq n}$ (by
the definition of $A^{T}$). Thus, $A^{T}$ is an $m\times n$-matrix and
satisfies%
\begin{equation}
\left(  A^{T}\right)  _{i,j}=A_{j,i}\ \ \ \ \ \ \ \ \ \ \text{for all }%
i\in\left\{  1,2,\ldots,m\right\}  \text{ and }j\in\left\{  1,2,\ldots
,n\right\}  . \label{pf.prop.transpose.invol.AT}%
\end{equation}
Hence,%
\begin{equation}
\left(  A^{T}\right)  _{j,i}=A_{i,j}\ \ \ \ \ \ \ \ \ \ \text{for all }%
i\in\left\{  1,2,\ldots,n\right\}  \text{ and }j\in\left\{  1,2,\ldots
,m\right\}  . \label{pf.prop.transpose.invol.AT2}%
\end{equation}
(Indeed, this follows by applying (\ref{pf.prop.transpose.invol.AT}) to $j$
and $i$ instead of $i$ and $j$.)

Now, the definition of $\left(  A^{T}\right)  ^{T}$ yields%
\[
\left(  A^{T}\right)  ^{T}=\left(  \underbrace{\left(  A^{T}\right)  _{j,i}%
}_{\substack{=A_{i,j}\\\text{(by (\ref{pf.prop.transpose.invol.AT2}))}%
}}\right)  _{1\leq i\leq n,\ 1\leq j\leq m}=\left(  A_{i,j}\right)  _{1\leq
i\leq n,\ 1\leq j\leq m}=A
\]
(by (\ref{eq.matrix.A-through-Aij})). This proves Proposition
\ref{prop.transpose.invol}.
\end{proof}

\subsection{Addition, scaling and multiplication}

Matrices can (sometimes) be added, (always) be scaled and (sometimes) be
multiplied. Let me explain:

\begin{definition}
\label{def.matrix-add}Let $A$ and $B$ be two matrices of the same dimensions
(that is, they have the same number of rows, and the same number of columns).
Then, $A+B$ denotes the matrix obtained by adding each entry of $A$ to the
corresponding entry of $B$. Or, to write it more formally: If $A$ and $B$ are
two $n\times m$-matrices, then%
\[
A+B=\left(  A_{i,j}+B_{i,j}\right)  _{1\leq i\leq n,\ 1\leq j\leq m}.
\]

\end{definition}

For example, $\left(
\begin{array}
[c]{ccc}%
a & b & c\\
d & e & f
\end{array}
\right)  +\left(
\begin{array}
[c]{ccc}%
a^{\prime} & b^{\prime} & c^{\prime}\\
d^{\prime} & e^{\prime} & f^{\prime}%
\end{array}
\right)  =\left(
\begin{array}
[c]{ccc}%
a+a^{\prime} & b+b^{\prime} & c+c^{\prime}\\
d+d^{\prime} & e+e^{\prime} & f+f^{\prime}%
\end{array}
\right)  $. (I am increasingly using variables instead of actual numbers in my
examples, because they make it easier to see what entry is going where.) On
the other hand, the two matrices $\left(
\begin{array}
[c]{c}%
a\\
b
\end{array}
\right)  $ and $\left(
\begin{array}
[c]{cc}%
c & d
\end{array}
\right)  $ cannot be added (since they have different dimensions\footnote{The
dimensions of the former matrix are $2$ and $1$, whereas the dimensions of the
latter matrix are $1$ and $2$. Even though they are equal \textbf{up to
order}, they do not count as equal.}).

Definition \ref{def.matrix-add} is often laconically summarized as follows:
\textquotedblleft Matrices are added entry by entry\textquotedblright\ (or
\textquotedblleft entrywise\textquotedblright). This simply means that each
entry of the sum $A+B$ is the sum of the corresponding entries of $A$ and $B$;
nothing fancy is going on.

So we now know how to add two matrices.

\begin{definition}
\label{def.matrix-scale}Let $A$ be a matrix, and $\lambda$ be a number. Then,
$\lambda A$ (or $\lambda\cdot A$) denotes the matrix obtained by multiplying
each entry of $A$ by $\lambda$. In other words: If $A$ is an $n\times
m$-matrix, then%
\[
\lambda A=\left(  \lambda A_{i,j}\right)  _{1\leq i\leq n,\ 1\leq j\leq m}.
\]
The matrix $\lambda A$ is often called \textquotedblleft$\lambda$ times
$A$\textquotedblright. The procedure of transforming $A$ into $\lambda A$ is
called \textit{scaling the matrix }$A$ \textit{by }$\lambda$. (Sometimes we
say \textquotedblleft multiplying\textquotedblright\ instead of
\textquotedblleft scaling\textquotedblright, but \textquotedblleft
scaling\textquotedblright\ is more precise.)

We write $-A$ for $\left(  -1\right)  A$.
\end{definition}

For example, $\lambda\left(
\begin{array}
[c]{ccc}%
a & b & c\\
d & e & f
\end{array}
\right)  =\left(
\begin{array}
[c]{ccc}%
\lambda a & \lambda b & \lambda c\\
\lambda d & \lambda e & \lambda f
\end{array}
\right)  $.

So now we know how to scale a matrix. (\textquotedblleft To
scale\textquotedblright\ means to multiply by a number.) Definition
\ref{def.matrix-scale} is summarized as follows: \textquotedblleft Matrices
are scaled entry by entry\textquotedblright.

\textquotedblleft Scaling\textquotedblright\ is often called \textquotedblleft
scalar multiplication\textquotedblright\ (but this is confusing terminology,
since \textquotedblleft scalar product\textquotedblright\ means something
completely different). If $A$ is a matrix, then a \textit{scalar multiple} of
$A$ is defined as a matrix of the form $\lambda A$ for some number $\lambda$.

With scaling and addition defined, we obtain subtraction for free:

\begin{definition}
\label{def.matrix-diff}Let $A$ and $B$ be two matrices of the same dimensions.
Then, $A-B$ denotes the matrix $A+\left(  -B\right)  =A+\left(  -1\right)  B$.
\end{definition}

For example, $\left(
\begin{array}
[c]{ccc}%
a & b & c\\
d & e & f
\end{array}
\right)  -\left(
\begin{array}
[c]{ccc}%
a^{\prime} & b^{\prime} & c^{\prime}\\
d^{\prime} & e^{\prime} & f^{\prime}%
\end{array}
\right)  =\left(
\begin{array}
[c]{ccc}%
a-a^{\prime} & b-b^{\prime} & c-c^{\prime}\\
d-d^{\prime} & e-e^{\prime} & f-f^{\prime}%
\end{array}
\right)  $.

Now, to the more interesting part: multiplying matrices. This is \textbf{not}
done by multiplying corresponding entries! (Why not? Well, it wouldn't make
for a particularly useful notion.) Instead, the definition goes as follows:

\begin{definition}
\label{def.AB}Let $n\in\mathbb{N}$, $m\in\mathbb{N}$ and $p\in\mathbb{N}$. Let
$A$ be an $n\times m$-matrix. Let $B$ be an $m\times p$-matrix. (Thus, $A$ has
to have $m$ columns, while $B$ has to have $m$ rows; other than this, the two
matrices do not need to have any relation to each other.) The product $AB$ of
these two matrices is defined as follows:%
\[
AB=\left(  \underbrace{A_{i,1}B_{1,j}+A_{i,2}B_{2,j}+\cdots+A_{i,m}B_{m,j}%
}_{\substack{\text{This is the sum of the }m\text{ terms of the form}%
\\A_{i,k}B_{k,j}\text{, for }k\text{ ranging over }\left\{  1,2,\ldots
,m\right\}  }}\right)  _{1\leq i\leq n,\ 1\leq j\leq p}.
\]
This is an $n\times p$-matrix.
\end{definition}

This definition is somewhat overwhelming, so let me rewrite it in words and
give some examples:

It says that the product $AB$ is well-defined whenever $A$ has as many columns
as $B$ has rows. In this case, $AB$ is the $n\times p$-matrix whose $\left(
i,j\right)  $-th entry is obtained by adding together:

\begin{itemize}
\item the product $A_{i,1}B_{1,j}$ of the $\left(  i,1\right)  $-th entry of
$A$ with the $\left(  1,j\right)  $-th entry of $B$;

\item the product $A_{i,2}B_{2,j}$ of the $\left(  i,2\right)  $-th entry of
$A$ with the $\left(  2,j\right)  $-th entry of $B$;

\item and so on;

\item the product $A_{i,m}B_{m,j}$ of the $\left(  i,m\right)  $-th entry of
$A$ with the $\left(  m,j\right)  $-th entry of $B$.
\end{itemize}

In other words, $AB$ is the matrix whose $\left(  i,j\right)  $-th entry is
obtained by multiplying each entry of the $i$-th row of $A$ with the
corresponding entry of the $j$-th column of $B$, and then adding together all
these products. The word \textquotedblleft corresponding\textquotedblright%
\ means that the $1$-st entry of the $i$-th row of $A$ gets multiplied with
the $1$-st entry of the $j$-th column of $B$, the $2$-nd entry with the $2$-nd
entry, etc.. In particular, for this to make sense, the $i$-th row of $A$ and
the $j$-th column of $B$ have to have the same number of entries. This is why
we required that $A$ has as many columns as $B$ has rows!

I promised examples. Here are four:%
\begin{align*}
\left(
\begin{array}
[c]{cc}%
a & b\\
a^{\prime} & b^{\prime}%
\end{array}
\right)  \left(
\begin{array}
[c]{cc}%
x & x^{\prime}\\
y & y^{\prime}%
\end{array}
\right)   &  =\left(
\begin{array}
[c]{cc}%
ax+by & ax^{\prime}+by^{\prime}\\
a^{\prime}x+b^{\prime}y & a^{\prime}x^{\prime}+b^{\prime}y^{\prime}%
\end{array}
\right)  ;\\
\left(
\begin{array}
[c]{ccc}%
a & b & c\\
a^{\prime} & b^{\prime} & c^{\prime}%
\end{array}
\right)  \left(
\begin{array}
[c]{c}%
x\\
y\\
z
\end{array}
\right)   &  =\left(
\begin{array}
[c]{c}%
ax+by+cz\\
a^{\prime}x+b^{\prime}y+c^{\prime}z
\end{array}
\right)  ;\\
\left(
\begin{array}
[c]{cc}%
a & b
\end{array}
\right)  \left(
\begin{array}
[c]{c}%
x\\
y
\end{array}
\right)   &  =\left(
\begin{array}
[c]{c}%
ax+by
\end{array}
\right)  ;\\
\left(
\begin{array}
[c]{c}%
a\\
b
\end{array}
\right)  \left(
\begin{array}
[c]{cc}%
x & y
\end{array}
\right)   &  =\left(
\begin{array}
[c]{cc}%
ax & ay\\
bx & by
\end{array}
\right)
\end{align*}
(note how in the fourth example, we don't see any plus signs, because each of
the sums has only one addend).

We can also denote the product $AB$ by $A\cdot B$ (though few people ever do
this\footnote{\textbf{Warning:} The notation $A\cdot B$ is somewhat
nonstandard. Many authors (for example, Olver and Shakiban in \cite[\S 3.1]%
{OlvSha06}) define the \textquotedblleft dot product\textquotedblright\ of two
column vectors $v=\left(  v_{1},v_{2},\ldots,v_{n}\right)  ^{T}$ and
$w=\left(  w_{1},w_{2},\ldots,w_{n}\right)  ^{T}$ (of the same size) to be the
number $v_{1}w_{1}+v_{2}w_{2}+\cdots+v_{n}w_{n}$; they furthermore denote this
dot product by $v\cdot w$. This notation is in conflict with our notation
$A\cdot B$, because the dot product of $v$ and $w$ is not what we call $v\cdot
w$ (it is, in fact, what we call $v^{T}\cdot w$). The reason why I have picked
the somewhat nonstandard convention to regard $A\cdot B$ as a synonym for $AB$
is my belief that a dot should always denote the same multiplication as
juxtaposition (i.e., that $A\cdot B$ should always mean the same as $AB$).}).

We have thus learnt how to multiply matrices. Notice that the $\left(
i,j\right)  $-th entry of the product $AB$ depends only on the $i$-th row of
$A$ and the $j$-th column of $B$. Why did we pick this strange definition,
rather than something simpler, like multiplying entry by entry, or at least
row by row? Well, \textquotedblleft entry by entry\textquotedblright\ is too
simple (you will see later what matrix multiplication is good for;
\textquotedblleft entry by entry\textquotedblright\ is useless in comparison),
whereas \textquotedblleft row by row\textquotedblright\ would be lacking many
of the nice properties that we will see later (e.g., our matrix multiplication
satisfies the associativity law $\left(  AB\right)  C=A\left(  BC\right)  $,
while \textquotedblleft row by row\textquotedblright\ does not).

\begin{exercise}
\label{exe.AB-defined}Let $A=\left(
\begin{array}
[c]{cc}%
1 & -1\\
2 & 0\\
3 & 5
\end{array}
\right)  $ and $B=\left(
\begin{array}
[c]{cc}%
1 & 2\\
1 & 6
\end{array}
\right)  $.

\textbf{(a)} The matrix $A$ is of size $3\times2$. What is the size of $B$ ?

\textbf{(b)} Is $AB$ defined? If it is, compute it.

\textbf{(c)} Is $BA$ defined? If it is, compute it.
\end{exercise}

\begin{noncompile}
Exercise \ref{exe.AB-defined} is essentially from Quiz 1 in Lewis's.
\end{noncompile}

\begin{exercise}
\label{exe.some-prods}\textbf{(a)} Compute%
\[
\left(
\begin{array}
[c]{ccc}%
1 & 1 & 1\\
0 & 1 & 1\\
0 & 0 & 1
\end{array}
\right)  \left(
\begin{array}
[c]{ccc}%
1 & 1 & 1\\
0 & 1 & 1\\
0 & 0 & 1
\end{array}
\right)
\]
and%
\[
\left(
\begin{array}
[c]{cccc}%
1 & 1 & 1 & 1\\
0 & 1 & 1 & 1\\
0 & 0 & 1 & 1\\
0 & 0 & 0 & 1
\end{array}
\right)  \left(
\begin{array}
[c]{cccc}%
1 & 1 & 1 & 1\\
0 & 1 & 1 & 1\\
0 & 0 & 1 & 1\\
0 & 0 & 0 & 1
\end{array}
\right)  .
\]


\textbf{(b)} Compute $\left(
\begin{array}
[c]{ccc}%
a & b & c\\
a^{\prime} & b^{\prime} & c^{\prime}\\
a^{\prime\prime} & b^{\prime\prime} & c^{\prime\prime}%
\end{array}
\right)  \left(
\begin{array}
[c]{ccc}%
1 & 1 & 1\\
1 & 1 & 1\\
1 & 1 & 1
\end{array}
\right)  $ for an arbitrary $3\times3$-matrix $\left(
\begin{array}
[c]{ccc}%
a & b & c\\
a^{\prime} & b^{\prime} & c^{\prime}\\
a^{\prime\prime} & b^{\prime\prime} & c^{\prime\prime}%
\end{array}
\right)  $.

\textbf{(c)} Compute $\left(
\begin{array}
[c]{cccc}%
1 & 1 & 1 & 1\\
0 & 1 & 1 & 1\\
0 & 0 & 1 & 1\\
0 & 0 & 0 & 1
\end{array}
\right)  \left(
\begin{array}
[c]{c}%
a\\
b\\
c\\
d
\end{array}
\right)  $ for an arbitrary $4\times1$-matrix $\left(
\begin{array}
[c]{c}%
a\\
b\\
c\\
d
\end{array}
\right)  $.
\end{exercise}

\begin{exercise}
\label{exe.some-products.mt1p}\textbf{(a)} Let $A_{3}=\left(
\begin{array}
[c]{ccc}%
0 & 1 & 0\\
1 & 0 & 1\\
0 & 1 & 0
\end{array}
\right)  $ and $B_{3}=\left(
\begin{array}
[c]{ccc}%
1 & 0 & 1\\
0 & 1 & 0\\
1 & 0 & 1
\end{array}
\right)  $. Compute $A_{3}^{2}$, $B_{3}^{2}$, $A_{3}B_{3}$ and $B_{3}A_{3}$.

\textbf{(b)} Let $A_{4}=\left(
\begin{array}
[c]{cccc}%
0 & 1 & 0 & 1\\
1 & 0 & 1 & 0\\
0 & 1 & 0 & 1\\
1 & 0 & 1 & 0
\end{array}
\right)  $ and $B_{4}=\left(
\begin{array}
[c]{cccc}%
1 & 0 & 1 & 0\\
0 & 1 & 0 & 1\\
1 & 0 & 1 & 0\\
0 & 1 & 0 & 1
\end{array}
\right)  $. Compute $A_{4}^{2}$, $B_{4}^{2}$, $A_{4}B_{4}$ and $B_{4}A_{4}$.

\textbf{(c)} For any $n\in\mathbb{N}$, define two \textquotedblleft
checkerboard-pattern\textquotedblright\ $n\times n$-matrices $A_{n}$ and
$B_{n}$ by%
\[
A_{n}=\left(  \left(  i+j\right)  \%2\right)  _{1\leq i\leq n,\ 1\leq j\leq
n},\ \ \ \ \ \ \ \ \ \ B_{n}=\left(  \left(  i+j-1\right)  \%2\right)  _{1\leq
i\leq n,\ 1\leq j\leq n},
\]
where $k\%2$ denotes the remainder left when $k$ is divided by $2$ (so $k\%2=%
\begin{cases}
1, & \text{if }k\text{ is odd;}\\
0, & \text{if }k\text{ is even}%
\end{cases}
$). (The matrices $A_{3}$ and $B_{3}$ in part \textbf{(a)} of this problem, as
well as the matrices $A_{4}$ and $B_{4}$ in its part \textbf{(b)}, are
particular cases of this construction.) Prove that each \textbf{even}
$n\in\mathbb{N}$ satisfies $A_{n}^{2}=B_{n}^{2}$ and $A_{n}B_{n}=B_{n}A_{n}$.
Prove that each \textbf{odd} $n\geq3$ satisfies $A_{n}B_{n}\neq B_{n}A_{n}$.
\end{exercise}

\subsection{The matrix product rewritten}

Let me show another way to restate our above definition of a product of two
matrices. First, one more notation:

\begin{definition}
Let $A$ be an $n\times m$-matrix.

\textbf{(a)} If $i\in\left\{  1,2,\ldots,n\right\}  $, then
$\operatorname*{row}\nolimits_{i}A$ will denote the $i$-th row of $A$. This is
a row vector of size $m$ (that is, a $1\times m$-matrix), and is formally
defined as
\[
\left(  A_{i,y}\right)  _{1\leq x\leq1,\ 1\leq y\leq m}=\left(
\begin{array}
[c]{cccc}%
A_{i,1} & A_{i,2} & \cdots & A_{i,m}%
\end{array}
\right)
\]
(notice how $i$ is kept fixed but $y$ is ranging from $1$ to $m$ here).

\textbf{(b)} If $j\in\left\{  1,2,\ldots,m\right\}  $, then
$\operatorname*{col}\nolimits_{j}A$ will denote the $j$-th column of $A$. This
is a column vector of size $n$ (that is, an $n\times1$-matrix), and is
formally defined as
\[
\left(  A_{x,j}\right)  _{1\leq x\leq n,\ 1\leq y\leq1}=\left(
\begin{array}
[c]{c}%
A_{1,j}\\
A_{2,j}\\
\vdots\\
A_{n,j}%
\end{array}
\right)  .
\]

\end{definition}

\begin{example}
If $A=\left(
\begin{array}
[c]{ccc}%
a & b & c\\
d & e & f
\end{array}
\right)  $, then $\operatorname*{row}\nolimits_{2}A=\left(
\begin{array}
[c]{ccc}%
d & e & f
\end{array}
\right)  $ and $\operatorname*{col}\nolimits_{2}A=\left(
\begin{array}
[c]{c}%
b\\
e
\end{array}
\right)  $.
\end{example}

Now, we observe that if $R$ is a row vector of some size $m$, and if $C$ is a
column vector of size $m$, then $RC$ is a $1\times1$-matrix. More precisely:
The product of a row vector $\left(
\begin{array}
[c]{cccc}%
r_{1} & r_{2} & \cdots & r_{m}%
\end{array}
\right)  $ and a column vector $\left(
\begin{array}
[c]{c}%
c_{1}\\
c_{2}\\
\vdots\\
c_{m}%
\end{array}
\right)  $ is given by
\begin{equation}
\left(
\begin{array}
[c]{cccc}%
r_{1} & r_{2} & \cdots & r_{m}%
\end{array}
\right)  \left(
\begin{array}
[c]{c}%
c_{1}\\
c_{2}\\
\vdots\\
c_{m}%
\end{array}
\right)  =\left(
\begin{array}
[c]{c}%
r_{1}c_{1}+r_{2}c_{2}+\cdots+r_{m}c_{m}%
\end{array}
\right)  . \label{eq.matrix-prod.RC}%
\end{equation}
We shall often equate a $1\times1$-matrix with its (unique) entry; so the
equality (\ref{eq.matrix-prod.RC}) rewrites as%
\begin{equation}
\left(
\begin{array}
[c]{cccc}%
r_{1} & r_{2} & \cdots & r_{m}%
\end{array}
\right)  \left(
\begin{array}
[c]{c}%
c_{1}\\
c_{2}\\
\vdots\\
c_{m}%
\end{array}
\right)  =r_{1}c_{1}+r_{2}c_{2}+\cdots+r_{m}c_{m}. \label{eq.matrix-prod.RC.1}%
\end{equation}


Now I will show a little collection of formulas for the product of two
matrices. They are all pretty straightforward to obtain (essentially, they are
the definition of the product viewed from different angles), but they are
helpful when it comes to manipulating products:

\begin{proposition}
\label{prop.matrix-prod.rc}Let $n\in\mathbb{N}$, $m\in\mathbb{N}$ and
$p\in\mathbb{N}$. Let $A$ be an $n\times m$-matrix. Let $B$ be an $m\times p$-matrix.

\textbf{(a)} For every $i\in\left\{  1,2,\ldots,n\right\}  $ and $j\in\left\{
1,2,\ldots,p\right\}  $, we have%
\[
\left(  AB\right)  _{i,j}=A_{i,1}B_{1,j}+A_{i,2}B_{2,j}+\cdots+A_{i,m}%
B_{m,j}.
\]


\textbf{(b)} For every $i\in\left\{  1,2,\ldots,n\right\}  $ and $j\in\left\{
1,2,\ldots,p\right\}  $, the $\left(  i,j\right)  $-th entry of $AB$ equals
the product of the $i$-th row of $A$ and the $j$-th column of $B$. In
formulas:%
\begin{equation}
\left(  AB\right)  _{i,j}=\operatorname*{row}\nolimits_{i}A\cdot
\operatorname*{col}\nolimits_{j}B \label{eq.prop.matrix-prod.rc.1}%
\end{equation}
for every $i\in\left\{  1,2,\ldots,n\right\}  $ and $j\in\left\{
1,2,\ldots,p\right\}  $ (where the expression $\operatorname*{row}%
\nolimits_{i}A\cdot\operatorname*{col}\nolimits_{j}B$ should be read as
$\left(  \operatorname*{row}\nolimits_{i}A\right)  \cdot\left(
\operatorname*{col}\nolimits_{j}B\right)  $). Thus,
\begin{align*}
AB  &  =\left(  \operatorname*{row}\nolimits_{i}A\cdot\operatorname*{col}%
\nolimits_{j}B\right)  _{1\leq i\leq n,\ 1\leq j\leq p}\\
&  =\left(
\begin{array}
[c]{cccc}%
\operatorname*{row}\nolimits_{1}A\cdot\operatorname*{col}\nolimits_{1}B &
\operatorname*{row}\nolimits_{1}A\cdot\operatorname*{col}\nolimits_{2}B &
\cdots & \operatorname*{row}\nolimits_{1}A\cdot\operatorname*{col}%
\nolimits_{p}B\\
\operatorname*{row}\nolimits_{2}A\cdot\operatorname*{col}\nolimits_{1}B &
\operatorname*{row}\nolimits_{2}A\cdot\operatorname*{col}\nolimits_{2}B &
\cdots & \operatorname*{row}\nolimits_{2}A\cdot\operatorname*{col}%
\nolimits_{p}B\\
\vdots & \vdots & \ddots & \vdots\\
\operatorname*{row}\nolimits_{n}A\cdot\operatorname*{col}\nolimits_{1}B &
\operatorname*{row}\nolimits_{n}A\cdot\operatorname*{col}\nolimits_{2}B &
\cdots & \operatorname*{row}\nolimits_{n}A\cdot\operatorname*{col}%
\nolimits_{p}B
\end{array}
\right)  .
\end{align*}


\textbf{(c)} For every $i\in\left\{  1,2,\ldots,n\right\}  $, we have%
\[
\operatorname*{row}\nolimits_{i}\left(  AB\right)  =\left(
\operatorname*{row}\nolimits_{i}A\right)  \cdot B.
\]


\textbf{(d)} For every $j\in\left\{  1,2,\ldots,p\right\}  $, we have%
\[
\operatorname*{col}\nolimits_{j}\left(  AB\right)  =A\cdot\operatorname*{col}%
\nolimits_{j}B.
\]

\end{proposition}

Proposition \ref{prop.matrix-prod.rc} \textbf{(c)} says that if $A$ and $B$
are two matrices (for which $AB$ makes sense), then each row of $AB$ equals
the corresponding row of $A$ multiplied by $B$. Similarly, Proposition
\ref{prop.matrix-prod.rc} \textbf{(d)} says that each column of $AB$ equals
$A$ multiplied by the corresponding column of $B$. These are fairly simple
observations, but they are surprisingly useful.

\begin{proof}
[Proof of Proposition \ref{prop.matrix-prod.rc}.]\textbf{(a)} By the
definition of $AB$, we have%
\[
AB=\left(  A_{i,1}B_{1,j}+A_{i,2}B_{2,j}+\cdots+A_{i,m}B_{m,j}\right)  _{1\leq
i\leq n,\ 1\leq j\leq p}.
\]
In other words,%
\begin{equation}
\left(  AB\right)  _{i,j}=A_{i,1}B_{1,j}+A_{i,2}B_{2,j}+\cdots+A_{i,m}B_{m,j}
\label{pf.prop.matrix-prod.rc.1}%
\end{equation}
for every $i\in\left\{  1,2,\ldots,n\right\}  $ and $j\in\left\{
1,2,\ldots,p\right\}  $. This proves Proposition \ref{prop.matrix-prod.rc}
\textbf{(a)}.

\textbf{(b)} Now, let us prove Proposition \ref{prop.matrix-prod.rc}
\textbf{(b)}. It is clearly enough to prove (\ref{eq.prop.matrix-prod.rc.1})
(because all the other statements of Proposition \ref{prop.matrix-prod.rc}
\textbf{(b)} are just restatements of (\ref{eq.prop.matrix-prod.rc.1})). So
let's do this. Let $i\in\left\{  1,2,\ldots,n\right\}  $ and $j\in\left\{
1,2,\ldots,p\right\}  $. Then,%
\begin{align}
\operatorname*{row}\nolimits_{i}A  &  =\left(
\begin{array}
[c]{cccc}%
A_{i,1} & A_{i,2} & \cdots & A_{i,m}%
\end{array}
\right)  \ \ \ \ \ \ \ \ \ \ \text{and}\label{pf.prop.matrix-prod.rc.row}\\
\operatorname*{col}\nolimits_{j}B  &  =\left(
\begin{array}
[c]{c}%
B_{1,j}\\
B_{2,j}\\
\vdots\\
B_{m,j}%
\end{array}
\right)  . \label{pf.prop.matrix-prod.rc.col}%
\end{align}
Hence,%
\begin{align*}
\operatorname*{row}\nolimits_{i}A\cdot\operatorname*{col}\nolimits_{j}B  &
=\left(
\begin{array}
[c]{cccc}%
A_{i,1} & A_{i,2} & \cdots & A_{i,m}%
\end{array}
\right)  \left(
\begin{array}
[c]{c}%
B_{1,j}\\
B_{2,j}\\
\vdots\\
B_{m,j}%
\end{array}
\right) \\
&  =A_{i,1}B_{1,j}+A_{i,2}B_{2,j}+\cdots+A_{i,m}B_{m,j}.
\end{align*}
Comparing this with (\ref{pf.prop.matrix-prod.rc.1}), we obtain $\left(
AB\right)  _{i,j}=\operatorname*{row}\nolimits_{i}A\cdot\operatorname*{col}%
\nolimits_{j}B$. Thus, we have proven (\ref{eq.prop.matrix-prod.rc.1}). Hence,
Proposition \ref{prop.matrix-prod.rc} \textbf{(b)} is proven.

\textbf{(c)} Let $i\in\left\{  1,2,\ldots,n\right\}  $. Set
$C=\operatorname*{row}\nolimits_{i}A$. Notice that $C$ is a row vector of size
$m$, thus a $1\times m$-matrix. We can refer to any given entry of $C$ either
as \textquotedblleft the $j$-th entry\textquotedblright\ or as
\textquotedblleft the $\left(  1,j\right)  $-th entry\textquotedblright%
\ (where $j$ is the number of the column the entry is located in).

We have%
\[
C=\operatorname*{row}\nolimits_{i}A=\left(
\begin{array}
[c]{cccc}%
A_{i,1} & A_{i,2} & \cdots & A_{i,m}%
\end{array}
\right)  .
\]
Thus,%
\begin{equation}
C_{1,k}=A_{i,k}\ \ \ \ \ \ \ \ \ \ \text{for every }k\in\left\{
1,2,\ldots,m\right\}  . \label{pf.prop.matrix-prod.rc.c.1}%
\end{equation}


Let $j\in\left\{  1,2,\ldots,p\right\}  $. Then,%
\begin{align*}
&  \left(  \text{the }j\text{-th entry of }\operatorname*{row}\nolimits_{i}%
\left(  AB\right)  \right) \\
&  =\left(  \text{the }\left(  i,j\right)  \text{-th entry of }AB\right)
=\left(  AB\right)  _{i,j}\\
&  =A_{i,1}B_{1,j}+A_{i,2}B_{2,j}+\cdots+A_{i,m}B_{m,j}%
\ \ \ \ \ \ \ \ \ \ \left(  \text{by (\ref{pf.prop.matrix-prod.rc.1})}\right)
.
\end{align*}
Comparing this with%
\begin{align*}
&  \left(  \text{the }j\text{-th entry of }CB\right) \\
&  =\left(  \text{the }\left(  1,j\right)  \text{-th entry of }CB\right)
\ \ \ \ \ \ \ \ \ \ \left(  \text{since }CB\text{ is a row vector}\right) \\
&  =\underbrace{C_{1,1}}_{\substack{=A_{i,1}\\\text{(by
(\ref{pf.prop.matrix-prod.rc.c.1}))}}}B_{1,j}+\underbrace{C_{1,2}%
}_{\substack{=A_{i,2}\\\text{(by (\ref{pf.prop.matrix-prod.rc.c.1}))}}%
}B_{2,j}+\cdots+\underbrace{C_{1,m}}_{\substack{=A_{i,m}\\\text{(by
(\ref{pf.prop.matrix-prod.rc.c.1}))}}}B_{m,j}\\
&  \ \ \ \ \ \ \ \ \ \ \left(
\begin{array}
[c]{c}%
\text{by Proposition \ref{prop.matrix-prod.rc} \textbf{(a)}, applied to
}1\text{, }C\text{ and }1\\
\text{instead of }n\text{, }A\text{ and }i
\end{array}
\right) \\
&  =A_{i,1}B_{1,j}+A_{i,2}B_{2,j}+\cdots+A_{i,m}B_{m,j},
\end{align*}
we obtain%
\begin{equation}
\left(  \text{the }j\text{-th entry of }\operatorname*{row}\nolimits_{i}%
\left(  AB\right)  \right)  =\left(  \text{the }j\text{-th entry of
}CB\right)  . \label{pf.prop.matrix-prod.rc.c.5}%
\end{equation}


Now, forget that we fixed $j$. We thus have shown that
(\ref{pf.prop.matrix-prod.rc.c.5}) holds for each $j\in\left\{  1,2,\ldots
,p\right\}  $. In other words, each entry of the row vector
$\operatorname*{row}\nolimits_{i}\left(  AB\right)  $ equals the corresponding
entry of the row vector $CB$. Hence, $\operatorname*{row}\nolimits_{i}\left(
AB\right)  $ equals $CB$. Thus, $\operatorname*{row}\nolimits_{i}\left(
AB\right)  =\underbrace{C}_{=\operatorname*{row}\nolimits_{i}A}B=\left(
\operatorname*{row}\nolimits_{i}A\right)  \cdot B$. This proves Proposition
\ref{prop.matrix-prod.rc} \textbf{(c)}.

\textbf{(d)} The proof of Proposition \ref{prop.matrix-prod.rc} \textbf{(d)}
is similar to that of Proposition \ref{prop.matrix-prod.rc} \textbf{(c)}. Let
me nevertheless show it, for the sake of completeness. (The proof below is
essentially a copy-pasted version of the above proof of Proposition
\ref{prop.matrix-prod.rc} \textbf{(c)}, with only the necessary changes made.
This is both practical for me, as it saves me some work, and hopefully helpful
for you, as it highlights the similarities.)

Let $j\in\left\{  1,2,\ldots,p\right\}  $. Set $D=\operatorname*{col}%
\nolimits_{j}B$. Notice that $D$ is a column vector of size $m$, thus an
$m\times1$-matrix. We can refer to any given entry of $D$ either as
\textquotedblleft the $i$-th entry\textquotedblright\ or as \textquotedblleft
the $\left(  i,1\right)  $-th entry\textquotedblright\ (where $i$ is the
number of the row the entry is located in).

We have%
\[
D=\operatorname*{col}\nolimits_{j}B=\left(
\begin{array}
[c]{c}%
B_{1,j}\\
B_{2,j}\\
\vdots\\
B_{m,j}%
\end{array}
\right)  .
\]
Thus,%
\begin{equation}
D_{k,1}=B_{k,j}\ \ \ \ \ \ \ \ \ \ \text{for every }k\in\left\{
1,2,\ldots,m\right\}  . \label{pf.prop.matrix-prod.rc.d.1}%
\end{equation}


Let $i\in\left\{  1,2,\ldots,n\right\}  $. Then,%
\begin{align*}
&  \left(  \text{the }i\text{-th entry of }\operatorname*{col}\nolimits_{j}%
\left(  AB\right)  \right) \\
&  =\left(  \text{the }\left(  i,j\right)  \text{-th entry of }AB\right)
=\left(  AB\right)  _{i,j}\\
&  =A_{i,1}B_{1,j}+A_{i,2}B_{2,j}+\cdots+A_{i,m}B_{m,j}%
\ \ \ \ \ \ \ \ \ \ \left(  \text{by (\ref{pf.prop.matrix-prod.rc.1})}\right)
.
\end{align*}
Comparing this with%
\begin{align*}
&  \left(  \text{the }i\text{-th entry of }AD\right) \\
&  =\left(  \text{the }\left(  i,1\right)  \text{-th entry of }AD\right)
\ \ \ \ \ \ \ \ \ \ \left(  \text{since }AD\text{ is a column vector}\right)
\\
&  =A_{i,1}\underbrace{D_{1,1}}_{\substack{=B_{1,j}\\\text{(by
(\ref{pf.prop.matrix-prod.rc.d.1}))}}}+A_{i,2}\underbrace{D_{2,1}%
}_{\substack{=B_{2,j}\\\text{(by (\ref{pf.prop.matrix-prod.rc.d.1}))}}%
}+\cdots+A_{i,m}\underbrace{D_{m,1}}_{\substack{=B_{m,j}\\\text{(by
(\ref{pf.prop.matrix-prod.rc.d.1}))}}}\\
&  \ \ \ \ \ \ \ \ \ \ \left(
\begin{array}
[c]{c}%
\text{by Proposition \ref{prop.matrix-prod.rc} \textbf{(a)}, applied to
}1\text{, }D\text{ and }1\\
\text{instead of }p\text{, }B\text{ and }j
\end{array}
\right) \\
&  =A_{i,1}B_{1,j}+A_{i,2}B_{2,j}+\cdots+A_{i,m}B_{m,j},
\end{align*}
we obtain%
\begin{equation}
\left(  \text{the }i\text{-th entry of }\operatorname*{col}\nolimits_{j}%
\left(  AB\right)  \right)  =\left(  \text{the }i\text{-th entry of
}AD\right)  . \label{pf.prop.matrix-prod.rc.d.5}%
\end{equation}


Now, forget that we fixed $i$. We thus have shown that
(\ref{pf.prop.matrix-prod.rc.d.5}) holds for each $i\in\left\{  1,2,\ldots
,n\right\}  $. In other words, each entry of the column vector
$\operatorname*{col}\nolimits_{j}\left(  AB\right)  $ equals the corresponding
entry of the column vector $AD$. Hence, $\operatorname*{col}\nolimits_{j}%
\left(  AB\right)  $ equals $AD$. Thus, $\operatorname*{col}\nolimits_{j}%
\left(  AB\right)  =A\underbrace{D}_{=\operatorname*{col}\nolimits_{j}%
B}=A\cdot\operatorname*{col}\nolimits_{j}B$. This proves Proposition
\ref{prop.matrix-prod.rc} \textbf{(d)}.
\end{proof}

\subsection{Properties of matrix operations}

The operations of adding, scaling and multiplying matrices, in many aspects,
\textquotedblleft behave almost as nicely as numbers\textquotedblright.
Specifically, I mean that they satisfy a bunch of laws that numbers satisfy:

\begin{proposition}
\label{prop.matrix-laws.1}Let $n\in\mathbb{N}$ and $m\in\mathbb{N}$. Then:

\textbf{(a)} We have $A+B=B+A$ for any two $n\times m$-matrices $A$ and $B$.
(This is called \textquotedblleft commutativity of addition\textquotedblright.)

\textbf{(b)} We have $A+\left(  B+C\right)  =\left(  A+B\right)  +C$ for any
three $n\times m$-matrices $A$, $B$ and $C$. (This is called \textquotedblleft
associativity of addition\textquotedblright.)

\textbf{(c}$_{1}$\textbf{)} We have $\lambda\left(  A+B\right)  =\lambda
A+\lambda B$ for any number $\lambda$ and any two $n\times m$-matrices $A$ and
$B$.

\textbf{(c}$_{2}$\textbf{)} We have $\lambda\left(  \mu A\right)  =\left(
\lambda\mu\right)  A$ and $\left(  \lambda+\mu\right)  A=\lambda A+\mu A$ for
any numbers $\lambda$ and $\mu$ and any $n\times m$-matrix $A$.

\textbf{(c}$_{3}$\textbf{)} We have $1A=A$ for any $n\times m$-matrix $A$.

Let furthermore $p\in\mathbb{N}$. Then:

\textbf{(d)} We have $A\left(  B+C\right)  =AB+AC$ for any $n\times m$-matrix
$A$ and any two $m\times p$-matrices $B$ and $C$. (This is called
\textquotedblleft left distributivity\textquotedblright.)

\textbf{(e)} We have $\left(  A+B\right)  C=AC+BC$ for any two $n\times
m$-matrices $A$ and $B$ and any $m\times p$-matrix $C$. (This is called
\textquotedblleft right distributivity\textquotedblright.)

\textbf{(f)} We have $\lambda\left(  AB\right)  =\left(  \lambda A\right)
B=A\left(  \lambda B\right)  $ for any number $\lambda$, any $n\times
m$-matrix $A$ and any $m\times p$-matrix $B$.

Finally, let $q\in\mathbb{N}$. Then:

\textbf{(g)} We have $A\left(  BC\right)  =\left(  AB\right)  C$ for any
$n\times m$-matrix $A$, any $m\times p$-matrix $B$ and any $p\times q$-matrix
$C$. (This is called \textquotedblleft associativity of
multiplication\textquotedblright.)
\end{proposition}

\begin{example}
Most parts of Proposition \ref{prop.matrix-laws.1} are fairly easy to
visualize and to prove. Let me give an example for the least obvious one: part
\textbf{(g)}.

Part \textbf{(g)} essentially says that $A\left(  BC\right)  =\left(
AB\right)  C$ holds for any three matrices $A$, $B$ and $C$ for which the
products $AB$ and $BC$ are well-defined (i.e., $A$ has as many columns as $B$
has rows, and $B$ has as many columns as $C$ has rows). For example, take
$n=1$, $m=3$, $p=2$ and $q=3$. Set%
\[
A=\left(
\begin{array}
[c]{ccc}%
a & b & c
\end{array}
\right)  ,\ \ \ \ \ \ \ \ \ \ B=\left(
\begin{array}
[c]{cc}%
d & d^{\prime}\\
e & e^{\prime}\\
f & f^{\prime}%
\end{array}
\right)  ,\ \ \ \ \ \ \ \ \ \ C=\left(
\begin{array}
[c]{ccc}%
x & y & z\\
x^{\prime} & y^{\prime} & z^{\prime}%
\end{array}
\right)  .
\]
Then,%
\[
AB=\left(
\begin{array}
[c]{cc}%
ad+be+cf & ad^{\prime}+be^{\prime}+cf^{\prime}%
\end{array}
\right)
\]
and thus%
\begin{align*}
\left(  AB\right)  C  &  =\left(
\begin{array}
[c]{cc}%
ad+be+cf & ad^{\prime}+be^{\prime}+cf^{\prime}%
\end{array}
\right)  \left(
\begin{array}
[c]{ccc}%
x & y & z\\
x^{\prime} & y^{\prime} & z^{\prime}%
\end{array}
\right) \\
&  =\left(
\begin{array}
[c]{c}%
ad^{\prime}x^{\prime}+be^{\prime}x^{\prime}+cf^{\prime}x^{\prime
}+bex+adx+cfx\\
ad^{\prime}y^{\prime}+be^{\prime}y^{\prime}+cf^{\prime}y^{\prime
}+bey+ady+cfy\\
ad^{\prime}z^{\prime}+be^{\prime}z^{\prime}+cf^{\prime}z^{\prime}+bez+adz+cfz
\end{array}
\right)  ^{T}%
\end{align*}
after some computation. (Here, we have written the result as a transpose of a
column vector, because if we had written it as a row vector, it would not fit
on this page.) But
\[
BC=\left(
\begin{array}
[c]{cc}%
d & d^{\prime}\\
e & e^{\prime}\\
f & f^{\prime}%
\end{array}
\right)  \left(
\begin{array}
[c]{ccc}%
x & y & z\\
x^{\prime} & y^{\prime} & z^{\prime}%
\end{array}
\right)  =\left(
\begin{array}
[c]{ccc}%
d^{\prime}x^{\prime}+dx & d^{\prime}y^{\prime}+dy & d^{\prime}z^{\prime}+dz\\
e^{\prime}x^{\prime}+ex & e^{\prime}y^{\prime}+ey & e^{\prime}z^{\prime}+ez\\
f^{\prime}x^{\prime}+fx & f^{\prime}y^{\prime}+fy & f^{\prime}z^{\prime}+fz
\end{array}
\right)
\]
and as before
\[
A\left(  BC\right)  =\left(
\begin{array}
[c]{c}%
ad^{\prime}x^{\prime}+be^{\prime}x^{\prime}+cf^{\prime}x^{\prime
}+bex+adx+cfx\\
ad^{\prime}y^{\prime}+be^{\prime}y^{\prime}+cf^{\prime}y^{\prime
}+bey+ady+cfy\\
ad^{\prime}z^{\prime}+be^{\prime}z^{\prime}+cf^{\prime}z^{\prime}+bez+adz+cfz
\end{array}
\right)  ^{T}.
\]
Hence, $\left(  AB\right)  C=A\left(  BC\right)  $. Thus, our example confirms
Proposition \ref{prop.matrix-laws.1} \textbf{(g)}.
\end{example}

The laws of Proposition \ref{prop.matrix-laws.1} allow you to do many formal
manipulations with matrices similarly to how you are used to work with
numbers. For example, if you have $n$ matrices $A_{1},A_{2},\ldots,A_{n}$ such
that successive matrices can be multiplied (i.e., for each $i\in\left\{
1,2,\ldots,n-1\right\}  $, the matrix $A_{i}$ has as many columns as $A_{i+1}$
has rows), then the product $A_{1}A_{2}\cdots A_{n}$ is well-defined: you can
parenthesize it in any order, and the result will always be the same. For
example, the product $ABCD$ of four matrices $A,B,C,D$ can be computed in any
of the five ways%
\[
\left(  \left(  AB\right)  C\right)  D,\ \ \ \ \ \ \ \ \ \ \left(  AB\right)
\left(  CD\right)  ,\ \ \ \ \ \ \ \ \ \ \left(  A\left(  BC\right)  \right)
D,\ \ \ \ \ \ \ \ \ \ A\left(  \left(  BC\right)  D\right)
,\ \ \ \ \ \ \ \ \ \ A\left(  B\left(  CD\right)  \right)  ,
\]
and all of them lead to the same result. This is called \textit{general
associativity} and is not obvious (even if you know that Proposition
\ref{prop.matrix-laws.1} \textbf{(g)} holds)\footnote{If you are curious about
the proofs:
\par
We shall prove Proposition \ref{prop.matrix-laws.1} \textbf{(g)} further below
(in Section \ref{sect.intro.sum}). General associativity can be derived from
Proposition \ref{prop.matrix-laws.1} \textbf{(g)} in the general context of
\textquotedblleft binary operations\textquotedblright; see (for
example)\ \cite{Zuker14} for this argument.}. Let me state this result again
as a proposition, just to stress its importance:

\begin{proposition}
\label{prop.matrix-laws.ga-mult}Let $A_{1},A_{2},\ldots,A_{n}$ be $n$
matrices. Assume that, for each $i\in\left\{  1,2,\ldots,n-1\right\}  $, the
number of columns of $A_{i}$ equals the number of rows of $A_{i+1}$ (so that
the product $A_{i}A_{i+1}$ makes sense). Then, the product $A_{1}A_{2}\cdots
A_{n}$ is well-defined: Any way to compute this product (by parenthesizing it)
yields the same result. In particular, it can be computed both as
$A_{1}\left(  A_{2}\left(  A_{3}\left(  \cdots\left(  A_{n-1}A_{n}\right)
\right)  \right)  \right)  $ and as $\left(  \left(  \left(  \left(
A_{1}A_{2}\right)  A_{3}\right)  \cdots\right)  A_{n-1}\right)  A_{n}$.
\end{proposition}

Please take a moment to appreciate general associativity! Without it, we could
not make sense of products like $ABC$ and $ABCDE$, because their values could
depend on how we choose to compute them. This is one reason why, in the
definition of $AB$, we multiply entries of the $i$-th row of $A$ with entries
of the $j$-th column of $B$. Using rows both times would break
associativity!\footnote{Of course, our formulation of general associativity
was far from rigorous. After all, we have not defined what a \textquotedblleft
way to compute a product\textquotedblright\ means, or what \textquotedblleft
parenthesizing a product\textquotedblright\ means. There are several ways to
make Proposition \ref{prop.matrix-laws.ga-mult} rigorous. See
\cite{m.se709196} for a discussion of such ways. (Note that the simplest way
actually avoids defining \textquotedblleft parenthesizing\textquotedblright.
Instead, it defines the product $A_{1}A_{2}\cdots A_{n}$ by recursion on $n$,
namely defining it to be $A_{1}$ when $n=1$, and defining it to be $\left(
A_{1}A_{2}\cdots A_{n-1}\right)  A_{n}$ otherwise (where we are using the
already-defined product $A_{1}A_{2}\cdots A_{n-1}$). Informally speaking, this
means that the product $A_{1}A_{2}\cdots A_{n}$ is defined as $\left(  \left(
\left(  \left(  A_{1}A_{2}\right)  A_{3}\right)  \cdots\right)  A_{n-1}%
\right)  A_{n}$. Now, general associativity says that this product $A_{1}%
A_{2}\cdots A_{n}$ equals $\left(  A_{1}A_{2}\cdots A_{k}\right)  \left(
A_{k+1}A_{k+2}\cdots A_{n}\right)  $ for each $k\in\left\{  1,2,\ldots
,n-1\right\}  $. (This is not too hard to prove by induction over $n$.)
Informally speaking, this shows that our product $A_{1}A_{2}\cdots A_{n}$ also
equals the result of any way of computing it (not only the $\left(  \left(
\left(  \left(  A_{1}A_{2}\right)  A_{3}\right)  \cdots\right)  A_{n-1}%
\right)  A_{n}$ way).)}

There is also a general associativity law for addition:

\begin{proposition}
\label{prop.matrix-laws.ga-add}Let $A_{1},A_{2},\ldots,A_{n}$ be $n$ matrices
of the same size. Then, the sum $A_{1}+A_{2}+\cdots+A_{n}$ is well-defined:
Any way to compute this sum (by parenthesizing it) yields the same result. In
particular, it can be computed both as $A_{1}+\left(  A_{2}+\left(
A_{3}+\left(  \cdots+\left(  A_{n-1}+A_{n}\right)  \right)  \right)  \right)
$ and as $\left(  \left(  \left(  \left(  A_{1}+A_{2}\right)  +A_{3}\right)
+\cdots\right)  +A_{n-1}\right)  +A_{n}$.
\end{proposition}

There is also another variant of general associativity that concerns the
interplay of matrix multiplication and scaling. It claims that products of
matrices and numbers can be parenthesized in any order. For example, the
product $\lambda\mu AB$ of two numbers $\lambda$ and $\mu$ and two matrices
$A$ and $B$ can be computed in any of the five ways%
\[
\left(  \left(  \lambda\mu\right)  A\right)  B,\ \ \ \ \ \ \ \ \ \ \left(
\lambda\mu\right)  \left(  AB\right)  ,\ \ \ \ \ \ \ \ \ \ \left(
\lambda\left(  \mu A\right)  \right)  B,\ \ \ \ \ \ \ \ \ \ \lambda\left(
\left(  \mu A\right)  B\right)  ,\ \ \ \ \ \ \ \ \ \ \lambda\left(  \mu\left(
AB\right)  \right)  ,
\]
and all of them lead to the same result. This can be deduced from parts
\textbf{(c}$_{2}$\textbf{)}, \textbf{(f)} and \textbf{(g)} of Proposition
\ref{prop.matrix-laws.1}.

We shall give proofs of parts \textbf{(d)} and \textbf{(g)} of Proposition
\ref{prop.matrix-laws.1} in Section \ref{sect.intro.sum} below.

Various other identities follow from Proposition \ref{prop.matrix-laws.1}. For
example, if $A$, $B$ and $C$ are three matrices of the same size, then
$A-\left(  B+C\right)  =A-B-C$. For another example, if $A$ and $B$ are two
$n\times m$-matrices (for some $n\in\mathbb{N}$ and $m\in\mathbb{N}$) and if
$C$ is an $m\times p$-matrix (for some $p\in\mathbb{N}$), then $\left(
A-B\right)  C=AC-BC$. These identities are proven similarly as the analogous
properties of numbers are proven; we shall not linger on them.

\subsection{Non-properties of matrix operations}

Conspicuously absent from Proposition \ref{prop.matrix-laws.1} is one
important law that is well-known to hold for numbers: commutativity of
multiplication (that is, $ab=ba$). This has a reason: it is false for
matrices. There are at least three reasons why it is false:

\begin{enumerate}
\item If $A$ and $B$ are matrices, then it can happen that $AB$ is
well-defined (i.e., $A$ has as many columns as $B$ has rows) but $BA$ is not
(i.e., $B$ does not have as many columns as $A$ has rows). For example, if
$A=\left(
\begin{array}
[c]{c}%
a\\
b\\
c
\end{array}
\right)  $ and $B=\left(
\begin{array}
[c]{cc}%
x & y
\end{array}
\right)  $, then $AB$ is well-defined but $BA$ is not.

\item If $A$ and $B$ are matrices such that both $AB$ and $BA$ are
well-defined, then $AB$ and $BA$ might still have different dimensions.
Namely, if $A$ is an $n\times m$-matrix and $B$ is an $m\times n$-matrix, then
$AB$ is an $n\times n$-matrix, but $BA$ is an $m\times m$-matrix. So comparing
$AB$ and $BA$ makes no sense unless $n=m$.

\item Even if $AB$ and $BA$ are of the same dimensions, they can still be
distinct. For example, if $A=\left(
\begin{array}
[c]{cc}%
1 & 1\\
0 & 1
\end{array}
\right)  $ and $B=A^{T}=\left(
\begin{array}
[c]{cc}%
1 & 0\\
1 & 1
\end{array}
\right)  $, then $AB=\left(
\begin{array}
[c]{cc}%
2 & 1\\
1 & 1
\end{array}
\right)  $ whereas $BA=\left(
\begin{array}
[c]{cc}%
1 & 1\\
1 & 2
\end{array}
\right)  $.
\end{enumerate}

Two matrices $A$ and $B$ are said to \textit{commute} if $AB=BA$ (which, in
particular, means that both $AB$ and $BA$ are well-defined). You will
encounter many cases when matrices $A$ and $B$ happen to commute (for example,
every $n\times n$-matrix commutes with the $n\times n$ identity matrix; see
below for what this means); but in general there is no reason to expect two
randomly chosen matrices to commute.

As a consequence of matrices refusing to commute (in general), we cannot
reasonably define division of matrices. Actually, there are two reasons why we
cannot reasonably define division of matrices: First, if $A$ and $B$ are two
matrices, then it is not clear whether $\dfrac{A}{B}$ should mean a matrix $C$
satisfying $BC=A$, or a matrix $C$ satisfying $CB=A$. (The failure of
commutativity implies that these are two different things.) Second, in
general, neither of these matrices $C$ is necessarily unique; nor is it
guaranteed to exist. This is similar to the fact that we cannot divide by $0$
(in fact, $\dfrac{0}{0}$ would not be unique, while $\dfrac{1}{0}$ would not
exist); but with matrices, $0$ is not the only forbidden denominator. Here is
an example:

\begin{example}
\textbf{(a)} Let $A=\left(
\begin{array}
[c]{cc}%
1 & 0\\
0 & 0
\end{array}
\right)  $ and $B=A$. Then, $BC=A$ holds for $C=A$, but also for $C=\left(
\begin{array}
[c]{cc}%
1 & 0\\
0 & 1
\end{array}
\right)  $ (and also for many other matrices $C$). So the matrix $C$
satisfying $BC=A$ is not unique.

\textbf{(b)} Let $A=\left(
\begin{array}
[c]{cc}%
1 & 0\\
0 & 1
\end{array}
\right)  $ and $B=\left(
\begin{array}
[c]{cc}%
1 & 0\\
0 & 0
\end{array}
\right)  $. Then, there exists no matrix $C$ satisfying $BC=A$. Indeed, if
$C=\left(
\begin{array}
[c]{cc}%
x & y\\
z & w
\end{array}
\right)  $ is any matrix, then $BC=\left(
\begin{array}
[c]{cc}%
x & y\\
0 & 0
\end{array}
\right)  $ has its second row filled with zeroes, but $A$ does not; so $BC$
cannot equal $A$.
\end{example}

\begin{exercise}
\label{exe.commutativity-example}\textbf{(a)} Let $A=\left(
\begin{array}
[c]{cc}%
1 & 0\\
0 & 2
\end{array}
\right)  $. Show that the $2\times2$-matrices $B$ satisfying $AB=BA$ are
precisely the matrices of the form $\left(
\begin{array}
[c]{cc}%
a & 0\\
0 & d
\end{array}
\right)  $ (where $a$ and $d$ are any numbers). [\textbf{Hint:} Set $B=\left(
%
\begin{array}
[c]{cc}%
x & y\\
z & w
\end{array}
\right)  $, and rewrite $AB=BA$ as a system of linear equations in $x,y,z,w$.
Solve this system.]

\textbf{(b)} Let $A=\left(
\begin{array}
[c]{cc}%
1 & 1\\
0 & 1
\end{array}
\right)  $. Characterize the $2\times2$-matrices $B$ satisfying $AB=BA$.
\end{exercise}

\subsection{\label{sect.intro.sum}(*) The summation sign, and a proof of
$\left(  AB\right)  C=A\left(  BC\right)  $}

We now take a break from studying matrices to introduce an important symbol:
the summation sign ($\sum$). This sign is one of the hallmarks of abstract
mathematics (and also computer science), and helps manipulate matrices
comfortably. Here is a quick (but informal) definition of the summation sign
$\sum$:

\begin{definition}
\label{def.sum}Let $p$ and $q$ be two integers such that $p\leq q+1$. Let
$a_{p},a_{p+1},\ldots,a_{q}$ be some numbers. Then, $\sum_{k=p}^{q}a_{k}$
means the sum $a_{p}+a_{p+1}+\cdots+a_{q}$. The symbol $\sum$ is called the
summation sign; we pronounce the expression $\sum_{k=p}^{q}a_{k}$ as
\textquotedblleft sum of $a_{k}$ for all $k$ ranging from $p$ to
$q$\textquotedblright.
\end{definition}

This definition needs some clarifications; but before I give them, let me show
some examples:

\begin{itemize}
\item We have $\sum_{k=p}^{q}k=p+\left(  p+1\right)  +\cdots+q$. For example,
$\sum_{k=1}^{n}k=1+2+\cdots+n$. (A well-known formula says that this sum
$\sum_{k=1}^{n}k=1+2+\cdots+n$ equals $\dfrac{n\left(  n+1\right)  }{2}$. For
a concrete example, $\sum_{k=1}^{3}k=1+2+3=\dfrac{3\left(  3+1\right)  }{2}%
=6$.) For another example, $\sum_{k=-n}^{n}k=\left(  -n\right)  +\left(
-n+1\right)  +\cdots+n$. (This latter sum equals $0$, because it contains, for
each its addend, also its negative\footnote{except for the addend $0$, but
this $0$ doesn't change the sum anyway}.)

\item We have $\sum_{k=p}^{q}k^{2}=p^{2}+\left(  p+1\right)  ^{2}+\cdots
+q^{2}$. For example, $\sum_{k=1}^{n}k^{2}=1^{2}+2^{2}+\cdots+n^{2}$. (A
well-known formula says that this sum $\sum_{k=1}^{n}k^{2}=1^{2}+2^{2}%
+\cdots+n^{2}$ equals $\dfrac{n\left(  n+1\right)  \left(  2n+1\right)  }{6}$.)

\item We have $\sum_{k=p}^{q}1=\underbrace{1+1+\cdots+1}_{q-p+1\text{ times}%
}=q-p+1$. This illustrates the fact that the $a_{k}$ in a sum $\sum_{k=p}%
^{q}a_{k}$ needs not depend on $k$ (although the cases where it does not
depend on $k$ are fairly trivial).

\item If $p=q$, then $\sum_{k=p}^{q}a_{k}=a_{p}$. (A sum of only one number is
simply this number.)
\end{itemize}

As I have said, a few remarks and clarifications on the summation sign are in order:

\begin{definition}
\label{def.sum.2}\textbf{(a)} In the expression $\sum_{k=p}^{q}a_{k}$ (as
defined in Definition \ref{def.sum}), the letter $k$ is called the
\textit{summation index}. It stands for the \textquotedblleft moving
part\textquotedblright\ in the sum (e.g., the part in which the addends
differ). For example, $\sum_{k=p}^{q}\dfrac{1}{k+3}$ is the sum of the
fractions $\dfrac{1}{k+3}$ for $k$ ranging from $p$ to $q$; its addends all
have the form $\dfrac{1}{k+3}$, but for different values of $k$.

The summation index doesn't have to be called $k$; any letter is legitimate
(as long as it is not already used otherwise). For example, $\sum_{i=p}%
^{q}a_{i}$ and $\sum_{x=p}^{q}a_{x}$ are two synonymous ways to write
$\sum_{k=p}^{q}a_{k}$. Just make sure that you are using the same letter under
the $\sum$ sign and to its right (so you should not write $\sum_{i=p}^{q}%
a_{k}$, unless you mean the sum $\underbrace{a_{k}+a_{k}+\cdots+a_{k}%
}_{q-p+1\text{ times}}$).

\textbf{(b)} You might be wondering what Definition \ref{def.sum} means in the
case when $p=q+1$; after all, in this case, there are no numbers
$a_{p},a_{p+1},\ldots,a_{q}$, and the sum $a_{p}+a_{p+1}+\cdots+a_{q}$ has no
addends. (For example, how should $\sum_{k=2}^{1}k=2+3+\cdots+1$ be
understood?) However, there is a general convention in mathematics that a sum
with no addends is always defined to be $0$, and is called an \textit{empty
sum}. Thus, $\sum_{k=p}^{q}a_{k}=0$ whenever $p=q+1$. For example, $\sum
_{k=2}^{1}k=0$ and $\sum_{k=1}^{0}\dfrac{1}{k}=0$ (even though $\dfrac{1}{k}$
makes no sense for $k=0$). (This convention might sound arbitrary, but is
logically adequate: In fact, it ensures that the equality $a_{p}%
+a_{p+1}+\cdots+a_{q}=\left(  a_{p}+a_{p+1}+\cdots+a_{q-1}\right)  +a_{q}$
holds not only for $q>p$, but also for $q=p$.)

Many authors define the sum $\sum_{k=p}^{q}a_{k}$ to be $0$ in the case when
$p>q+1$ as well; thus, the sum $\sum_{k=p}^{q}a_{k}$ is defined for
\textbf{any} two integers $p$ and $q$ (without the requirement that $p\leq
q+1$). However, this convention is somewhat slippery: for instance, it entails
$\sum_{k=1}^{n}k=0$ for all negative $n$, and thus the equality $\sum
_{k=1}^{n}k=\dfrac{n\left(  n+1\right)  }{2}$ does \textbf{not} hold for
negative $n$.

\textbf{(c)} From a fully rigorous point of view, Definition \ref{def.sum} did
not define $\sum_{k=p}^{q}a_{k}$ at all. Indeed, it defined $\sum_{k=p}%
^{q}a_{k}$ to be $a_{p}+a_{p+1}+\cdots+a_{q}$, but what does $a_{p}%
+a_{p+1}+\cdots+a_{q}$ mean? The rigorous way to define $\sum_{k=p}^{q}a_{k}$
is as follows (by recursion):

\begin{itemize}
\item If $q=p-1$, then $\sum_{k=p}^{q}a_{k}$ is defined to be $0$.

\item If $q>p-1$, then $\sum_{k=p}^{q}a_{k}$ is defined to be $\left(
\sum_{k=p}^{q-1}a_{k}\right)  +a_{q}$.
\end{itemize}

This is a recursive definition (since it defines $\sum_{k=p}^{q}a_{k}$ in
terms of $\sum_{k=p}^{q-1}a_{k}$), and provides an algorithm to compute
$\sum_{k=p}^{q}a_{k}$. From a formal point of view, \textquotedblleft%
$a_{p}+a_{p+1}+\cdots+a_{q}$\textquotedblright\ is just a colloquial way to
say \textquotedblleft$\sum_{k=p}^{q}a_{k}$\textquotedblright.
\end{definition}

Notice that the expression \textquotedblleft$\sum_{k=p}^{q}a_{k}%
$\textquotedblright\ is both a more compact and a more rigorous way to say
\textquotedblleft$a_{p}+a_{p+1}+\cdots+a_{q}$\textquotedblright. A computer
would not understand the expression \textquotedblleft$a_{p}+a_{p+1}%
+\cdots+a_{q}$\textquotedblright\ (it could only guess what the
\textquotedblleft$\cdots$\textquotedblright\ means, and computers are bad at
guessing); but the expression \textquotedblleft$\sum_{k=p}^{q}a_{k}%
$\textquotedblright\ has a well-defined meaning that can be rigorously defined
and can be explained to a computer\footnote{See Definition \ref{def.sum.2}
\textbf{(c)} for the rigorous definition of $\sum_{k=p}^{q}a_{k}$.}. Thus, if
you want to tell a computer to compute a sum, the command you have to use will
be closer to \textquotedblleft$\sum_{k=p}^{q}a_{k}$\textquotedblright\ than to
\textquotedblleft$a_{p}+a_{p+1}+\cdots+a_{q}$\textquotedblright. For example,
in Python, you would have to write \textquotedblleft\texttt{sum(a[k] for k in
range(p, q+1))}\textquotedblright\ (where \textquotedblleft\texttt{a[k]}%
\textquotedblright\ is understood to return $a_{k}$)\ \ \ \ \footnote{Why
\textquotedblleft\texttt{q+1}\textquotedblright\ and not \textquotedblleft%
\texttt{q}\textquotedblright? Because Python defines \texttt{range(u, v)} as
the list $\left(  u,u+1,\ldots,v-1\right)  $ (that is, the list that starts at
$u$ and ends \textbf{just before} $v$). So \texttt{range(p, q)} would be
$\left(  p,p+1,\ldots,q-1\right)  $, but we want $\left(  p,p+1,\ldots
,q\right)  $.}.

Using the summation sign, we can rewrite the product $AB$ of two matrices $A$
and $B$ (see Definition \ref{def.AB}) more nicely:

\begin{proposition}
\label{prop.AB}Let $n\in\mathbb{N}$, $m\in\mathbb{N}$ and $p\in\mathbb{N}$.
Let $A$ be an $n\times m$-matrix. Let $B$ be an $m\times p$-matrix. Then,%
\[
\left(  AB\right)  _{i,j}=\sum_{k=1}^{m}A_{i,k}B_{k,j}%
\ \ \ \ \ \ \ \ \ \ \text{for all }i\in\left\{  1,2,\ldots,n\right\}  \text{
and }j\in\left\{  1,2,\ldots,p\right\}  .
\]

\end{proposition}

\begin{proof}
[Proof of Proposition \ref{prop.AB}.]For all $i\in\left\{  1,2,\ldots
,n\right\}  $ and $j\in\left\{  1,2,\ldots,p\right\}  $, we have%
\begin{align*}
\left(  AB\right)  _{i,j}  &  =A_{i,1}B_{1,j}+A_{i,2}B_{2,j}+\cdots
+A_{i,m}B_{m,j}\ \ \ \ \ \ \ \ \ \ \left(  \text{by Proposition
\ref{prop.matrix-prod.rc} \textbf{(a)}}\right) \\
&  =\sum_{k=1}^{m}A_{i,k}B_{k,j}%
\end{align*}
(because $\sum_{k=1}^{m}A_{i,k}B_{k,j}$ is exactly $A_{i,1}B_{1,j}%
+A_{i,2}B_{2,j}+\cdots+A_{i,m}B_{m,j}$, by its definition). Proposition
\ref{prop.AB} is proven.
\end{proof}

Here are two properties of sums that are fairly clear if you understand how
sums are defined:

\begin{proposition}
\label{prop.sum.dist}Let $p$ and $q$ be two integers such that $p\leq q+1$.
Let $a_{p},a_{p+1},\ldots,a_{q}$ be some numbers. Let $b$ be a number. Then,
\[
\sum_{k=p}^{q}ba_{k}=b\sum_{k=p}^{q}a_{k}.
\]
(The expression $\sum_{k=p}^{q}ba_{k}$ has to be read as $\sum_{k=p}%
^{q}\left(  ba_{k}\right)  $.)
\end{proposition}

\begin{proof}
[Proof of Proposition \ref{prop.sum.dist}.]By the definition of $\sum$, we
have%
\[
\sum_{k=p}^{q}ba_{k}=ba_{p}+ba_{p+1}+\cdots+ba_{q}=b\underbrace{\left(
a_{p}+a_{p+1}+\cdots+a_{q}\right)  }_{\substack{=\sum_{k=p}^{q}a_{k}%
\\\text{(by the definition of }\sum_{k=p}^{q}a_{k}\text{)}}}=b\sum_{k=p}%
^{q}a_{k}.
\]

\end{proof}

\begin{proposition}
\label{prop.sum.a+b}Let $p$ and $q$ be two integers such that $p\leq q+1$. Let
$a_{p},a_{p+1},\ldots,a_{q}$ be some numbers. Let $b_{p},b_{p+1},\ldots,b_{q}$
be some numbers. Then,
\[
\sum_{k=p}^{q}\left(  a_{k}+b_{k}\right)  =\sum_{k=p}^{q}a_{k}+\sum_{k=p}%
^{q}b_{k}.
\]
(The expression $\sum_{k=p}^{q}a_{k}+\sum_{k=p}^{q}b_{k}$ has to be read as
$\left(  \sum_{k=p}^{q}a_{k}\right)  +\left(  \sum_{k=p}^{q}b_{k}\right)  $.)
\end{proposition}

\begin{proof}
[Proof of Proposition \ref{prop.sum.a+b}.]By the definition of $\sum$, we have%
\begin{align*}
\sum_{k=p}^{q}\left(  a_{k}+b_{k}\right)   &  =\left(  a_{p}+b_{p}\right)
+\left(  a_{p+1}+b_{p+1}\right)  +\cdots+\left(  a_{q}+b_{q}\right) \\
&  =\underbrace{\left(  a_{p}+a_{p+1}+\cdots+a_{q}\right)  }_{\substack{=\sum
_{k=p}^{q}a_{k}\\\text{(by the definition of }\sum_{k=p}^{q}a_{k}\text{)}%
}}+\underbrace{\left(  b_{p}+b_{p+1}+\cdots+b_{q}\right)  }_{\substack{=\sum
_{k=p}^{q}b_{k}\\\text{(by the definition of }\sum_{k=p}^{q}b_{k}\text{)}}}\\
&  =\sum_{k=p}^{q}a_{k}+\sum_{k=p}^{q}b_{k}.
\end{align*}

\end{proof}

Our goal in this section is to prove Proposition \ref{prop.matrix-laws.1}
\textbf{(g)}, illustrating the use and manipulation of the $\sum$ sign.
However, as a warmup, let us first prove Proposition \ref{prop.matrix-laws.1}
\textbf{(d)} (which is simple enough that you can easily check it without
$\sum$ signs, but is nevertheless worth proving using the $\sum$ sign just to
demonstrate how to work with the $\sum$ sign):

\begin{proof}
[Proof of Proposition \ref{prop.matrix-laws.1} \textbf{(d)}.]Let $A$ be an
$n\times m$-matrix. Let $B$ and $C$ be two $m\times p$-matrices.

We shall show that $\left(  A\left(  B+C\right)  \right)  _{i,j}=\left(
AB+AC\right)  _{i,j}$ for all $i\in\left\{  1,2,\ldots,n\right\}  $ and
$j\in\left\{  1,2,\ldots,p\right\}  $. Once this is proven, this will entail
that corresponding entries of the two $n\times p$-matrices $A\left(
B+C\right)  $ and $AB+AC$ are equal; and thus, these two matrices have to be equal.

Proposition \ref{prop.AB} yields%
\begin{equation}
\left(  AB\right)  _{i,j}=\sum_{k=1}^{m}A_{i,k}B_{k,j}
\label{pf.prop.matrix-laws.1.d.1}%
\end{equation}
for all $i\in\left\{  1,2,\ldots,n\right\}  $ and $j\in\left\{  1,2,\ldots
,p\right\}  $.

Proposition \ref{prop.AB} (applied to $C$ instead of $B$) yields%
\begin{equation}
\left(  AC\right)  _{i,j}=\sum_{k=1}^{m}A_{i,k}C_{k,j}
\label{pf.prop.matrix-laws.1.d.2}%
\end{equation}
for all $i\in\left\{  1,2,\ldots,n\right\}  $ and $j\in\left\{  1,2,\ldots
,p\right\}  $.

Finally, Proposition \ref{prop.AB} (applied to $B+C$ instead of $B$) yields%
\begin{align*}
\left(  A\left(  B+C\right)  \right)  _{i,j}  &  =\sum_{k=1}^{m}%
A_{i,k}\underbrace{\left(  B+C\right)  _{k,j}}_{\substack{=B_{k,j}%
+C_{k,j}\\\text{(since matrices are}\\\text{added entry by entry)}}%
}=\sum_{k=1}^{m}\underbrace{A_{i,k}\left(  B_{k,j}+C_{k,j}\right)  }%
_{=A_{i,k}B_{k,j}+A_{i,k}C_{k,j}}\\
&  =\sum_{k=1}^{m}\left(  A_{i,k}B_{k,j}+A_{i,k}C_{k,j}\right)
=\underbrace{\sum_{k=1}^{m}A_{i,k}B_{k,j}}_{\substack{=\left(  AB\right)
_{i,j}\\\text{(by (\ref{pf.prop.matrix-laws.1.d.1}))}}}+\underbrace{\sum
_{k=1}^{m}A_{i,k}C_{k,j}}_{\substack{=\left(  AC\right)  _{i,j}\\\text{(by
(\ref{pf.prop.matrix-laws.1.d.2}))}}}\\
&  \ \ \ \ \ \ \ \ \ \ \left(
\begin{array}
[c]{c}%
\text{by Proposition \ref{prop.sum.a+b}, applied to }1\text{, }m\text{,}\\
A_{i,k}B_{k,j}\text{ and }A_{i,k}C_{k,j}\text{ instead of }p\text{, }q\text{,
}a_{k}\text{ and }b_{k}%
\end{array}
\right) \\
&  =\left(  AB\right)  _{i,j}+\left(  AC\right)  _{i,j}\\
&  =\left(  AB+AC\right)  _{i,j}\ \ \ \ \ \ \ \ \ \ \left(
\begin{array}
[c]{c}%
\text{again because matrices}\\
\text{are added entry by entry}%
\end{array}
\right)
\end{align*}
for all $i\in\left\{  1,2,\ldots,n\right\}  $ and $j\in\left\{  1,2,\ldots
,p\right\}  $. In other words, each entry of the $n\times p$-matrix $A\left(
B+C\right)  $ equals the corresponding entry of $AB+AC$. Thus, the matrix
$A\left(  B+C\right)  $ equals $AB+AC$. This proves Proposition
\ref{prop.matrix-laws.1} \textbf{(d)}.
\end{proof}

Before we can prove Proposition \ref{prop.matrix-laws.1} \textbf{(g)}, we need
another fact about sums:

\begin{proposition}
\label{prop.sum.ij}Let $m\in\mathbb{N}$ and $p\in\mathbb{N}$. Assume that a
number $a_{k,\ell}$ is given for every $k\in\left\{  1,2,\ldots,m\right\}  $
and $\ell\in\left\{  1,2,\ldots,p\right\}  $. Then,%
\[
\sum_{k=1}^{m}\sum_{\ell=1}^{p}a_{k,\ell}=\sum_{\ell=1}^{p}\sum_{k=1}%
^{m}a_{k,\ell}.
\]
(Note that an expression like $\sum_{k=1}^{m}\sum_{\ell=1}^{p}a_{k,\ell}$ has
to be understood as $\sum_{k=1}^{m}\left(  \sum_{\ell=1}^{p}a_{k,\ell}\right)
$. It is a \textquotedblleft nested sum\textquotedblright, i.e., a sum of
sums. For example,%
\begin{align*}
\sum_{k=1}^{3}\underbrace{\sum_{\ell=1}^{4}k\cdot\ell}_{=k\cdot1+k\cdot
2+k\cdot3+k\cdot4}  &  =\sum_{k=1}^{3}\left(  k\cdot1+k\cdot2+k\cdot
3+k\cdot4\right) \\
&  =\left(  1\cdot1+1\cdot2+1\cdot3+1\cdot4\right)  +\left(  2\cdot
1+2\cdot2+2\cdot3+2\cdot4\right) \\
&  \ \ \ \ \ \ \ \ \ \ +\left(  3\cdot1+3\cdot2+3\cdot3+3\cdot4\right)  .
\end{align*}
)
\end{proposition}

\begin{example}
For $m=2$ and $p=3$, Proposition \ref{prop.sum.ij} says that%
\[
\sum_{k=1}^{2}\sum_{\ell=1}^{3}a_{k,\ell}=\sum_{\ell=1}^{3}\sum_{k=1}%
^{2}a_{k,\ell}.
\]
In other words,%
\[
\left(  a_{1,1}+a_{1,2}+a_{1,3}\right)  +\left(  a_{2,1}+a_{2,2}%
+a_{2,3}\right)  =\left(  a_{1,1}+a_{2,1}\right)  +\left(  a_{1,2}%
+a_{2,2}\right)  +\left(  a_{1,3}+a_{2,3}\right)  .
\]

\end{example}

\begin{proof}
[Proof of Proposition \ref{prop.sum.ij}.]Comparing%
\begin{align*}
&  \sum_{k=1}^{m}\underbrace{\sum_{\ell=1}^{p}a_{k,\ell}}_{\substack{=a_{k,1}%
+a_{k,2}+\cdots+a_{k,p}\\\text{(by the definition of the }\sum\text{ sign)}%
}}\\
&  =\sum_{k=1}^{m}\left(  a_{k,1}+a_{k,2}+\cdots+a_{k,p}\right) \\
&  =\left(  a_{1,1}+a_{1,2}+\cdots+a_{1,p}\right)  +\left(  a_{2,1}%
+a_{2,2}+\cdots+a_{2,p}\right)  +\cdots+\left(  a_{m,1}+a_{m,2}+\cdots
+a_{m,p}\right) \\
&  \ \ \ \ \ \ \ \ \ \ \left(  \text{by the definition of the }\sum\text{
sign}\right) \\
&  =\left(  \text{the sum of all possible numbers }a_{k,\ell}\right)
\end{align*}
with%
\begin{align*}
&  \sum_{\ell=1}^{p}\underbrace{\sum_{k=1}^{m}a_{k,\ell}}%
_{\substack{=a_{1,\ell}+a_{2,\ell}+\cdots+a_{m,\ell}\\\text{(by the definition
of the }\sum\text{ sign)}}}\\
&  =\sum_{\ell=1}^{p}\left(  a_{1,\ell}+a_{2,\ell}+\cdots+a_{m,\ell}\right) \\
&  =\left(  a_{1,1}+a_{2,1}+\cdots+a_{m,1}\right)  +\left(  a_{1,2}%
+a_{2,2}+\cdots+a_{m,2}\right)  +\cdots+\left(  a_{1,p}+a_{2,p}+\cdots
+a_{m,p}\right) \\
&  \ \ \ \ \ \ \ \ \ \ \left(  \text{by the definition of the }\sum\text{
sign}\right) \\
&  =\left(  \text{the sum of all possible numbers }a_{k,\ell}\right)  ,
\end{align*}
we obtain $\sum_{k=1}^{m}\left(  \sum_{\ell=1}^{p}a_{k,\ell}\right)
=\sum_{\ell=1}^{p}\left(  \sum_{k=1}^{m}a_{k,\ell}\right)  $.

(A more rigorous proof could be given using induction; but I don't want to
move to that level of formalism in these notes. Notice the visual meaning of
the above proof: If we place the $mp$ numbers $a_{k,\ell}$ into a matrix
$\left(  a_{k,\ell}\right)  _{1\leq k\leq m,\ 1\leq\ell\leq p}$, then

\begin{itemize}
\item the number $\sum_{k=1}^{m}\sum_{\ell=1}^{p}a_{k,\ell}$ is obtained by
summing the entries in each row of the matrix, and then summing the resulting sums;

\item the number $\sum_{\ell=1}^{p}\sum_{k=1}^{m}a_{k,\ell}$ is obtained by
summing the entries in each column of the matrix, and then summing the
resulting sums.
\end{itemize}

Thus, clearly, both numbers are equal (namely, equal to the sum of all entries
of the matrix).)
\end{proof}

Now, we can prove Proposition \ref{prop.matrix-laws.1} \textbf{(g)}:

\begin{proof}
[Proof of Proposition \ref{prop.matrix-laws.1} \textbf{(g)}.]Let $A$ be an
$n\times m$-matrix. Let $B$ be an $m\times p$-matrix. Let $C$ be a $p\times q$-matrix.

We must show that $A\left(  BC\right)  =\left(  AB\right)  C$. In order to do
so, it suffices to show that $\left(  A\left(  BC\right)  \right)
_{i,j}=\left(  \left(  AB\right)  C\right)  _{i,j}$ for all $i\in\left\{
1,2,\ldots,n\right\}  $ and $j\in\left\{  1,2,\ldots,q\right\}  $ (because
this will show that respective entries of the two $n\times q$-matrices
$A\left(  BC\right)  $ and $\left(  AB\right)  C$ are equal, and thus the two
matrices are equal).

We know that $A$ is an $n\times m$-matrix, and that $BC$ is an $m\times
q$-matrix. Hence, we can apply Proposition \ref{prop.AB} to $n$, $m$, $q$, $A$
and $BC$ instead of $n$, $m$, $p$, $A$ and $B$. We thus obtain%
\begin{equation}
\left(  A\left(  BC\right)  \right)  _{i,j}=\sum_{k=1}^{m}A_{i,k}\left(
BC\right)  _{k,j} \label{pf.prop.matrix-laws.1.g.A(BC)}%
\end{equation}
for all $i\in\left\{  1,2,\ldots,n\right\}  $ and $j\in\left\{  1,2,\ldots
,q\right\}  $. Also, we can apply Proposition \ref{prop.AB} to $m$, $p$, $q$,
$B$ and $C$ instead of $n$, $m$, $p$, $A$ and $B$. We thus obtain%
\begin{align}
\left(  BC\right)  _{i,j}  &  =\sum_{k=1}^{p}B_{i,k}C_{k,j}=\sum_{\ell=1}%
^{p}B_{i,\ell}C_{\ell,j}\label{pf.prop.matrix-laws.1.g.BC}\\
&  \ \ \ \ \ \ \ \ \ \ \left(  \text{here, we renamed the summation index
}k\text{ as }\ell\right) \nonumber
\end{align}
for all $i\in\left\{  1,2,\ldots,m\right\}  $ and $j\in\left\{  1,2,\ldots
,q\right\}  $. Furthermore, we can apply Proposition \ref{prop.AB} to $n$,
$p$, $q$, $AB$ and $C$ instead of $n$, $m$, $p$, $A$ and $B$. We thus find%
\begin{align}
\left(  \left(  AB\right)  C\right)  _{i,j}  &  =\sum_{k=1}^{p}\left(
AB\right)  _{i,k}C_{k,j}=\sum_{\ell=1}^{p}\left(  AB\right)  _{i,\ell}%
C_{\ell,j}\label{pf.prop.matrix-laws.1.g.(AB)C}\\
&  \ \ \ \ \ \ \ \ \ \ \left(  \text{here, we renamed the summation index
}k\text{ as }\ell\right) \nonumber
\end{align}
for all $i\in\left\{  1,2,\ldots,n\right\}  $ and $j\in\left\{  1,2,\ldots
,q\right\}  $. Finally, Proposition \ref{prop.AB} (applied verbatim) yields%
\begin{equation}
\left(  AB\right)  _{i,j}=\sum_{k=1}^{m}A_{i,k}B_{k,j}
\label{pf.prop.matrix-laws.1.g.AB}%
\end{equation}
for all $i\in\left\{  1,2,\ldots,n\right\}  $ and $j\in\left\{  1,2,\ldots
,p\right\}  $.

Now that we have found formulas for the entries of all matrices involved, we
can perform our computation: For all $i\in\left\{  1,2,\ldots,n\right\}  $ and
$j\in\left\{  1,2,\ldots,q\right\}  $, we have%
\begin{align*}
\left(  A\left(  BC\right)  \right)  _{i,j}  &  =\sum_{k=1}^{m}A_{i,k}%
\underbrace{\left(  BC\right)  _{k,j}}_{\substack{=\sum_{\ell=1}^{p}B_{k,\ell
}C_{\ell,j}\\\text{(by (\ref{pf.prop.matrix-laws.1.g.BC}), applied to }k\text{
instead of }i\text{)}}}\ \ \ \ \ \ \ \ \ \ \left(  \text{by
(\ref{pf.prop.matrix-laws.1.g.A(BC)})}\right) \\
&  =\sum_{k=1}^{m}\underbrace{A_{i,k}\left(  \sum_{\ell=1}^{p}B_{k,\ell
}C_{\ell,j}\right)  }_{\substack{=\sum_{\ell=1}^{p}A_{i,k}B_{k,\ell}C_{\ell
,j}\\\text{(by an application of}\\\text{Proposition \ref{prop.sum.dist})}%
}}=\sum_{k=1}^{m}\sum_{\ell=1}^{p}A_{i,k}B_{k,\ell}C_{\ell,j}\\
&  =\sum_{\ell=1}^{p}\sum_{k=1}^{m}A_{i,k}B_{k,\ell}C_{\ell,j}%
\ \ \ \ \ \ \ \ \ \ \left(  \text{by Proposition \ref{prop.sum.ij}, applied to
}a_{k,\ell}=A_{i,k}B_{k,\ell}C_{\ell,j}\right)
\end{align*}
and%
\begin{align*}
\left(  \left(  AB\right)  C\right)  _{i,j}  &  =\sum_{\ell=1}^{p}%
\underbrace{\left(  AB\right)  _{i,\ell}}_{\substack{=\sum_{k=1}^{m}%
A_{i,k}B_{k,\ell}\\\text{(by (\ref{pf.prop.matrix-laws.1.g.AB}), applied to
}\ell\\\text{instead of }j\text{)}}}C_{\ell,j}\ \ \ \ \ \ \ \ \ \ \left(
\text{by (\ref{pf.prop.matrix-laws.1.g.(AB)C})}\right) \\
&  =\sum_{\ell=1}^{p}\underbrace{\left(  \sum_{k=1}^{m}A_{i,k}B_{k,\ell
}\right)  C_{\ell,j}}_{\substack{=C_{\ell,j}\sum_{k=1}^{m}A_{i,k}B_{k,\ell
}=\sum_{k=1}^{m}C_{\ell,j}A_{i,k}B_{k,\ell}\\\text{(by an application
of}\\\text{Proposition \ref{prop.sum.dist})}}}=\sum_{\ell=1}^{p}\sum_{k=1}%
^{m}\underbrace{C_{\ell,j}A_{i,k}B_{k,\ell}}_{=A_{i,k}B_{k,\ell}C_{\ell,j}}\\
&  =\sum_{\ell=1}^{p}\sum_{k=1}^{m}A_{i,k}B_{k,\ell}C_{\ell,j}.
\end{align*}
Comparing these two equalities, we obtain
\[
\left(  A\left(  BC\right)  \right)  _{i,j}=\left(  \left(  AB\right)
C\right)  _{i,j}%
\]
for all $i\in\left\{  1,2,\ldots,n\right\}  $ and $j\in\left\{  1,2,\ldots
,q\right\}  $. In other words, each entry of the matrix $A\left(  BC\right)  $
equals the corresponding entry of the matrix $\left(  AB\right)  C$. Thus, the
matrices $A\left(  BC\right)  $ and $\left(  AB\right)  C$ are equal. This
proves Proposition \ref{prop.matrix-laws.1} \textbf{(g)}.
\end{proof}

We have just proven the hardest part of Proposition \ref{prop.matrix-laws.1}.
The rest is fairly straightforward.

\subsection{The zero matrix}

\begin{definition}
Let $n\in\mathbb{N}$ and $m\in\mathbb{N}$. Then, the $n\times m$\textit{ zero
matrix} means the matrix $\left(  0\right)  _{1\leq i\leq n,\ 1\leq j\leq m}$.
This is the $n\times m$-matrix filled with zeroes. It is called $0_{n\times
m}$. (When no confusion with the number $0$ can arise, we will just call it
$0$.)
\end{definition}

For example, the $2\times3$ zero matrix is $\left(
\begin{array}
[c]{ccc}%
0 & 0 & 0\\
0 & 0 & 0
\end{array}
\right)  $.

Some authors (for example, Olver and Shakiban in \cite{OlvSha06}) denote the
zero matrix $0_{n\times m}$ by $\operatorname*{O}\nolimits_{n\times m}$ or
$O_{n\times m}$ (thus using the letter $O$ instead of the number $0$), or
simply by $\operatorname*{O}$.

The zero matrix behaves very much like the number $0$:

\begin{proposition}
\label{prop.matrix-laws.0}Let $n\in\mathbb{N}$ and $m\in\mathbb{N}$. Then:

\textbf{(a)} We have $0_{n\times m}+A=A+0_{n\times m}=A$ for each $n\times
m$-matrix $A$.

\textbf{(b)} We have $0_{n\times m}A=0_{n\times p}$ for each $p\in\mathbb{N}$
and each $m\times p$-matrix $A$.

\textbf{(c)} We have $A0_{n\times m}=0_{p\times m}$ for each $p\in\mathbb{N}$
and each $p\times n$-matrix $A$.

\textbf{(d)} We have $0A=0_{n\times m}$ for each $n\times m$-matrix $A$.

\textbf{(e)} We have $\lambda0_{n\times m}=0_{n\times m}$ for each number
$\lambda$.
\end{proposition}

\begin{remark}
Numbers are known to be zero-divisor-free: If a product $ab$ of two numbers
$a$ and $b$ is $0$, then one of $a$ and $b$ must be $0$. This fails for
matrices: If $A=\left(
\begin{array}
[c]{cc}%
1 & 0\\
0 & 0
\end{array}
\right)  $ and $B=\left(
\begin{array}
[c]{cc}%
0 & 0\\
0 & 1
\end{array}
\right)  $, then $AB=0_{2\times2}$ is the zero matrix, although neither $A$
nor $B$ is the zero matrix.
\end{remark}

\subsection{The identity matrix}

\begin{definition}
Let $n\in\mathbb{N}$. The \textit{diagonal entries} of an $n\times n$-matrix
$A$ are its entries $A_{1,1},A_{2,2},\ldots,A_{n,n}$. In other words, they are
the entries $A_{i,j}$ for $i=j$.
\end{definition}

For example, the diagonal entries of $\left(
\begin{array}
[c]{cc}%
a & b\\
c & d
\end{array}
\right)  $ are $a$ and $d$. The name \textquotedblleft diagonal
entries\textquotedblright\ comes from the visualization of an $n\times
n$-matrix as a square table: When we say \textquotedblleft
diagonal\textquotedblright, we always mean the diagonal of the square that
connects the upper-left corner with the lower-right corner\footnote{Often,
this diagonal is also called the \textquotedblleft main
diagonal\textquotedblright.}; the diagonal entries are simply the entries
along this diagonal. (The other diagonal is called the \textquotedblleft
antidiagonal\textquotedblright\ in linear algebra.)

\begin{definition}
If $i$ and $j$ are two objects (for example, numbers or sets or functions),
then we set%
\begin{equation}
\delta_{i,j}=%
\begin{cases}
1, & \text{if }i=j;\\
0, & \text{if }i\neq j
\end{cases}
. \label{eq.def.In.eq}%
\end{equation}

\end{definition}

For example, $\delta_{3,3}=1$ (since $3=3$) but $\delta_{1,2}=0$ (since
$1\neq2$). For another example, $\delta_{\left(  1,2\right)  ,\left(
1,3\right)  }=0$ (because $\left(  1,2\right)  \neq\left(  1,3\right)  $);
here we are using the notation $\delta_{i,j}$ in a situation where $i$ and $j$
are pairs of numbers.

The notation $\delta_{i,j}$ defined in (\ref{eq.def.In.eq}) is called the
\textit{Kronecker delta}; it is extremely simple and yet highly useful. It has
the property that $\delta_{i,j}=\delta_{j,i}$ for any $i$ and $j$ (because
$i=j$ holds if and only if $j=i$).

\begin{definition}
Let $n\in\mathbb{N}$. Then, the $n\times n$ \textit{identity matrix} means the
matrix $\left(  \delta_{i,j}\right)  _{1\leq i\leq n,\ 1\leq j\leq n}$. This
is the $n\times n$-matrix whose diagonal entries all equal $1$, and whose all
other entries equal $0$. It is denoted by $I_{n}$. (Other people call it $I$
or $E$ or $E_{n}$.)
\end{definition}

The $n\times n$ identity matrix $I_{n}$ looks as follows:%
\[
I_{n}=\left(
\begin{array}
[c]{ccccc}%
1 & 0 & 0 & \cdots & 0\\
0 & 1 & 0 & \cdots & 0\\
0 & 0 & 1 & \cdots & 0\\
\vdots & \vdots & \vdots & \ddots & \vdots\\
0 & 0 & 0 & \cdots & 1
\end{array}
\right)
\]
(with $n$ rows and $n$ columns). For example, the $3\times3$ identity matrix
is $I_{3}=\left(
\begin{array}
[c]{ccc}%
1 & 0 & 0\\
0 & 1 & 0\\
0 & 0 & 1
\end{array}
\right)  $.

The identity matrix behaves very much like the number $1$:

\begin{proposition}
\label{prop.matrix-laws.id}Let $n\in\mathbb{N}$ and $m\in\mathbb{N}$.

\textbf{(a)} We have $I_{n}A=A$ for each $n\times m$-matrix $A$.

\textbf{(b)} We have $AI_{m}=A$ for each $n\times m$-matrix $A$.
\end{proposition}

Proposition \ref{prop.matrix-laws.id} says that multiplying a matrix $A$ by an
identity matrix (from either side) does not change $A$. Thus, identity
matrices have no effect inside a product, and so can be \textquotedblleft
cancelled\textquotedblright\ (or, more precisely, dropped). For example, if
$A$, $B$, $C$ and $D$ are four $n\times n$-matrices, then $I_{n}ABI_{n}%
I_{n}CI_{n}D=ABCD$. (Of course, this is similar to dropping $1$'s from
products of numbers: $1ab\cdot1\cdot1c\cdot1d=abcd$.)

\subsection{(*) Proof of $AI_{n}=A$}

Let me give a proof of Proposition \ref{prop.matrix-laws.id} \textbf{(b)}, to
illustrate the following simple, yet important point about summations and the
Kronecker delta\footnote{This is something that often comes up in computations
(particularly in physics and computer science, where the use of the Kronecker
delta is widespread).}:

\begin{proposition}
\label{prop.sum.delta}Let $p$ and $q$ be two integers such that $p\leq q$. Let
$r\in\left\{  p,p+1,\ldots,q\right\}  $. Let $a_{p},a_{p+1},\ldots,a_{q}$ be
some numbers. Then,%
\[
\sum_{k=p}^{q}a_{k}\delta_{k,r}=a_{r}.
\]

\end{proposition}

\begin{example}
For $p=1$, $q=5$ and $r=4$, Proposition \ref{prop.sum.delta} says that
$\sum_{k=1}^{5}a_{k}\delta_{k,4}=a_{4}$. This is easy to check:%
\begin{align*}
\sum_{k=1}^{5}a_{k}\delta_{k,4}  &  =a_{1}\underbrace{\delta_{1,4}%
}_{\substack{=0\\\text{(since }1\neq4\text{)}}}+a_{2}\underbrace{\delta_{2,4}%
}_{\substack{=0\\\text{(since }2\neq4\text{)}}}+a_{3}\underbrace{\delta_{3,4}%
}_{\substack{=0\\\text{(since }3\neq4\text{)}}}+a_{4}\underbrace{\delta_{4,4}%
}_{\substack{=1\\\text{(since }4=4\text{)}}}+a_{5}\underbrace{\delta_{5,4}%
}_{\substack{=0\\\text{(since }5\neq4\text{)}}}\\
&  =a_{1}0+a_{2}0+a_{3}0+a_{4}1+a_{5}0=a_{4}1=a_{4}.
\end{align*}
What you should see on this example is that all but one addends of the sum
$\sum_{k=p}^{q}a_{k}\delta_{k,r}$ are zero, and the remaining one addend is
$a_{r}\underbrace{\delta_{r,r}}_{=1}=a_{r}1=a_{r}$. The proof below is just
writing this down in the general situation.
\end{example}

\begin{proof}
[Proof of Proposition \ref{prop.sum.delta}.]Let us first notice something
simple: For any $k\in\left\{  p,p+1,\ldots,q\right\}  $ such that $k\neq r$,
we have%
\[
a_{k}\underbrace{\delta_{k,r}}_{\substack{=0\\\text{(since }k\neq r\text{)}%
}}=a_{k}0=0.
\]
In other words, all terms of the form $a_{k}\delta_{k,r}$ with $k\neq r$ are
$0$. Hence, the sum of all these terms is $0$ as well. In other words,%
\begin{equation}
\left(  \text{the sum of all terms of the form }a_{k}\delta_{k,r}\text{ with
}k\neq r\right)  =0. \label{pf.prop.sum.delta.1}%
\end{equation}


By the definition of the $\sum$ sign, we have%
\begin{align*}
\sum_{k=p}^{q}a_{k}\delta_{k,r}  &  =a_{p}\delta_{p,r}+a_{p+1}\delta
_{p+1,r}+\cdots+a_{q}\delta_{q,r}\\
&  =\left(  \text{the sum of all terms of the form }a_{k}\delta_{k,r}\right)
\\
&  =a_{r}\underbrace{\delta_{r,r}}_{\substack{=1\\\text{(since }r=r\text{)}%
}}+\underbrace{\left(  \text{the sum of all terms of the form }a_{k}%
\delta_{k,r}\text{ with }k\neq r\right)  }_{\substack{=0\\\text{(by
(\ref{pf.prop.sum.delta.1}))}}}\\
&  \ \ \ \ \ \ \ \ \ \ \left(  \text{here, we have pulled out the addend
}a_{r}\delta_{r,r}\text{ out of the sum}\right) \\
&  =a_{r}1+0=a_{r}.
\end{align*}

\end{proof}

\begin{proof}
[Proof of Proposition \ref{prop.matrix-laws.id} \textbf{(b)}.]We have
$I_{m}=\left(  \delta_{i,j}\right)  _{1\leq i\leq m,\ 1\leq j\leq m}$ (this is
how we defined $I_{m}$), and thus%
\begin{equation}
\left(  I_{m}\right)  _{u,v}=\delta_{u,v}\ \ \ \ \ \ \ \ \ \ \text{for all
}u\in\left\{  1,2,\ldots,m\right\}  \text{ and }v\in\left\{  1,2,\ldots
,m\right\}  . \label{pf.prop.matrix-laws.id.b.Im}%
\end{equation}


Let $A$ be an $n\times m$-matrix. For every $i\in\left\{  1,2,\ldots
,n\right\}  $ and $j\in\left\{  1,2,\ldots,m\right\}  $, we have%
\begin{align*}
\left(  AI_{m}\right)  _{i,j}  &  =\sum_{k=1}^{m}A_{i,k}\underbrace{\left(
I_{m}\right)  _{k,j}}_{\substack{=\delta_{k,j}\\\text{(by
(\ref{pf.prop.matrix-laws.id.b.Im}))}}}\ \ \ \ \ \ \ \ \ \ \left(  \text{by
Proposition \ref{prop.AB}, applied to }p=m\text{ and }B=I_{m}\right) \\
&  =\sum_{k=1}^{m}A_{i,k}\delta_{k,j}=A_{i,j}%
\end{align*}
(by Proposition \ref{prop.sum.delta}, applied to $p=1$, $q=m$, $r=j$ and
$a_{k}=A_{i,k}$). In other words, each entry of the $n\times m$-matrix
$AI_{m}$ equals the corresponding entry of the $n\times m$-matrix $A$. In
other words, $AI_{m}$ equals $A$. This proves Proposition
\ref{prop.matrix-laws.id} \textbf{(b)}.
\end{proof}

Proposition \ref{prop.matrix-laws.id} \textbf{(a)} can be proven similarly
(but this time, instead of the sum $\sum_{k=1}^{m}A_{i,k}\delta_{k,j}$, we
must consider the sum $\sum_{k=1}^{n}\delta_{i,k}A_{k,j}=\sum_{k=1}^{n}%
A_{k,j}\delta_{i,k}=A_{i,j}$).

\subsection{Powers of a matrix}

The $k$-th power of a number $a$ (where $k\in\mathbb{N}$) is defined by
repeated multiplication: We start with $a^{0}=1$\ \ \ \ \footnote{Yes, this is
how $a^{0}$ is defined, for all $a$. Anyone who tells you that the number
$0^{0}$ is undefined is merely spreading their confusion.}, and we define each
next power of $a$ by multiplying the previous one by $a$. In formulas:
$a^{k+1}=a\cdot a^{k}$ for each $k\in\mathbb{N}$. Thus,%
\begin{align*}
a^{1}  &  =a\cdot\underbrace{a^{0}}_{=1}=a\cdot1=a;\\
a^{2}  &  =a\cdot\underbrace{a^{1}}_{=a}=a\cdot a;\\
a^{3}  &  =a\cdot\underbrace{a^{2}}_{=a\cdot a}=a\cdot a\cdot a,
\end{align*}
etc.. We can explicitly write
\[
a^{k}=\underbrace{a\cdot a\cdot\cdots\cdot a}_{k\text{ times }a}%
\ \ \ \ \ \ \ \ \ \ \text{for each }k\in\mathbb{N},
\]
where we understand $\underbrace{a\cdot a\cdot\cdots\cdot a}_{0\text{ times
}a}$ to mean $1$\ \ \ \ \footnote{This is a standard convention: An empty
product of numbers always means $1$.}.

We can play the same game with square matrices, but instead of the number $1$
we now take the $n\times n$ identity matrix $I_{n}$:

\begin{definition}
Let $n\in\mathbb{N}$. Let $A$ be an $n\times n$-matrix. Then, the $k$-th power
of the matrix $A$ (where $k\in\mathbb{N}$) is defined by repeated
multiplication: We start with $A^{0}=I_{n}$, and we define each next power of
$A$ by multiplying the previous one by $A$. In formulas: $A^{k+1}=A\cdot
A^{k}$ for each $k\in\mathbb{N}$. Explicitly, $A^{k}=\underbrace{A\cdot
A\cdot\cdots\cdot A}_{k\text{ times }A}$ for each $k\in\mathbb{N}$, where the
empty product of $n\times n$-matrices is defined to be $I_{n}$. (An
\textquotedblleft empty product\textquotedblright\ is a product with no
factors. Thus, $A^{0}=\underbrace{A\cdot A\cdot\cdots\cdot A}_{0\text{ times
}A}$ is an empty product.)
\end{definition}

Notice that we have been a bit sloppy when we said \textquotedblleft
multiplying the previous one by $A$\textquotedblright: When we multiply a
matrix $B$ by $A$, we might mean either $AB$ or $BA$, and as we know, these
two products can be different (matrices don't always commute!). However, in
the above definition, this makes no matter, because both definitions lead to
the same explicit formula $A^{k}=\underbrace{A\cdot A\cdot\cdots\cdot
A}_{k\text{ times }A}$ (which is well-defined because of general associativity).

We now have a first moderately interesting example of commuting matrices: Any
two powers of a square matrix commute. (In other words: For any $n\times
n$-matrix $A$, and any $u\in\mathbb{N}$ and $v\in\mathbb{N}$, the two matrices
$A^{u}$ and $A^{v}$ commute. This follows by observing that $A^{u}%
A^{v}=A^{u+v}=A^{v}A^{u}$.)

\subsection{\label{sect.intro.sum-matrices}(*) The summation sign for
matrices}

In Definition \ref{def.sum}, we have introduced a notation for sums of
arbitrary (finite) lists of numbers. The same notation can be used for sums of
arbitrary (finite) lists of matrices:

\begin{definition}
\label{def.sum.matrices}Let $n\in\mathbb{N}$ and $m\in\mathbb{N}$. Let $p$ and
$q$ be two integers such that $p\leq q+1$. Let $A_{p},A_{p+1},\ldots,A_{q}$ be
some $n\times m$-matrices. Then, $\sum_{k=p}^{q}A_{k}$ means the sum
$A_{p}+A_{p+1}+\cdots+A_{q}$. (This sum is well-defined due to Proposition
\ref{prop.matrix-laws.ga-add}. Moreover, the order of its addends does not
matter, as can be proven using Proposition \ref{prop.matrix-laws.1}
\textbf{(a)}.)

The notation $\sum_{k=p}^{q}A_{k}$ is analogous to the notation $\sum
_{k=p}^{q}a_{k}$ introduced in Definition \ref{def.sum}. The same remarks and
clarifications done for the latter notation in Definition \ref{def.sum.2}
apply to the former notation. There is only one difference: When the sum
$\sum_{k=p}^{q}A_{k}$ has no addends (i.e., when $p\geq q+1$), its value is
defined to be the zero matrix $0_{n\times m}$ rather than the number $0$.
(This is not much of a difference, seeing that the zero matrix $0_{n\times m}$
and the number $0$ behave similarly; see, e.g., Proposition
\ref{prop.matrix-laws.0}.)
\end{definition}

Recall that addition of matrices was defined entry by entry. Let me restate
this in terms of a single entry of a sum:

\begin{proposition}
\label{prop.sum.matrices.A+B}Let $n\in\mathbb{N}$ and $m\in\mathbb{N}$. Let
$A$ and $B$ be two $n\times m$-matrices. For every $i\in\left\{
1,2,\ldots,n\right\}  $ and $j\in\left\{  1,2,\ldots,m\right\}  $, we have%
\[
\left(  A+B\right)  _{i,j}=A_{i,j}+B_{i,j}.
\]

\end{proposition}

\begin{proof}
[Proof of Proposition \ref{prop.sum.matrices.A+B}.]We have $A+B=\left(
A_{i,j}+B_{i,j}\right)  _{1\leq i\leq n,\ 1\leq j\leq m}$ (by the definition
of $A+B$). Thus, $\left(  A+B\right)  _{i,j}=A_{i,j}+B_{i,j}$ for each
$i\in\left\{  1,2,\ldots,n\right\}  $ and $j\in\left\{  1,2,\ldots,m\right\}
$. This proves Proposition \ref{prop.sum.matrices.A+B}.
\end{proof}

Proposition \ref{prop.sum.matrices.A+B} concerns the sum of two matrices.
Using the summation sign, we can state an analogue of Proposition
\ref{prop.sum.matrices.A+B} for the sum of \textbf{several} matrices:

\begin{proposition}
\label{prop.sum.matrices.multi}Let $n\in\mathbb{N}$ and $m\in\mathbb{N}$. Let
$h\in\mathbb{N}$. Let $A_{1},A_{2},\ldots,A_{h}$ be some $n\times m$-matrices.
For every $i\in\left\{  1,2,\ldots,n\right\}  $ and $j\in\left\{
1,2,\ldots,m\right\}  $, we have%
\begin{equation}
\left(  \sum_{k=1}^{h}A_{k}\right)  _{i,j}=\sum_{k=1}^{h}\left(  A_{k}\right)
_{i,j}. \label{eq.prop.sum.matrices.multi.eq}%
\end{equation}

\end{proposition}

Notice that the $\sum$ sign on the left hand side of
(\ref{eq.prop.sum.matrices.multi.eq}) stands for a sum of several matrices,
while the $\sum$ sign on the right hand side stands for a sum of several numbers.

\begin{proof}
[Proof of Proposition \ref{prop.sum.matrices.multi}.]We could say that proving
Proposition \ref{prop.sum.matrices.multi} is just a matter of applying
Proposition \ref{prop.sum.matrices.A+B} several times (since a sum of finitely
many matrices can be obtained by repeatedly adding one matrix to another).
This single sentence might not be a rigorous proof in itself, but it would
pass for a proof in any mathematical paper or textbook, because any
mathematician can easily make it as rigorous as she wants to have it by
filling in the missing (straightforward) details.\footnote{Actually, there is
one more subtlety involved: Namely, if $h=0$, then the sums appearing in
(\ref{eq.prop.sum.matrices.multi.eq}) are not obtained by addition (in fact,
they are empty), and therefore we cannot use Proposition
\ref{prop.sum.matrices.A+B} to prove (\ref{eq.prop.sum.matrices.multi.eq}) in
this case. However, (\ref{eq.prop.sum.matrices.multi.eq}) is completely
obvious in this case anyway (since it just says that $\left(  0_{n\times
m}\right)  _{i,j}=0$).}

However, these notes are supposed to double as an introduction to proofs, so
let me actually show how a rigorous proof of Proposition
\ref{prop.sum.matrices.multi} looks like (even though it is exactly as
straightforward and dull as you would expect it to be).

Let $i\in\left\{  1,2,\ldots,n\right\}  $ and $j\in\left\{  1,2,\ldots
,m\right\}  $. We claim that%
\begin{equation}
\left(  \sum_{k=1}^{\ell}A_{k}\right)  _{i,j}=\sum_{k=1}^{\ell}\left(
A_{k}\right)  _{i,j} \label{pf.prop.sum.matrices.multi.1}%
\end{equation}
for every $\ell\in\left\{  0,1,\ldots,h\right\}  $. (Once this is proven, then
we will be able to obtain (\ref{eq.prop.sum.matrices.multi.eq}) simply by
applying (\ref{pf.prop.sum.matrices.multi.1}) to $\ell=h$.)

We will prove (\ref{pf.prop.sum.matrices.multi.1}) by \textit{induction over
}$\ell$. (If you have never seen a proof by induction: this here is an
example.) This means that we shall prove the following two claims:

\begin{statement}
\textit{Claim 1:} (\ref{pf.prop.sum.matrices.multi.1}) holds for $\ell=0$.
\end{statement}

\begin{statement}
\textit{Claim 2:} If $L\in\left\{  0,1,\ldots,h-1\right\}  $ is such that
(\ref{pf.prop.sum.matrices.multi.1}) holds for $\ell=L$, then
(\ref{pf.prop.sum.matrices.multi.1}) also holds for $\ell=L+1$.
\end{statement}

Once these two claims are proven, the \textit{principle of mathematical
induction} will yield that (\ref{pf.prop.sum.matrices.multi.1}) holds for all
$\ell\in\left\{  0,1,\ldots,h\right\}  $. In fact:

\begin{itemize}
\item Claim 1 shows that (\ref{pf.prop.sum.matrices.multi.1}) holds for
$\ell=0$;

\item thus, Claim 2 (applied to $L=0$) shows that
(\ref{pf.prop.sum.matrices.multi.1}) holds for $\ell=1$;

\item thus, Claim 2 (applied to $L=1$) shows that
(\ref{pf.prop.sum.matrices.multi.1}) holds for $\ell=2$;

\item thus, Claim 2 (applied to $L=2$) shows that
(\ref{pf.prop.sum.matrices.multi.1}) holds for $\ell=3$;

\item and so on, applying Claim 2 for higher and higher $L$, until we arrive
at $\ell=h$.
\end{itemize}

(See \cite[Chapter 5]{LeLeMe16} for an introduction to proofs by induction.)

Of course, we still have to prove the two claims.

\begin{enumerate}
\item \textit{Proof of Claim 1:} For $L=0$, the statement
(\ref{pf.prop.sum.matrices.multi.1}) claims that $\left(  \sum_{k=1}^{0}%
A_{k}\right)  _{i,j}=\sum_{k=1}^{0}\left(  A_{k}\right)  _{i,j}$. But
\[
\sum_{k=1}^{0}A_{k}=\left(  \text{an empty sum of }n\times m\text{-matrices}%
\right)  =0_{n\times m}%
\]
(since an empty sum of $n\times m$-matrices was \textbf{defined} to be
$0_{n\times m}$). Hence,%
\[
\left(  \sum_{k=1}^{0}A_{k}\right)  _{i,j}=\left(  0_{n\times m}\right)
_{i,j}=0
\]
(since every entry of the zero matrix $0_{n\times m}$ is $0$). Compared with
\[
\sum_{k=1}^{0}\left(  A_{k}\right)  _{i,j}=\left(  \text{an empty sum of
numbers}\right)  =0,
\]
this yields $\left(  \sum_{k=1}^{0}A_{k}\right)  _{i,j}=\sum_{k=1}^{0}\left(
A_{k}\right)  _{i,j}$. In other words, (\ref{pf.prop.sum.matrices.multi.1})
holds for $\ell=0$. This proves Claim 1.

\item \textit{Proof of Claim 2:} Let $L\in\left\{  0,1,\ldots,h-1\right\}  $
be such that (\ref{pf.prop.sum.matrices.multi.1}) holds for $\ell=L$. We must
show that (\ref{pf.prop.sum.matrices.multi.1}) holds for $\ell=L+1$.

Since (\ref{pf.prop.sum.matrices.multi.1}) holds for $\ell=L$, we have%
\begin{equation}
\left(  \sum_{k=1}^{L}A_{k}\right)  _{i,j}=\sum_{k=1}^{L}\left(  A_{k}\right)
_{i,j}. \label{pf.prop.sum.matrices.multi.1.pf.1}%
\end{equation}


Now,%
\[
\sum_{k=1}^{L+1}A_{k}=\sum_{k=1}^{L}A_{k}+A_{L+1}.
\]
Thus,%
\begin{align*}
\left(  \sum_{k=1}^{L+1}A_{k}\right)  _{i,j}  &  =\left(  \sum_{k=1}^{L}%
A_{k}+A_{L+1}\right)  _{i,j}=\underbrace{\left(  \sum_{k=1}^{L}A_{k}\right)
_{i,j}}_{\substack{=\sum_{k=1}^{L}\left(  A_{k}\right)  _{i,j}\\\text{(by
(\ref{pf.prop.sum.matrices.multi.1.pf.1}))}}}+\left(  A_{L+1}\right)  _{i,j}\\
&  \ \ \ \ \ \ \ \ \ \ \left(  \text{by Proposition
\ref{prop.sum.matrices.A+B}, applied to }A=\sum_{k=1}^{L}A_{k}\text{ and
}B=A_{L+1}\right) \\
&  =\sum_{k=1}^{L}\left(  A_{k}\right)  _{i,j}+\left(  A_{L+1}\right)
_{i,j}=\sum_{k=1}^{L+1}\left(  A_{k}\right)  _{i,j}.
\end{align*}
In other words, (\ref{pf.prop.sum.matrices.multi.1}) holds for $\ell=L+1$.
This proves Claim 2.
\end{enumerate}

Now, both Claims 1 and 2 are proven, and thus (as we have explained above) the
proof of (\ref{pf.prop.sum.matrices.multi.1}) is complete.

(The way we organized our proof of (\ref{pf.prop.sum.matrices.multi.1}) is
typical for a proof by mathematical induction. Usually, the proof of Claim 1
is called the \textquotedblleft induction base\textquotedblright, and the
proof of Claim 2 is called the \textquotedblleft induction
step\textquotedblright. In the induction step, we have made the assumption
that (\ref{pf.prop.sum.matrices.multi.1}) holds for $\ell=L$; this assumption
is called the \textquotedblleft induction hypothesis\textquotedblright.)

Now that (\ref{pf.prop.sum.matrices.multi.1}) is proven, we can simply apply
(\ref{pf.prop.sum.matrices.multi.1}) to $\ell=h$, and conclude that $\left(
\sum_{k=1}^{h}A_{k}\right)  _{i,j}=\sum_{k=1}^{h}\left(  A_{k}\right)  _{i,j}%
$. This proves Proposition \ref{prop.sum.matrices.multi}.
\end{proof}

The sums in Proposition \ref{prop.sum.matrices.multi} range from $k=1$ to $h$;
but the same statement holds for arbitrary sums:

\begin{proposition}
\label{prop.sum.matrices.multi.pq}Let $n\in\mathbb{N}$ and $m\in\mathbb{N}$.
Let $p$ and $q$ be two integers such that $p\leq q+1$. Let $A_{p}%
,A_{p+1},\ldots,A_{q}$ be some $n\times m$-matrices. For every $i\in\left\{
1,2,\ldots,n\right\}  $ and $j\in\left\{  1,2,\ldots,m\right\}  $, we have%
\begin{equation}
\left(  \sum_{k=p}^{q}A_{k}\right)  _{i,j}=\sum_{k=p}^{q}\left(  A_{k}\right)
_{i,j}. \label{eq.prop.sum.matrices.multi.pq.eq}%
\end{equation}

\end{proposition}

As a further illustration of manipulating sums, let me derive this proposition
from Proposition \ref{prop.sum.matrices.multi}:

\begin{proof}
[Proof of Proposition \ref{prop.sum.matrices.multi.pq}.]Let $h=q+1-p$. Then,
$h=q+1-p\geq0$ (since $p\leq q+1$), so that $h\in\mathbb{N}$. Also,
\[
p-1+\underbrace{h}_{=q+1-p}=p-1+q+1-p=q.
\]


Notice that $A_{p},A_{p+1},\ldots,A_{q}$ are $q+1-p$ matrices. In other words,
$A_{p},A_{p+1},\ldots,A_{q}$ are $h$ matrices (since $h=q+1-p$). Denote these
$h$ matrices by $B_{1},B_{2},\ldots,B_{h}$. Thus,
\begin{equation}
B_{k}=A_{p-1+k}\ \ \ \ \ \ \ \ \ \ \text{for every }k\in\left\{
1,2,\ldots,h\right\}  . \label{pf.prop.sum.matrices.multi.pq.Bk=}%
\end{equation}


Let $i\in\left\{  1,2,\ldots,n\right\}  $ and $j\in\left\{  1,2,\ldots
,m\right\}  $. We have
\begin{align}
\sum_{k=1}^{h}\underbrace{B_{k}}_{\substack{=A_{p-1+k}\\\text{(by
(\ref{pf.prop.sum.matrices.multi.pq.Bk=}))}}}  &  =\sum_{k=1}^{h}%
A_{p-1+k}=A_{p}+A_{p+1}+\cdots+A_{p-1+h}\nonumber\\
&  =A_{p}+A_{p+1}+\cdots+A_{q}\ \ \ \ \ \ \ \ \ \ \left(  \text{since
}p-1+h=q\right) \nonumber\\
&  =\sum_{k=p}^{q}A_{k} \label{pf.prop.sum.matrices.multi.pq.1}%
\end{align}
and%
\begin{align}
\sum_{k=1}^{h}\left(  \underbrace{B_{k}}_{\substack{=A_{p-1+k}\\\text{(by
(\ref{pf.prop.sum.matrices.multi.pq.Bk=}))}}}\right)  _{i,j}  &  =\sum
_{k=1}^{h}\left(  A_{p-1+k}\right)  _{i,j}=\left(  A_{p}\right)
_{i,j}+\left(  A_{p+1}\right)  _{i,j}+\cdots+\left(  A_{p-1+h}\right)
_{i,j}\nonumber\\
&  =\left(  A_{p}\right)  _{i,j}+\left(  A_{p+1}\right)  _{i,j}+\cdots+\left(
A_{q}\right)  _{i,j}\ \ \ \ \ \ \ \ \ \ \left(  \text{since }p-1+h=q\right)
\nonumber\\
&  =\sum_{k=p}^{q}\left(  A_{k}\right)  _{i,j}.
\label{pf.prop.sum.matrices.multi.pq.2}%
\end{align}
But Proposition \ref{prop.sum.matrices.multi} (applied to $B_{1},B_{2}%
,\ldots,B_{h}$ instead of $A_{1},A_{2},\ldots,A_{h}$) shows that%
\[
\left(  \sum_{k=1}^{h}B_{k}\right)  _{i,j}=\sum_{k=1}^{h}\left(  B_{k}\right)
_{i,j}.
\]
In view of (\ref{pf.prop.sum.matrices.multi.pq.1}) and
(\ref{pf.prop.sum.matrices.multi.pq.2}), this rewrites as%
\[
\left(  \sum_{k=p}^{q}A_{k}\right)  _{i,j}=\sum_{k=p}^{q}\left(  A_{k}\right)
_{i,j}.
\]
Thus, Proposition \ref{prop.sum.matrices.multi.pq} is proven.
\end{proof}

\subsection{(*) Application: Fibonacci numbers}

Here is a simple application of matrix multiplication to elementary mathematics.

\begin{definition}
\label{def.fibonacci}The \textit{Fibonacci sequence} is the sequence $\left(
0,1,1,2,3,5,8,13,21,34,55,\ldots\right)  $ that is defined as follows: Its
first two entries are $0$ and $1$, and each further entry is the sum of the
previous two entries. In more formal terms, it is the sequence $\left(
f_{0},f_{1},f_{2},f_{3},\ldots\right)  $ (we start the labelling at $0$)
defined recursively by%
\begin{align*}
f_{0}  &  =0,\ \ \ \ \ \ \ \ \ \ f_{1}=1,\ \ \ \ \ \ \ \ \ \ \text{and}\\
f_{n}  &  =f_{n-1}+f_{n-2}\ \ \ \ \ \ \ \ \ \ \text{for every }n\geq2.
\end{align*}
The elements of this sequence are called the \textit{Fibonacci numbers}.
\end{definition}

Definition \ref{def.fibonacci} gives a straightforward way to compute each
particular Fibonacci number $f_{n}$, by computing the first $n+1$ Fibonacci
numbers $f_{0},f_{1},\ldots,f_{n}$ one after the others. For example, it
gives
\begin{align*}
f_{0} = 0; \qquad f_{1} = 1; \qquad f_{2} = 1+0 = 1; \qquad f_{3} = 1+1 = 2;
\qquad f_{4} = 2+1 = 3;\\
f_{5} = 3+2 = 5; \qquad f_{6} = 5+3 = 8; \qquad f_{7} = 8+5 = 13; \qquad f_{8}
= 13+8 = 21,
\end{align*}
and so on. However, when $n$ is large, computing $f_{n}$ by this method is
time-consuming (each of the $n+1$ first Fibonacci numbers has to be
computed!). Is there a faster way to compute Fibonacci numbers?

It turns out that there is. It is based on the following fact:

\begin{proposition}
\label{prop.fibonacci.matrix}Let $B$ be the $2\times2$-matrix $\left(
\begin{array}
[c]{cc}%
1 & 1\\
1 & 0
\end{array}
\right)  $. Then, for every positive integer $n$, we have%
\begin{equation}
B^{n}=\left(
\begin{array}
[c]{cc}%
f_{n+1} & f_{n}\\
f_{n} & f_{n-1}%
\end{array}
\right)  . \label{eq.prop.fibonacci.matrix.1}%
\end{equation}

\end{proposition}

\begin{proof}
[Proof of Proposition \ref{prop.fibonacci.matrix}.]We shall prove Proposition
\ref{prop.fibonacci.matrix} by induction over $n$:

\textit{Induction base:} We have $B^{1}=B=\left(
\begin{array}
[c]{cc}%
1 & 1\\
1 & 0
\end{array}
\right)  $. Comparing this with%
\[
\left(
\begin{array}
[c]{cc}%
f_{1+1} & f_{1}\\
f_{1} & f_{1-1}%
\end{array}
\right)  =\left(
\begin{array}
[c]{cc}%
1 & 1\\
1 & 0
\end{array}
\right)  \ \ \ \ \ \ \ \ \ \ \left(  \text{since }f_{1+1}=f_{2}=1\text{,
}f_{1}=1\text{ and }f_{1-1}=f_{0}=0\right)  ,
\]
we obtain $B^{1}=\left(
\begin{array}
[c]{cc}%
f_{1+1} & f_{1}\\
f_{1} & f_{1-1}%
\end{array}
\right)  $. In other words, Proposition \ref{prop.fibonacci.matrix} holds for
$n=1$. This completes the induction base. (This was a completely
straightforward computation. In the future, we will often leave such
computations to the reader.)

\textit{Induction step:} Let $N$ be a positive integer. Assume that
Proposition \ref{prop.fibonacci.matrix} holds for $n=N$. We must show that
Proposition \ref{prop.fibonacci.matrix} also holds for $n=N+1$.

The definition of the Fibonacci sequence shows that $f_{N+2}=f_{N+1}+f_{N}$
and $f_{N+1}=f_{N}+f_{N-1}$.

We have assumed that Proposition \ref{prop.fibonacci.matrix} holds for $n=N$.
In other words,%
\[
B^{N}=\left(
\begin{array}
[c]{cc}%
f_{N+1} & f_{N}\\
f_{N} & f_{N-1}%
\end{array}
\right)  .
\]
Now,
\begin{align*}
B^{N+1}  &  =\underbrace{B^{N}}_{=\left(
\begin{array}
[c]{cc}%
f_{N+1} & f_{N}\\
f_{N} & f_{N-1}%
\end{array}
\right)  }\underbrace{B}_{=\left(
\begin{array}
[c]{cc}%
1 & 1\\
1 & 0
\end{array}
\right)  }=\left(
\begin{array}
[c]{cc}%
f_{N+1} & f_{N}\\
f_{N} & f_{N-1}%
\end{array}
\right)  \left(
\begin{array}
[c]{cc}%
1 & 1\\
1 & 0
\end{array}
\right) \\
&  =\left(
\begin{array}
[c]{cc}%
f_{N+1}\cdot1+f_{N}\cdot1 & f_{N+1}\cdot1+f_{N}\cdot0\\
f_{N}\cdot1+f_{N-1}\cdot1 & f_{N}\cdot1+f_{N-1}\cdot0
\end{array}
\right) \\
&  \ \ \ \ \ \ \ \ \ \ \left(  \text{by the definition of a product of two
matrices}\right) \\
&  =\left(
\begin{array}
[c]{cc}%
f_{N+1}+f_{N} & f_{N+1}\\
f_{N}+f_{N-1} & f_{N}%
\end{array}
\right)  =\left(
\begin{array}
[c]{cc}%
f_{N+2} & f_{N+1}\\
f_{N+1} & f_{N}%
\end{array}
\right)
\end{align*}
(since $f_{N+1}+f_{N}=f_{N+2}$ and $f_{N}+f_{N-1}=f_{N+1}$). In other words,
Proposition \ref{prop.fibonacci.matrix} holds for $n=N+1$. This completes the
induction step; hence, Proposition \ref{prop.fibonacci.matrix} is proven.
\end{proof}

How does Proposition \ref{prop.fibonacci.matrix} help us compute $f_{n}$
quickly? Naively computing $B^{n}$ by multiplying $B$ with itself $n$ times is
not any faster than computing $f_{n}$ directly using Definition
\ref{def.fibonacci} (in fact, it is slower, since multiplying matrices takes
longer than adding numbers). However, there is a trick for computing powers
quickly, called \textit{binary exponentiation}; this trick works just as well
for matrices as it does for numbers. The trick uses the following observations:

\begin{enumerate}
\item For every $m\in\mathbb{N}$, we have $B^{2m}=\left(  B^{m}\right)  ^{2}$.

\item For every $m\in\mathbb{N}$, we have $B^{2m+1}=B\left(  B^{m}\right)
^{2}$.
\end{enumerate}

These observations allow us to quickly compute $B^{2m}$ and $B^{2m+1}$ using
only $B^{m}$; thus, we can \textquotedblleft jump up\textquotedblright\ from
$B^{m}$ directly to $B^{2m}$ and to $B^{2m+1}$ without the intermediate steps
$B^{m+1},B^{m+2},\ldots,B^{2m-1}$. Let us use this to compute $B^{90}$ (and
thus $f_{90}$) quickly (without computing $91$ Fibonacci numbers):

\begin{itemize}
\item We want to find $B^{90}$. Since $90=2\cdot45$, we have $B^{90}=\left(
B^{45}\right)  ^{2}$ (by the formula $B^{2m}=\left(  B^{m}\right)  ^{2}$).

\item We thus want to find $B^{45}$. Since $45=2\cdot22+1$, we have
$B^{45}=B\left(  B^{22}\right)  ^{2}$ (by the formula $B^{2m+1}=B\left(
B^{m}\right)  ^{2}$).

\item We thus want to find $B^{22}$. Since $22=2\cdot11$, we have
$B^{22}=\left(  B^{11}\right)  ^{2}$ (by the formula $B^{2m}=\left(
B^{m}\right)  ^{2}$).

\item We thus want to find $B^{11}$. Since $11=2\cdot5+1$, we have
$B^{11}=B\left(  B^{5}\right)  ^{2}$ (by the formula $B^{2m+1}=B\left(
B^{m}\right)  ^{2}$).

\item We thus want to find $B^{5}$. Since $5=2\cdot2+1$, we have
$B^{5}=B\left(  B^{2}\right)  ^{2}$ (by the formula $B^{2m+1}=B\left(
B^{m}\right)  ^{2}$).

\item We thus want to find $B^{2}$. Since $2=2\cdot1$, we have $B^{2}=\left(
B^{1}\right)  ^{2}$ (by the formula $B^{2m}=\left(  B^{m}\right)  ^{2}$, but
this was obvious anyway).
\end{itemize}

We know what $B^{1}$ is: $B^{1}=B=\left(
\begin{array}
[c]{cc}%
1 & 1\\
1 & 0
\end{array}
\right)  $. Hence, $B^{2}=\left(  B^{1}\right)  ^{2}$ becomes $B^{2}=\left(
\begin{array}
[c]{cc}%
1 & 1\\
1 & 0
\end{array}
\right)  ^{2}=\left(
\begin{array}
[c]{cc}%
2 & 1\\
1 & 1
\end{array}
\right)  $. Hence, $B^{5}=B\left(  B^{2}\right)  ^{2}$ becomes $B^{5}=\left(
\begin{array}
[c]{cc}%
1 & 1\\
1 & 0
\end{array}
\right)  \left(
\begin{array}
[c]{cc}%
2 & 1\\
1 & 1
\end{array}
\right)  ^{2}=\left(
\begin{array}
[c]{cc}%
8 & 5\\
5 & 3
\end{array}
\right)  $. Hence, $B^{11}=B\left(  B^{5}\right)  ^{2}$ becomes $B^{11}%
=\left(
\begin{array}
[c]{cc}%
1 & 1\\
1 & 0
\end{array}
\right)  \left(
\begin{array}
[c]{cc}%
8 & 5\\
5 & 3
\end{array}
\right)  ^{2}=\left(
\begin{array}
[c]{cc}%
144 & 89\\
89 & 55
\end{array}
\right)  $. Hence, $B^{22}=\left(  B^{11}\right)  ^{2}$ becomes $B^{22}%
=\left(
\begin{array}
[c]{cc}%
144 & 89\\
89 & 55
\end{array}
\right)  ^{2}=\left(
\begin{array}
[c]{cc}%
28657 & 17711\\
17711 & 10946
\end{array}
\right)  $. Hence, $B^{45}=B\left(  B^{22}\right)  ^{2}$ becomes
\newline$B^{45}=\left(
\begin{array}
[c]{cc}%
1 & 1\\
1 & 0
\end{array}
\right)  \left(
\begin{array}
[c]{cc}%
28657 & 17711\\
17711 & 10946
\end{array}
\right)  ^{2}=\left(
\begin{array}
[c]{cc}%
1836311903 & 1134903170\\
1134903170 & 701408733
\end{array}
\right)  $. Hence, $B^{90}=\left(  B^{45}\right)  ^{2}$ becomes \newline%
$B^{90}=\left(
\begin{array}
[c]{cc}%
1836311903 & 1134903170\\
1134903170 & 701408733
\end{array}
\right)  ^{2}=\allowbreak\left(
\begin{array}
[c]{cc}%
4660046610375530309 & 2880067194370816120\\
2880067194370816120 & 1779979416004714189
\end{array}
\right)  $. Since $f_{90}$ is the $\left(  2,1\right)  $-th entry of $B^{90}$
(indeed, (\ref{eq.prop.fibonacci.matrix.1}) shows that $f_{n}$ is the $\left(
2,1\right)  $-th entry of $B^{n}$ for all positive integers $n$), we thus
obtain%
\[
f_{90}=2880067194370816120.
\]


\begin{exercise}
\label{exe.fibonacci.Fn+m}Show that, for any two positive integers $n$ and
$m$, we have%
\[
f_{n+m}=f_{n}f_{m+1}+f_{n-1}f_{m}.
\]


[\textbf{Hint:} Begin with the equality $B^{n}B^{m}=B^{n+m}$. Rewrite it using
Proposition \ref{prop.fibonacci.matrix}, and compare entries.]
\end{exercise}

\subsection{\label{sect.intro.numbers}(*) What is a number?}

So far, our notion of a matrix relies on a (somewhat vague) notion of a
\textquotedblleft number\textquotedblright. What does the word
\textquotedblleft number\textquotedblright\ mean? There are several possible
candidates for a meaning of this word: for example, \textquotedblleft
number\textquotedblright\ might mean \textquotedblleft rational
number\textquotedblright, but might also mean \textquotedblleft real
number\textquotedblright\ or \textquotedblleft complex
number\textquotedblright. For what we have been doing so far, the precise
choice of meaning does not matter. However, it eventually \textbf{will}
matter, so let me discuss it briefly. (See any book on abstract algebra for a
more detailed and systematic discussion.)

First, let me introduce some well-known sets:

\begin{itemize}
\item As explained above, $\mathbb{N}$ means the set of all nonnegative
integers: $\mathbb{N}=\left\{  0,1,2,\ldots\right\}  $.

\item Furthermore, $\mathbb{Z}$ means the set of all integers: $\mathbb{Z}%
=\left\{  \ldots,-2,-1,0,1,2,\ldots\right\}  $.

\item Moreover, $\mathbb{Q}$ means the set of all rational numbers:
$\mathbb{Q}=\left\{  \dfrac{a}{b}\ \mid\ a\in\mathbb{Z},\ b\in\mathbb{Z}%
\setminus\left\{  0\right\}  \right\}  $.

\item Furthermore, $\mathbb{R}$ means the set of all real numbers. This
contains rational numbers such as $-2$ and $\dfrac{5}{3}$, but also irrational
numbers such as $\sqrt{2}$ and $\dfrac{\sqrt{3}}{1+\sqrt[3]{5}}$ and $\pi$
(and various others, many of which cannot even be described in
words\footnote{This is not a poetic metaphor. What I am saying is that there
are real numbers which cannot be described by any finite formula or computed
(to arbitrary precision) by any finite algorithm. The reason is simply that
there are uncountably many real numbers, but only countably many formulas and
algorithms. If you find this unintuitive, imagine an immortal monkey typing an
infinite decimal number on an infinite-memory computer:
\texttt{9.461328724290054}... If the monkey is typing truly at random, then no
finite rule or formula will suffice to predict every single digit he types;
thus, the real number he defines is undescribable. (This, of course, is not a
proof.)}).

\item Finally, $\mathbb{C}$ means the set of all complex numbers. They will be
rarely used in these notes (indeed, most of linear algebra can be done without
them, except for eigenvalue/eigenvector theory), so you do not actually have
to know them in order to read these notes. However, let me give a quick
briefing on complex numbers (probably more of a reminder for those who have
already seen them):

Complex numbers can be formally defined as pairs of real numbers $\left(
a,b\right)  $ with an entrywise addition (that is, $\left(  a,b\right)
+\left(  a^{\prime},b^{\prime}\right)  =\left(  a+a^{\prime},b+b^{\prime
}\right)  $, exactly like $1\times2$-matrices) and a somewhat strange-looking
multiplication (namely, $\left(  a,b\right)  \left(  a^{\prime},b^{\prime
}\right)  =\left(  aa^{\prime}-bb^{\prime},ab^{\prime}+ba^{\prime}\right)
$).\ \ \ \ \footnote{This sort of definition is not unlike our definition of
matrices: they are also tables with entrywise addition and a less simple
multiplication.} But the way everyone thinks about complex numbers
(informally) is that they are an extension of real numbers (so $\mathbb{R}$ is
a subset of $\mathbb{C}$) by adding a new \textquotedblleft imaginary
number\textquotedblright\ $i$ which satisfies $i^{2}=-1$. They are supposed to
behave like real numbers as far as laws of addition and multiplication are
concerned (thus, for instance, $a\left(  b+c\right)  =ab+ac$ and $a\left(
bc\right)  =\left(  ab\right)  c$); using these laws and the requirement that
$i^{2}=-1$, one can easily see how to multiply and add arbitrary complex
numbers. These two definitions (the formal one as pairs of real numbers, and
the informal one as \textquotedblleft extended real numbers\textquotedblright)
are equivalent, and the complex number $\left(  a,b\right)  $ (according to
the first definition) corresponds to the complex number $a+bi$ (according to
the second definition).

For a detailed introduction to complex numbers, see \cite[\S 2]{LaNaSc16}.

\item We shall occasionally use another set: the set $\overline{\mathbb{Q}}$
of all \textit{algebraic numbers}. These are complex numbers that are roots of
nonzero polynomials with rational coefficients. For instance, $\sqrt{2}$ is an
algebraic number\footnote{because it is a root of the polynomial $X^{2}-2$},
and $i$ is an algebraic number\footnote{since it is a root of $X^{2}+1$}, and
$\dfrac{5}{2}$ is an algebraic number\footnote{since it is a root of
$X-\dfrac{5}{2}$ (this sounds stupid, but it is perfectly valid, since
$-\dfrac{5}{2}$ is rational)}, and $\sqrt{2}+\sqrt{3}$ is an algebraic
number\footnote{since it is a root of $X^{4}-10X^{2}+1$ (check this!)}; but
$\pi$ (for example) is not an algebraic number\footnote{... which is the
reason why the circle cannot be squared. The actual proof is rather difficult;
see \url{http://mathoverflow.net/questions/34055/transcendence-of-pi} and
\url{http://math.stackexchange.com/questions/31798/prove-that-pi-is-a-transcendental-number}
for some references.}.

Again, you can read these notes without understanding algebraic numbers, but
let me explain why they are useful.

First, let me explain why real numbers are \textquotedblleft
bad\textquotedblright. Computations with rational numbers can be done
\textbf{exactly} on a computer: for example, a computer algebra system (such
as \href{http://www.sagemath.org/}{SageMath}) can evaluate to $\dfrac
{\dfrac{12}{13}-\dfrac{1}{15}}{\dfrac{2}{3}+3}$ to $\dfrac{167}{715}$ with a
100\% accuracy and a 100\% guarantee of correctness\footnote{barring bugs in
the software}. Similarly, it can do all kinds of computations with rational
numbers with 100\% accuracy (as long as it doesn't run out of memory).
However, computations with real numbers are \textbf{doomed to be
inexact}\footnote{Complex numbers suffer from the same problem.}. A real
number can encode \textquotedblleft an infinite amount of
information\textquotedblright\footnote{Think of a real number as an infinite
decimal fraction, encoded by an infinite sequence of digits.}; no computer can
even store such a thing\footnote{This is precisely the above-mentioned problem
of the ineffable real numbers.}. The consequence is that computers will work
with approximations when you give them real numbers. Even real numbers like
$\pi$ that can be \textbf{defined} in finite time still cannot be
\textbf{computed with} automatically without approximation. Typically,
computers approximate real numbers by \textbf{floating-point numbers}, which
store only a fixed number of significant digits.\footnote{How many depends on
the system and the application. For example, the Python 2.7.10 on my computer
seems to treat $1.0000000000000000000001$ and $1$ as identical, but it can
tell $1.00000000000001$ apart from $1$. See
\url{https://docs.python.org/2/tutorial/floatingpoint.html} for a more
thorough explanation of how Python approximates real numbers.} When you use
approximations, you need to be prepared to get wrong results; sometimes the
error in the result will be completely out of proportion to the little
inaccuracies in the approximation! Here is an example: The system of linear
equations%
\begin{equation}
\left\{
\begin{array}
[c]{c}%
x+2y=2;\\
3x+6y=6
\end{array}
\right.  \label{eq.inexact.1}%
\end{equation}
in two variables $x$ and $y$ has infinitely many solutions (namely, $\left(
x,y\right)  =\left(  2t,1-t\right)  $ is a solution for each number $t$). But
if we allow ourselves one little inaccuracy (the kind that computers
necessarily do when they work with real numbers) and replace the $3$ by
$3.000000000001$, then we obtain the system%
\begin{equation}
\left\{
\begin{array}
[c]{c}%
x+2y=2;\\
3.000000000001x+6y=6
\end{array}
\right.  , \label{eq.inexact.2}%
\end{equation}
which has only one solution (namely, $\left(  x,y\right)  =\left(  0,1\right)
$). On the other hand, if we instead change the second $6$ in the second
equation of (\ref{eq.inexact.1}) to a $6.00000000001$ (again, a typical little
imprecision), then the resulting system%
\begin{equation}
\left\{
\begin{array}
[c]{c}%
x+2y=2;\\
3x+6y=6.00000000001
\end{array}
\right.  \label{eq.inexact.3}%
\end{equation}
will have no solutions at all. Thus, the minuscule differences between the
three systems (\ref{eq.inexact.1}), (\ref{eq.inexact.2}) and
(\ref{eq.inexact.3}) have led to three wildly different results (infinitely
many, one or no solutions). The consequence is that \textbf{if you want a
computer to reliably solve the system (\ref{eq.inexact.1}), you must make sure
that it treats the coefficients (}$1,2,2,3,6,6$\textbf{) as rational numbers
(not as real numbers)} and avoids any approximations.

This, of course, only works well if the coefficients \textbf{are} rational
numbers. You cannot solve a system like%
\begin{equation}
\left\{
\begin{array}
[c]{c}%
\sqrt{3}x+\sqrt{2}y=\pi;\\
\left(  1-\sqrt{3}\right)  x+\pi y=0
\end{array}
\right.  \label{eq.inexact.4}%
\end{equation}
this way. More annoyingly, you cannot solve a system like%
\begin{equation}
\left\{
\begin{array}
[c]{c}%
\sqrt{3}x+\sqrt{2}y=1;\\
\left(  1-\sqrt{3}\right)  x+2y=0
\end{array}
\right.  \label{eq.inexact.5}%
\end{equation}
this way, although you probably could solve it by hand! Systems like
(\ref{eq.inexact.4}) are (in a sense) hopeless: Computers cannot reliably work
with real numbers without approximating them at some point. But
(\ref{eq.inexact.5}) can be salvaged: All the coefficients in
(\ref{eq.inexact.5}) are algebraic numbers, and modern computer algebra
systems (e.g., SageMath) can work with algebraic numbers with 100\%
precision.\footnote{This is \textbf{not} easy! For example, a classical puzzle
asks you to prove that $\sqrt[3]{2+\sqrt{5}}+\sqrt[3]{2-\sqrt{5}}=1$. This is
not obvious; there are no straightforward \textquotedblleft
simplifications\textquotedblright\ that take you from $\sqrt[3]{2+\sqrt{5}%
}+\sqrt[3]{2-\sqrt{5}}$ to $1$. Yet, this is one of the things that the
computer must be taught to do, since otherwise it could not decide whether the
linear equation $\left(  \sqrt[3]{2+\sqrt{5}}+\sqrt[3]{2-\sqrt{5}}-1\right)
x=0$ has one or infinitely many solutions.} Thus, even if you don't care much
about algebraic numbers, you will need to tell your computer that your numbers
are algebraic in order to have it solve systems like (\ref{eq.inexact.5}).

Of course, this all doesn't mean that linear algebra with real numbers is
useless in practice. The system (\ref{eq.inexact.1}) is a rather ill-behaved
system; many systems allow for a pretty good approximation of their solutions
even in spite of imprecisions, and even when a system is ill-behaved like
(\ref{eq.inexact.1}), there are methods that compute the \textquotedblleft
likeliest solution\textquotedblright\ (such as the least-squares method). We
shall (hopefully) see some of these methods in these notes. A less alarmist
slogan would thus be: You can do linear algebra with real numbers, but you
should be aware of its limitations and keep track of the errors
(\textquotedblleft numerical stability\textquotedblright).
\end{itemize}

Note that $\mathbb{N}\subseteq\mathbb{Z}\subseteq\mathbb{Q}\subseteq
\mathbb{R}\subseteq\mathbb{C}$ (at least if you identify a real number $a$
with the complex number $\left(  a,0\right)  =a+0i$) and $\mathbb{Q}%
\subseteq\overline{\mathbb{Q}}\subseteq\mathbb{C}$.

So what is a number? Integers, rational numbers, real numbers, complex numbers
and algebraic numbers all have good claims to the name \textquotedblleft
number\textquotedblright. And even though all of these numbers can be viewed
as complex numbers, it can be useful to \textbf{not} treat them as complex
numbers by default, for example when you are using a computer and want 100\% precision.

Here is a better question: What are the things we can fill a matrix with? We
have so far used numbers, but we don't have to; we could also (for example)
use polynomials. The matrix $\left(
\begin{array}
[c]{cc}%
2X^{2}+1 & -1\\
X & X+7
\end{array}
\right)  $ is a $2\times2$-matrix whose entries are not numbers, but
polynomials in the variable $X$ (with rational coefficients). Such matrices
can be highly useful (and, in fact, will be used when we come to eigenvalues).
Such matrices can be added and multiplied (since polynomials can be added and
multiplied). Of course, we could also fill a matrix with all kinds of things
(words, names, smilies, scribbles) that \textbf{cannot} be added or multiplied
(after all, matrices are just tables); but then we won't be able to add and
multiply the resulting matrices, so we don't gain anything by calling them
\textquotedblleft matrices\textquotedblright\ (we don't want just a fancy
synonym for \textquotedblleft tables\textquotedblright). So it sounds most
reasonable to expect that a matrix should be filled with things that can be
added and multiplied. Moreover, addition and multiplication of these things
should obey certain laws (such as $\left(  a+b\right)  c=ac+bc$ and $ab=ba$)
in order to ensure that addition, scaling and multiplication of matrices will
obey the usual laws (e.g., Proposition \ref{prop.matrix-laws.1}) as well.
Formalizing this idea, we arrive at the notion of a commutative ring:

\begin{definition}
\label{def.commring}A \textit{commutative ring} means a set $\mathbb{K}$
equipped with the following additional data:

\begin{itemize}
\item a binary operation called \textquotedblleft$+$\textquotedblright\ (that
is, a function that takes two elements $a\in\mathbb{K}$ and $b\in\mathbb{K}$
as inputs, and outputs a new element of $\mathbb{K}$ which is denoted by $a+b$);

\item a binary operation called \textquotedblleft$\cdot$\textquotedblright%
\ (that is, a function that takes two elements $a\in\mathbb{K}$ and
$b\in\mathbb{K}$ as inputs, and outputs a new element of $\mathbb{K}$ which is
denoted by $a\cdot b$);

\item an element of $\mathbb{K}$ called \textquotedblleft$0$\textquotedblright;

\item an element of $\mathbb{K}$ called \textquotedblleft$1$\textquotedblright
\end{itemize}

satisfying the following conditions (\textquotedblleft
axioms\textquotedblright):

\begin{itemize}
\item \textit{Commutativity of addition:} We have $a+b=b+a$ for all
$a\in\mathbb{K}$ and $b\in\mathbb{K}$.

\item \textit{Commutativity of multiplication:} We have $ab=ba$ for all
$a\in\mathbb{K}$ and $b\in\mathbb{K}$. Here and in the following, $ab$ is
shorthand for $a\cdot b$ (as is usual for products of numbers).

\item \textit{Associativity of addition:} We have $a+\left(  b+c\right)
=\left(  a+b\right)  +c$ for all $a\in\mathbb{K}$, $b\in\mathbb{K}$ and
$c\in\mathbb{K}$.

\item \textit{Associativity of multiplication:} We have $a\left(  bc\right)
=\left(  ab\right)  c$ for all $a\in\mathbb{K}$, $b\in\mathbb{K}$ and
$c\in\mathbb{K}$.

\item \textit{Neutrality of }$0$\textit{:} We have $a+0=0+a=a$ for all
$a\in\mathbb{K}$.

\item \textit{Existence of additive inverses:} For every $a\in\mathbb{K}$,
there exists an element $a^{\prime}\in\mathbb{K}$ such that $a+a^{\prime
}=a^{\prime}+a=0$. This $a^{\prime}$ is commonly denoted by $-a$ and called
the \textit{additive inverse} of $a$. (It is easy to check that it is unique.)

\item \textit{Unitality (a.k.a. neutrality of }$1$\textit{):} We have
$1a=a1=a$ for all $a\in\mathbb{K}$.

\item \textit{Annihilation:} We have $0a=a0=0$ for all $a\in\mathbb{K}$.

\item \textit{Distributivity:} We have $a\left(  b+c\right)  =ab+ac$ and
$\left(  a+b\right)  c=ac+bc$ for all $a\in\mathbb{K}$, $b\in\mathbb{K}$ and
$c\in\mathbb{K}$.
\end{itemize}
\end{definition}

This definition was a mouthful, but its intention is rather simple: It defines
a commutative ring as a set equipped with two operations which behave like
addition and multiplication of numbers, and two elements which behave like the
number $0$ and the number $1$. As a consequence, if we have a commutative ring
$\mathbb{K}$, then matrices filled with elements of $\mathbb{K}$ will behave
(at least with regard to their basic properties, such as Proposition
\ref{prop.matrix-laws.1}) like matrices filled with numbers.

Here are some examples of commutative rings:

\begin{itemize}
\item Each of the sets $\mathbb{Z}$, $\mathbb{Q}$, $\mathbb{R}$, $\mathbb{C}$
and $\overline{\mathbb{Q}}$ (endowed with the usual addition, the usual
multiplication, the usual $0$ and the usual $1$) is a commutative ring.

\item The set $\mathbb{Q}\left[  x\right]  $ of all polynomials (in the
variable $x$) with rational coefficients (equipped with addition of
polynomials, multiplication of polynomials, the polynomial $0$ and the
polynomial $1$) is a ring.

\item The set of all functions $\mathbb{R}\rightarrow\mathbb{R}$ (equipped
with pointwise addition, pointwise multiplication, the constant-$0$ function
and the constant-$1$ function) is a ring.

\item If you know what \textquotedblleft integers modulo $n$\textquotedblright%
\ are (for a given positive integer $n$): The integers modulo $n$ (for a given
$n$) also form a commutative ring.

\item Here is a weirder example:

For any two sets $A$ and $B$, we let $A\bigtriangleup B$ denote the
\textit{symmetric difference} of $A$ and $B$. This is the set of all elements
which lie in exactly one of the two sets $A$ and $B$. Thus, $A\bigtriangleup
B=\left(  A\setminus B\right)  \cup\left(  B\setminus A\right)  =\left(  A\cup
B\right)  \setminus\left(  A\cap B\right)  $.

Fix some set $S$, and let $\mathcal{P}\left(  S\right)  $ denote the set of
all subsets of $S$. The set $\mathcal{P}\left(  S\right)  $ equipped with the
operation $\bigtriangleup$ (playing the role of \textquotedblleft%
$+$\textquotedblright), the operation $\cap$ (playing the role of
\textquotedblleft$\cdot$\textquotedblright), the element\footnote{The notation
$\varnothing$ stands for the empty set, i.e., the set $\left\{  {}\right\}
$.} $\varnothing$ (playing the role of \textquotedblleft$0$\textquotedblright)
and the element $S$ (playing the role of \textquotedblleft$1$%
\textquotedblright) is a commutative ring. This is an example of a
\textit{Boolean ring} (and also an example of the fact that the operations
\textquotedblleft$+$\textquotedblright\ and \textquotedblleft$\cdot
$\textquotedblright\ don't always have anything to do with addition and
multiplication of numbers!).
\end{itemize}

Many more examples of commutative rings can be found in textbooks on abstract
algebra (e.g., \cite[Chapter 11]{Artin10}).

Matrices filled with elements of a commutative ring $\mathbb{K}$ are called
\textit{matrices over }$\mathbb{K}$. More precisely:

\begin{definition}
Let $\mathbb{K}$ be a commutative ring. If $n\in\mathbb{N}$ and $m\in
\mathbb{N}$, then an $n\times m$\textit{-matrix over }$\mathbb{K}$ simply
means a rectangular table with $n$ rows and $m$ columns, such that each cell
is filled with an element of $\mathbb{K}$.
\end{definition}

Thus, matrices over $\mathbb{Q}$ are matrices with rational entries; matrices
over $\mathbb{R}$ are matrices with real entries; matrices over $\mathbb{Z}$
are matrices with integer entries.

As we have already explained, matrices over commutative rings behave like
matrices filled with numbers, at least as far as simple laws of computation
are concerned. To wit, Proposition \ref{prop.matrix-laws.1}, Proposition
\ref{prop.matrix-laws.0} and Proposition \ref{prop.matrix-laws.id} (as well as
some other results that will be proven later) still hold if the matrices are
filled with elements of a commutative ring $\mathbb{K}$ instead of numbers.
Some other (deeper) results might not hold for every commutative ring
$\mathbb{K}$: For example, Gaussian elimination, which we will meet in the
next chapter, requires that we can \textbf{divide} by nonzero elements of
$\mathbb{K}$, but this is not always possible when $\mathbb{K}$ is a
commutative ring. There is a word for that, too:

\begin{definition}
\label{def.field}A commutative ring $\mathbb{K}$ is called a \textit{field} if
it satisfies the following two axioms:

\begin{itemize}
\item \textit{Nontriviality:} We have $0\neq1$. (The \textquotedblleft%
$0$\textquotedblright\ and \textquotedblleft$1$\textquotedblright\ here, of
course, are the two specific elements of $\mathbb{K}$ that we have chosen to
call \textquotedblleft$0$\textquotedblright\ and \textquotedblleft%
$1$\textquotedblright. They aren't always the same as the \textbf{numbers} $0$
and $1$. In particular, in a commutative ring $\mathbb{K}$ they can be equal;
but for a field we want to disallow this.)

\item \textit{Existence of multiplicative inverses:} For every $a\in
\mathbb{K}$, we have \textbf{either} $a=0$, \textbf{or} there is an element
$b\in\mathbb{K}$ satisfying $ab=ba=1$.
\end{itemize}

The element $b$ in the \textquotedblleft existence of multiplicative
inverses\textquotedblright\ axiom is called the \textit{inverse} of $a$ and is
denoted by $a^{-1}$; it can be proven that this element is unique. If $u$ and
$v$ are two elements of a field $\mathbb{K}$ such that $v\neq0$, then the
product $uv^{-1}$ is denoted by $\dfrac{u}{v}$. Thus, any two elements of a
field $\mathbb{K}$ can be divided by each other as long as the denominator is
$\neq0$.
\end{definition}

For example, $\mathbb{Q}$, $\mathbb{R}$, $\mathbb{C}$ and $\overline
{\mathbb{Q}}$ are fields, but $\mathbb{Z}$ is not a field (because the integer
$2$ is nonzero, yet does not have an \textbf{integer} inverse). Some results
about matrices are based on the possibility of dividing by nonzero numbers;
these results cannot be directly generalized to matrices over a commutative
ring $\mathbb{K}$. But most of them can be generalized to matrices over a
field $\mathbb{K}$.

\begin{remark}
\textbf{(a)} The notion of a \textquotedblleft commutative
ring\textquotedblright\ is not standard, unfortunately. Some authors (e.g.,
Dummit and Foote in \cite[Part II]{DumFoo04}, or Goodman in \cite[Chapters 1
and 6]{Goodma15}) omit the element \textquotedblleft$1$\textquotedblright%
\ (and the unitality axiom), while some (older) authors even omit the
associativity of multiplication. If you read any text on abstract algebra, it
is prudent to check whether the author's concept of a commutative ring agrees
with yours.

\textbf{(b)} As you might have guessed, there is also a notion of a
\textquotedblleft noncommutative ring\textquotedblright. It is defined
precisely as a \textquotedblleft commutative ring\textquotedblright, except
that we omit the \textquotedblleft commutativity of
multiplication\textquotedblright\ axiom. (\textquotedblleft Commutativity of
addition\textquotedblright\ is left in!) It turns out that we already know a
neat example of a noncommutative ring: For any commutative ring $\mathbb{K}$
and any $n\in\mathbb{N}$, the set of all $n\times n$-matrices over
$\mathbb{K}$ is a noncommutative ring!

\textbf{(c)} The word ``ring'' (without the adjectives ``commutative'' and
``noncommutative'') usually means either ``commutative ring'' or
``noncommutative ring'', depending on the author's preferences.
\end{remark}

\section{\label{sect.gauss}Gaussian elimination}

In this chapter, we shall take aim at understanding Gaussian elimination in
terms of matrices. However, we will not head straight to this aim; instead, we
will first introduce various classes of matrices (triangular matrices, matrix
units, elementary matrices, permutation matrices), which will allow us to view
certain pieces of the Gaussian elimination algorithm in isolation. Once we are
done with that, we will finally explain Gaussian elimination in the general setting.

\subsection{Linear equations and matrices}

First of all, let us see what solving linear equations has to do with matrices.

\begin{example}
\label{exam.systems}Consider the following system of equations in three
unknowns $x,y,z$:%
\begin{equation}
\left\{
\begin{array}
[c]{c}%
3x+6y-z=2;\\
7x+4y-3z=3;\\
-y+8z=1
\end{array}
\right.  . \label{eq.exam.systems.1}%
\end{equation}
I claim that this system of equations is equivalent to the single equation%
\begin{equation}
\left(
\begin{array}
[c]{ccc}%
3 & 6 & -1\\
7 & 4 & -3\\
0 & -1 & 8
\end{array}
\right)  \left(
\begin{array}
[c]{c}%
x\\
y\\
z
\end{array}
\right)  =\left(
\begin{array}
[c]{c}%
2\\
3\\
1
\end{array}
\right)  \label{eq.exam.systems.2}%
\end{equation}
(this is an equation between two column vectors of size $3$, so no wonder that
it encodes a whole system of linear equations).

Why are (\ref{eq.exam.systems.1}) and (\ref{eq.exam.systems.2}) equivalent?
Well, the left hand side of (\ref{eq.exam.systems.2}) is%
\[
\left(
\begin{array}
[c]{ccc}%
3 & 6 & -1\\
7 & 4 & -3\\
0 & -1 & 8
\end{array}
\right)  \left(
\begin{array}
[c]{c}%
x\\
y\\
z
\end{array}
\right)  =\left(
\begin{array}
[c]{c}%
3x+6y+\left(  -1\right)  z\\
7x+4y+\left(  -3\right)  z\\
0x+\left(  -1\right)  y+8z
\end{array}
\right)  =\left(
\begin{array}
[c]{c}%
3x+6y-z\\
7x+4y-3z\\
-y+8z
\end{array}
\right)  .
\]
Thus, (\ref{eq.exam.systems.2}) is equivalent to%
\[
\left(
\begin{array}
[c]{c}%
3x+6y-z\\
7x+4y-3z\\
-y+8z
\end{array}
\right)  =\left(
\begin{array}
[c]{c}%
2\\
3\\
1
\end{array}
\right)  .
\]
But this is clearly equivalent to (\ref{eq.exam.systems.1}).
\end{example}

More generally:

\begin{proposition}
\label{prop.systems.Ax=b}The system of $m$ linear equations
\[
\left\{
\begin{array}
[c]{c}%
a_{1,1}x_{1}+a_{1,2}x_{2}+\cdots+a_{1,n}x_{n}=b_{1};\\
a_{2,1}x_{1}+a_{2,2}x_{2}+\cdots+a_{2,n}x_{n}=b_{2};\\
\vdots\\
a_{m,1}x_{1}+a_{m,2}x_{2}+\cdots+a_{m,n}x_{n}=b_{m}%
\end{array}
\right.
\]
in $n$ unknowns $x_{1},x_{2},\ldots,x_{n}$ is equivalent to the vector
equation%
\begin{equation}
\left(
\begin{array}
[c]{cccc}%
a_{1,1} & a_{1,2} & \cdots & a_{1,n}\\
a_{2,1} & a_{2,2} & \cdots & a_{2,n}\\
\vdots & \vdots & \ddots & \vdots\\
a_{m,1} & a_{m,2} & \cdots & a_{m,n}%
\end{array}
\right)  \left(
\begin{array}
[c]{c}%
x_{1}\\
x_{2}\\
\vdots\\
x_{n}%
\end{array}
\right)  =\left(
\begin{array}
[c]{c}%
b_{1}\\
b_{2}\\
\vdots\\
b_{m}%
\end{array}
\right)  . \label{eq.exam.systems.gen}%
\end{equation}
In other words, it is equivalent to the vector equation%
\begin{equation}
Ax=b, \label{eq.exam.systems.gen.short}%
\end{equation}
where%
\begin{align*}
A  &  =\left(
\begin{array}
[c]{cccc}%
a_{1,1} & a_{1,2} & \cdots & a_{1,n}\\
a_{2,1} & a_{2,2} & \cdots & a_{2,n}\\
\vdots & \vdots & \ddots & \vdots\\
a_{m,1} & a_{m,2} & \cdots & a_{m,n}%
\end{array}
\right)  ,\\
x  &  =\left(
\begin{array}
[c]{c}%
x_{1}\\
x_{2}\\
\vdots\\
x_{n}%
\end{array}
\right)  ,\ \ \ \ \ \ \ \ \ \ b=\left(
\begin{array}
[c]{c}%
b_{1}\\
b_{2}\\
\vdots\\
b_{m}%
\end{array}
\right)  .
\end{align*}
(Some authors write the $A$, the $x$ and the $b$ in
(\ref{eq.exam.systems.gen.short}) in boldface in order to stress that these
are matrices, not numbers. We shall not.) The matrix $A$ and the vector $b$
are known; the vector $x$ is what we want to find.
\end{proposition}

Thus, matrices give us a way to rewrite systems of linear equations as single
equations between vectors. Moreover, as we will see, they give us a way to
manipulate these equations easily.

To solve a vector equation like (\ref{eq.exam.systems.gen.short}) means (in
some sense) to \textquotedblleft undo\textquotedblright\ a matrix
multiplication. In fact, if we could divide by a matrix, then we could
immediately solve $Ax=b$ by \textquotedblleft dividing by $A$%
\textquotedblright. Unfortunately, we cannot divide by a matrix in general.
But the idea is fruitful: In fact, some matrices $A$ are invertible (i.e.,
have an inverse $A^{-1}$), and for those matrices, we can transform $Ax=b$
into $x=A^{-1}b$, which gives us an explicit and unique solution for the
system (\ref{eq.exam.systems.gen}). This doesn't work for all $A$ (since not
all $A$ are invertible), and is not a very practical way of solving systems of
linear equations; but the notion of invertible matrices is rather important,
so we begin by studying them.

\subsection{Inverse matrices}

\begin{definition}
\label{def.inverse-matrix}Let $n\in\mathbb{N}$ and $m\in\mathbb{N}$. Let $A$
be an $n\times m$-matrix, and $B$ be an $m\times n$-matrix.

\textbf{(a)} We say that $B$ is a \textit{right inverse} of $A$ if $AB=I_{n}$.

\textbf{(b)} We say that $B$ is a \textit{left inverse} of $A$ if $BA=I_{m}$.

\textbf{(c)} We say that $B$ is an \textit{inverse} of $A$ if both $AB=I_{n}$
and $BA=I_{m}$.
\end{definition}

Notice that we are saying \textquotedblleft\textbf{a} right
inverse\textquotedblright\ (not \textquotedblleft\textbf{the} right
inverse\textquotedblright) in Definition \ref{def.inverse-matrix}, because a
given matrix $A$ can have several right inverses (but it can also have no
right inverses at all). For the same reason, we are saying \textquotedblleft%
\textbf{a} left inverse\textquotedblright\ (not \textquotedblleft\textbf{the}
left inverse\textquotedblright). However, when we are saying \textquotedblleft%
\textbf{an} inverse\textquotedblright\ (not \textquotedblleft\textbf{the}
inverse\textquotedblright), we are just being cautious: We will later (in
Corollary \ref{cor.inverses.unique}) see that $A$ can never have several
different inverses; thus, it would be legitimate to say \textquotedblleft%
\textbf{the} inverse\textquotedblright\ as well. But as long as we have not
proven this, we shall speak of \textquotedblleft\textbf{an}
inverse\textquotedblright.

\begin{example}
\label{exam.inverses}\textbf{(a)} Let $A=\left(  1,4\right)  $. (Recall that
this means the $1\times2$-matrix $\left(
\begin{array}
[c]{cc}%
1 & 4
\end{array}
\right)  $.) When is a matrix $B$ a right inverse of $A$ ?

First, if $B$ is a right inverse of $A$, then $B$ must be a $2\times1$-matrix
(since any right inverse of an $n\times m$-matrix has to be an $m\times
n$-matrix). So let us assume that $B$ is a $2\times1$-matrix. Thus, $B$ must
have the form $B=\left(
\begin{array}
[c]{c}%
u\\
v
\end{array}
\right)  $ for some numbers $u$ and $v$. Then, $AB=\left(
\begin{array}
[c]{cc}%
1 & 4
\end{array}
\right)  \left(
\begin{array}
[c]{c}%
u\\
v
\end{array}
\right)  =\left(
\begin{array}
[c]{c}%
1u+4v
\end{array}
\right)  $. In order for $B$ to be a right inverse of $A$, it is necessary and
sufficient that $AB=I_{1}$ (because this is how we defined \textquotedblleft
right inverse\textquotedblright). In other words, we must have $\left(
\begin{array}
[c]{c}%
1u+4v
\end{array}
\right)  =\left(
\begin{array}
[c]{c}%
1
\end{array}
\right)  $ (since $AB=\left(
\begin{array}
[c]{c}%
1u+4v
\end{array}
\right)  $ and $I_{1}=\left(
\begin{array}
[c]{c}%
1
\end{array}
\right)  $). In other words, we must have $1u+4v=1$.

Hence, a matrix $B$ is a right inverse of $A$ if and only if it has the form
$B=\left(
\begin{array}
[c]{c}%
u\\
v
\end{array}
\right)  $ for some numbers $u$ and $v$ satisfying $1u+4v=1$. How do we find
two such numbers $u$ and $v$ ? Well, we can view $1u+4v=1$ as a system of $1$
linear equation in $2$ variables, but actually we can just read off the
solution: $v$ can be chosen arbitrarily, and $u$ then has to be $1-4v$. Hence,
a matrix $B$ is a right inverse of $A$ if and only if it has the form
$B=\left(
\begin{array}
[c]{c}%
1-4v\\
v
\end{array}
\right)  $ for some number $v$. In particular, there are \textbf{infinitely
many} matrices $B$ that are right inverses of $A$ (because we have full
freedom in choosing $v$).

\textbf{(b)} Let $A=\left(  1,4\right)  $ again. When is a matrix $B$ a left
inverse of $A$ ?

Again, $B$ must be a $2\times1$-matrix in order for this to have any chance of
being true. So let us assume that $B$ is a $2\times1$-matrix, and write $B$ in
the form $B=\left(
\begin{array}
[c]{c}%
u\\
v
\end{array}
\right)  $ for some numbers $u$ and $v$. Then, $BA=\left(
\begin{array}
[c]{c}%
u\\
v
\end{array}
\right)  \left(  1,4\right)  =\left(
\begin{array}
[c]{cc}%
u\cdot1 & u\cdot4\\
v\cdot1 & v\cdot4
\end{array}
\right)  $. In order for $B$ to be a left inverse of $A$, it is necessary and
sufficient that $BA=I_{2}$ (because this is how we defined \textquotedblleft
left inverse\textquotedblright). In other words, we must have $\left(
\begin{array}
[c]{cc}%
u\cdot1 & u\cdot4\\
v\cdot1 & v\cdot4
\end{array}
\right)  =\left(
\begin{array}
[c]{cc}%
1 & 0\\
0 & 1
\end{array}
\right)  $ (since $BA=\left(
\begin{array}
[c]{cc}%
u\cdot1 & u\cdot4\\
v\cdot1 & v\cdot4
\end{array}
\right)  $ and $I_{2}=\left(
\begin{array}
[c]{cc}%
1 & 0\\
0 & 1
\end{array}
\right)  $). In other words, we must have%
\[
\left\{
\begin{array}
[c]{c}%
u\cdot1=1;\\
u\cdot4=0;\\
v\cdot1=0;\\
v\cdot4=1
\end{array}
\right.  .
\]
But this cannot happen! Indeed, the equations $u\cdot1=1$ and $u\cdot4=0$
contradict each other (because if $u\cdot1=1$, then $u\cdot4$ must be $4$).
Hence, $B$ can never be a left inverse of $A$. In other words, the matrix $A$
\textbf{has no} left inverse.

\textbf{(c)} Let $A=\left(
\begin{array}
[c]{c}%
1\\
4
\end{array}
\right)  $. When is a matrix $B$ a left inverse of $A$ ? When is a matrix $B$
a right inverse of $A$ ? I will let you figure this out (see Exercise
\ref{exe.exam.inverses} below).

\textbf{(d)} Let $A=\left(
\begin{array}
[c]{cc}%
1 & -1\\
1 & 1
\end{array}
\right)  $. When is a matrix $B$ a left inverse of $A$ ? When is a matrix $B$
a right inverse of $A$ ?

A left inverse of $A$ would have to be a $2\times2$-matrix. A $2\times
2$-matrix $B=\left(
\begin{array}
[c]{cc}%
x & y\\
z & w
\end{array}
\right)  $ is a left inverse of $A$ if and only if it satisfies $BA=I_{2}$,
that is, $\left(
\begin{array}
[c]{cc}%
x & y\\
z & w
\end{array}
\right)  \left(
\begin{array}
[c]{cc}%
1 & -1\\
1 & 1
\end{array}
\right)  =\left(
\begin{array}
[c]{cc}%
1 & 0\\
0 & 1
\end{array}
\right)  $, or, equivalently, $\left(
\begin{array}
[c]{cc}%
x+y & -x+y\\
w+z & w-z
\end{array}
\right)  =\left(
\begin{array}
[c]{cc}%
1 & 0\\
0 & 1
\end{array}
\right)  $, or, equivalently,%
\[
\left\{
\begin{array}
[c]{c}%
x+y=1;\\
-x+y=0;\\
w+z=0;\\
w-z=1
\end{array}
\right.  .
\]
This is a system of four linear equations in the four unknowns $x,y,z,w$; it
has the unique solution%
\[
\left(  x,y,z,w\right)  =\left(  \dfrac{1}{2},\dfrac{1}{2},-\dfrac{1}%
{2},\dfrac{1}{2}\right)  .
\]
Thus, a $2\times2$-matrix $B=\left(
\begin{array}
[c]{cc}%
x & y\\
z & w
\end{array}
\right)  $ is a left inverse of $A$ if and only if $\left(  x,y,z,w\right)
=\left(  \dfrac{1}{2},\dfrac{1}{2},-\dfrac{1}{2},\dfrac{1}{2}\right)  $.
Hence, there exists exactly one left inverse of $A$, and this left inverse is
$\left(
\begin{array}
[c]{cc}%
\dfrac{1}{2} & \dfrac{1}{2}\\
-\dfrac{1}{2} & \dfrac{1}{2}%
\end{array}
\right)  $.

A similar computation reveals that there exists exactly one right inverse of
$A$, and this right inverse is $\left(
\begin{array}
[c]{cc}%
\dfrac{1}{2} & \dfrac{1}{2}\\
-\dfrac{1}{2} & \dfrac{1}{2}%
\end{array}
\right)  $. So the unique left inverse of $A$ and the unique right inverse of
$A$ are actually equal (and thus are an inverse of $A$). This might not be
clear from the definitions, but as we shall soon see, this is not a coincidence.

\textbf{(e)} Generalizing Example \ref{exam.inverses} \textbf{(d)}, we might
wonder when a $2\times2$-matrix $A=\left(
\begin{array}
[c]{cc}%
a & b\\
c & d
\end{array}
\right)  $ has a left inverse, a right inverse or an inverse. For any given
four values of $a,b,c,d$, we can answer this question similarly to how we
answered it for the matrix $A=\left(
\begin{array}
[c]{cc}%
1 & -1\\
1 & 1
\end{array}
\right)  $ in Example \ref{exam.inverses} \textbf{(d)} (by solving a system of
linear equations). The procedure will depend on whether some numbers are zero
or not. (For example, we might want to divide the equation $bx+dy=0$ by $b$,
which requires $b\neq0$; the case $b=0$ will then have to be treated
separately.) But the final result will be the following:

\begin{itemize}
\item If $ad-bc=0$, then the matrix $A$ has no left inverses and no right inverses.

\item If $ad-bc\neq0$, then the matrix $A$ has a unique inverse, which is also
the unique left inverse and the unique right inverse. This inverse is $\left(
%
\begin{array}
[c]{cc}%
\dfrac{d}{ad-bc} & -\dfrac{b}{ad-bc}\\
-\dfrac{c}{ad-bc} & \dfrac{a}{ad-bc}%
\end{array}
\right)  $.
\end{itemize}

Again, this phenomenon of the left inverse equalling the right inverse
appears. Notably, the number $ad-bc$ plays an important role here; we will
later see more of it (it is an example of a \textit{determinant}).
\end{example}

\begin{exercise}
\label{exe.exam.inverses}Let $A=\left(
\begin{array}
[c]{c}%
1\\
4
\end{array}
\right)  $. When is a matrix $B$ a left inverse of $A$ ? When is a matrix $B$
a right inverse of $A$ ?
\end{exercise}

As we know from Example \ref{exam.inverses} \textbf{(a)}, a matrix may have
infinitely many right inverses. Similarly, a matrix may have infinitely many
left inverses. But can a matrix have both infinitely many right inverses and
infinitely many left inverses at the same time? The answer is
\textquotedblleft no\textquotedblright, and in fact, something stronger is true:

\begin{proposition}
\label{prop.inverses.L=R}Let $n\in\mathbb{N}$ and $m\in\mathbb{N}$. Let $A$ be
an $n\times m$-matrix. Let $L$ be a left inverse of $A$. Let $R$ be a right
inverse of $A$. Then:

\textbf{(a)} We have $L=R$.

\textbf{(b)} The matrix $L$ is the only left inverse of $A$.

\textbf{(c)} The matrix $R$ is the only right inverse of $A$.

\textbf{(d)} The matrix $L=R$ is the only inverse of $A$.
\end{proposition}

\begin{proof}
[Proof of Proposition \ref{prop.inverses.L=R}.]We know that $LA=I_{m}$ (since
$L$ is a left inverse of $A$) and that $AR=I_{n}$ (since $R$ is a right
inverse of $A$).

\textbf{(a)} Consider the product $LAR$. (Recall that this product is
well-defined, because Proposition \ref{prop.matrix-laws.1} \textbf{(g)} yields
$L\left(  AR\right)  =\left(  LA\right)  R$.)

One way to rewrite $LAR$ is as follows:%
\begin{equation}
L\underbrace{AR}_{=I_{n}}=LI_{n}=L\ \ \ \ \ \ \ \ \ \ \left(  \text{by
Proposition \ref{prop.matrix-laws.id} \textbf{(b)}}\right)  .
\label{pf.prop.inverses.L=R.1}%
\end{equation}
Another way is%
\begin{equation}
\underbrace{LA}_{=I_{m}}R=I_{m}R=R\ \ \ \ \ \ \ \ \ \ \left(  \text{by
Proposition \ref{prop.matrix-laws.id} \textbf{(a)}}\right)  .
\label{pf.prop.inverses.L=R.2}%
\end{equation}
Comparing (\ref{pf.prop.inverses.L=R.1}) with (\ref{pf.prop.inverses.L=R.2}),
we obtain $L=R$. This proves Proposition \ref{prop.inverses.L=R} \textbf{(a)}.

\textbf{(b)} Let $L^{\prime}$ be any left inverse of $A$. Then, we can apply
Proposition \ref{prop.inverses.L=R} \textbf{(a)} to $L^{\prime}$ instead of
$L$ (because all that was needed from $L$ in Proposition
\ref{prop.inverses.L=R} \textbf{(a)} was that it be a left inverse of $A$). As
a result, we obtain $L^{\prime}=R$.

Now, forget that we fixed $L^{\prime}$. We thus have shown that if $L^{\prime
}$ is any left inverse of $A$, then $L^{\prime}=R$. In other words, any left
inverse of $A$ equals $R$. Thus, there exists at most one left inverse of $A$.
Therefore, the matrix $L$ is the only left inverse of $A$ (since we already
know that $L$ is a left inverse of $A$). This proves Proposition
\ref{prop.inverses.L=R} \textbf{(b)}.

\textbf{(c)} The proof of Proposition \ref{prop.inverses.L=R} \textbf{(c)} is
analogous to the proof of Proposition \ref{prop.inverses.L=R} \textbf{(b)}.
(We again need to apply Proposition \ref{prop.inverses.L=R} \textbf{(a)}, but
this time, instead of a left inverse $L^{\prime}$, we have to introduce a
right inverse $R^{\prime}$. The details are left to the reader.)

\textbf{(d)} Proposition \ref{prop.inverses.L=R} \textbf{(a)} yields $L=R$.
Hence, $A\underbrace{L}_{=R}=AR=I_{n}$.

Now, the matrix $L$ is an inverse of $A$ (since $LA=I_{m}$ and $AL=I_{n}$). In
other words, the matrix $L=R$ is an inverse of $A$ (since $L=R$). It remains
to show that it is the only inverse of $A$. But this is easy: Let $L^{\prime}$
be any inverse of $A$. Then, $L^{\prime}A=I_{m}$, so that $L^{\prime}$ is a
left inverse of $A$. Proposition \ref{prop.inverses.L=R} \textbf{(a)} (applied
to $L^{\prime}$ instead of $L$) therefore yields $L^{\prime}=R$.

Now, forget that we fixed $L^{\prime}$. We thus have shown that if $L^{\prime
}$ is any inverse of $A$, then $L^{\prime}=R$. In other words, any inverse of
$A$ equals $R$. Thus, there exists at most one inverse of $A$. Therefore, the
matrix $L$ is the only inverse of $A$ (since we already know that $L$ is an
inverse of $A$). This proves Proposition \ref{prop.inverses.L=R} \textbf{(d)}.
\end{proof}

\begin{corollary}
\label{cor.inverses.unique}Let $A$ be a matrix. Then, $A$ has at most one inverse.
\end{corollary}

\begin{proof}
[Proof of Corollary \ref{cor.inverses.unique}.]We need to show that any two
inverses of $A$ are equal. So let $L$ and $R$ be two inverses of $A$. We must
show that $L=R$.

Let $n\in\mathbb{N}$ and $m\in\mathbb{N}$ be such that $A$ is an $n\times
m$-matrix. The matrix $L$ is an inverse of $A$, thus satisfies $LA=I_{m}$.
Hence, $L$ is a left inverse of $A$. Also, the matrix $R$ is an inverse of
$A$, thus satisfies $AR=I_{n}$. Hence, $R$ is a right inverse of $A$. Thus,
Proposition \ref{prop.inverses.L=R} \textbf{(a)} shows that $L=R$. This proves
Corollary \ref{cor.inverses.unique}.
\end{proof}

\begin{definition}
\textbf{(a)} A matrix $A$ is said to be \textit{invertible} if it has an
inverse. (Similarly, we can define the words \textquotedblleft
left-invertible\textquotedblright\ and \textquotedblleft
right-invertible\textquotedblright.)

\textbf{(b)} Let $A$ be an invertible matrix. Then, $A$ has an inverse. Due to
Corollary \ref{cor.inverses.unique}, we furthermore know that $A$ has at most
one inverse. Thus, $A$ has exactly one inverse. We can thus refer to this
inverse as \textquotedblleft\textbf{the} inverse of $A$\textquotedblright%
\ (not just \textquotedblleft\textbf{an} inverse of $A$\textquotedblright),
and denote it by $A^{-1}$. If $A$ is an $n\times m$-matrix, then this inverse
satisfies $A^{-1}A=I_{m}$ and $AA^{-1}=I_{n}$ (by its definition).
\end{definition}

Notice that the equalities $A^{-1}A=I_{m}$ and $AA^{-1}=I_{n}$ show that a
matrix $A$ and its inverse $A^{-1}$ cancel each other when they stand adjacent
in a product: for example, $BA^{-1}AC$ simplifies to $BC$. However, they do
not (generally) cancel each other when they appear apart from one another: for
example, $BA^{-1}CA$ does \textbf{not} simplify to $BC$.

So what matrices are invertible? The following theorem significantly narrows
the search down; we shall not prove it until later:

\begin{theorem}
\label{thm.invertible.size}Let $n\in\mathbb{N}$ and $m\in\mathbb{N}$. Let $A$
be an $n\times m$-matrix.

\textbf{(a)} If $A$ has a right inverse, then $n\leq m$ (that is, the matrix
$A$ has at least as many columns as it has rows).

\textbf{(b)} If $A$ has a left inverse, then $n\geq m$ (that is, the matrix
$A$ has at most as many columns as it has rows).

\textbf{(c)} If $A$ is invertible (i.e., has an inverse), then $n=m$ (that is,
the matrix $A$ is square).

\textbf{(d)} If $A$ is square (that is, $n=m$) and has a left inverse
\textbf{or} a right inverse, then $A$ is actually invertible (and so this left
or right inverse is the inverse of $A$). Notice that this is false for
rectangular matrices!
\end{theorem}

Let us now check some simpler facts about inverses:

\begin{proposition}
\label{prop.inverses.In}Let $n\in\mathbb{N}$. Then, the matrix $I_{n}$ is
invertible, and its inverse is $\left(  I_{n}\right)  ^{-1}=I_{n}$.
\end{proposition}

\begin{proof}
[Proof of Proposition \ref{prop.inverses.In}.]We have $I_{n}I_{n}=I_{n}$ and
$I_{n}I_{n}=I_{n}$. Hence, the matrix $I_{n}$ is an inverse of $I_{n}$ (by the
definition of \textquotedblleft inverse\textquotedblright). This proves
Proposition \ref{prop.inverses.In}.
\end{proof}

\begin{proposition}
\label{prop.inverses.AB}Let $A$ and $B$ be two invertible matrices such that
the product $AB$ is well-defined (i.e., such that $A$ has as many columns as
$B$ has rows). Then, the matrix $AB$ is also invertible, and its inverse is
\begin{equation}
\left(  AB\right)  ^{-1}=B^{-1}A^{-1}. \label{eq.prop.inverses.AB.eq}%
\end{equation}

\end{proposition}

\begin{proof}
[Proof of Proposition \ref{prop.inverses.AB}.]Let $n$, $m$ and $p$ be
nonnegative integers such that $A$ is an $n\times m$-matrix and $B$ is an
$m\times p$-matrix\footnote{We can indeed find such $n$, $m$ and $p$ because
$A$ has as many columns as $B$ has rows.}. (Actually, Theorem
\ref{thm.invertible.size} \textbf{(c)} reveals that the matrices $A$ and $B$
are square and therefore $n=m=p$; but I do not want to use Theorem
\ref{thm.invertible.size} \textbf{(c)} here, since I have not yet proven it.)

Recall once again that (by general associativity) products of matrices can be
written without parentheses. Thus, for example, the products $B^{-1}A^{-1}AB$
and $ABB^{-1}A^{-1}$ make sense. Let us simplify these products:%
\[
B^{-1}\underbrace{A^{-1}A}_{=I_{m}}B=B^{-1}I_{m}B=B^{-1}B=I_{p}%
\]
and%
\[
A\underbrace{BB^{-1}}_{=I_{m}}A^{-1}=AI_{m}A^{-1}=AA^{-1}=I_{n}.
\]
But these two equalities say precisely that $B^{-1}A^{-1}$ is an inverse of
$AB$. (If you don't believe me, rewrite them with parentheses: $\left(
B^{-1}A^{-1}\right)  \left(  AB\right)  =I_{p}$ and $\left(  AB\right)
\left(  B^{-1}A^{-1}\right)  =I_{n}$.) In particular, this shows that $AB$ is
invertible. This proves Proposition \ref{prop.inverses.AB}.
\end{proof}

In words, (\ref{eq.prop.inverses.AB.eq}) says that the inverse of a product of
two matrices is the product of their inverses, but \textbf{in opposite order}.
This takes some getting used to, but is really a natural thing; the same rule
holds for inverting the composition of functions\footnote{Namely: If $X$, $Y$
and $Z$ are three sets, and if $b:X\rightarrow Y$ and $a:Y\rightarrow Z$ are
two invertible functions (i.e., bijections), then $a\circ b:X\rightarrow Z$ is
an invertible function as well, and its inverse is $\left(  a\circ b\right)
^{-1}=b^{-1}\circ a^{-1}$. (Some authors liken this to the fact that if you
want to undo the process of putting on socks and then putting on shoes, you
have to first take off your shoes and then take off your socks. See
\url{https://proofwiki.org/wiki/Inverse_of_Product} .)}.

\begin{proposition}
\label{prop.inverses.A1Ak}Let $A_{1},A_{2},\ldots,A_{k}$ be $k$ invertible
matrices (where $k$ is a positive integer) such that the product $A_{1}%
A_{2}\cdots A_{k}$ is well-defined (i.e., such that $A_{i}$ has as many
columns as $A_{i+1}$ has rows, for each $i<k$). Then, the matrix $A_{1}%
A_{2}\cdots A_{k}$ is invertible, and its inverse is%
\[
\left(  A_{1}A_{2}\cdots A_{k}\right)  ^{-1}=A_{k}^{-1}A_{k-1}^{-1}\cdots
A_{1}^{-1}.
\]

\end{proposition}

Proposition \ref{prop.inverses.A1Ak} is a natural extension of Proposition
\ref{prop.inverses.AB} to products of more than $2$ matrices. The proof of
Proposition \ref{prop.inverses.A1Ak} is straightforward, and I am only showing
it as an example of proof by induction:

\begin{proof}
[Proof of Proposition \ref{prop.inverses.A1Ak}.]We prove Proposition
\ref{prop.inverses.A1Ak} by induction on $k$:

\textit{Induction base:} If $k=1$, then Proposition \ref{prop.inverses.A1Ak}
says that $A_{1}^{-1}=A_{1}^{-1}$; this is obviously true. Hence, Proposition
\ref{prop.inverses.A1Ak} holds for $k=1$. This completes the induction base.

\textit{Induction step:} Let $\ell$ be a positive integer. Assume (as our
\textit{induction hypothesis}) that Proposition \ref{prop.inverses.A1Ak} holds
for $k=\ell$. In other words, for any $\ell$ invertible matrices $A_{1}%
,A_{2},\ldots,A_{\ell}$ for which the product $A_{1}A_{2}\cdots A_{\ell}$ is
well-defined, the matrix $A_{1}A_{2}\cdots A_{\ell}$ is invertible, and its
inverse is%
\[
\left(  A_{1}A_{2}\cdots A_{\ell}\right)  ^{-1}=A_{\ell}^{-1}A_{\ell-1}%
^{-1}\cdots A_{1}^{-1}.
\]


We must now show that Proposition \ref{prop.inverses.A1Ak} also holds for
$k=\ell+1$. So let us fix $\ell+1$ invertible matrices $A_{1},A_{2}%
,\ldots,A_{\ell+1}$ for which the product $A_{1}A_{2}\cdots A_{\ell+1}$ is
well-defined. We must then show that the matrix $A_{1}A_{2}\cdots A_{\ell+1}$
is invertible, and that its inverse is
\[
\left(  A_{1}A_{2}\cdots A_{\ell+1}\right)  ^{-1}=A_{\ell+1}^{-1}A_{\ell}%
^{-1}\cdots A_{1}^{-1}.
\]


The product $A_{1}A_{2}\cdots A_{\ell}$ is well-defined (since the product
$A_{1}A_{2}\cdots A_{\ell+1}$ is well-defined). Hence, we can apply our
induction hypothesis, and conclude that the matrix $A_{1}A_{2}\cdots A_{\ell}$
is invertible, and its inverse is%
\[
\left(  A_{1}A_{2}\cdots A_{\ell}\right)  ^{-1}=A_{\ell}^{-1}A_{\ell-1}%
^{-1}\cdots A_{1}^{-1}.
\]


Now, the matrices $A_{1}A_{2}\cdots A_{\ell}$ and $A_{\ell+1}$ are invertible,
and their product \newline$\left(  A_{1}A_{2}\cdots A_{\ell}\right)
A_{\ell+1}=A_{1}A_{2}\cdots A_{\ell+1}$ is well-defined (by assumption).
Hence, Proposition \ref{prop.inverses.AB} (applied to $A=A_{1}A_{2}\cdots
A_{\ell}$ and $B=A_{\ell+1}$) shows that the matrix $\left(  A_{1}A_{2}\cdots
A_{\ell}\right)  A_{\ell+1}$ is also invertible, and its inverse is
\[
\left(  \left(  A_{1}A_{2}\cdots A_{\ell}\right)  A_{\ell+1}\right)
^{-1}=A_{\ell+1}^{-1}\left(  A_{1}A_{2}\cdots A_{\ell}\right)  ^{-1}.
\]
Since $\left(  A_{1}A_{2}\cdots A_{\ell}\right)  A_{\ell+1}=A_{1}A_{2}\cdots
A_{\ell+1}$ and \newline$A_{\ell+1}^{-1}\underbrace{\left(  A_{1}A_{2}\cdots
A_{\ell}\right)  ^{-1}}_{=A_{\ell}^{-1}A_{\ell-1}^{-1}\cdots A_{1}^{-1}%
}=A_{\ell+1}^{-1}\left(  A_{\ell}^{-1}A_{\ell-1}^{-1}\cdots A_{1}^{-1}\right)
=A_{\ell+1}^{-1}A_{\ell}^{-1}\cdots A_{1}^{-1}$, this rewrites as follows: The
matrix $A_{1}A_{2}\cdots A_{\ell+1}$ is invertible, and its inverse is
\[
\left(  A_{1}A_{2}\cdots A_{\ell+1}\right)  ^{-1}=A_{\ell+1}^{-1}A_{\ell}%
^{-1}\cdots A_{1}^{-1}.
\]
This is precisely what we wanted to show! Thus, Proposition
\ref{prop.inverses.A1Ak} holds for $k=\ell+1$. This completes the induction
step. Thus, Proposition \ref{prop.inverses.A1Ak} is proven by induction.

(I have written up this proof with a lot of detail. You do not have to! If you
are used to mathematical induction, then you can easily afford omitting many
of the incantations I made above, and taking certain shortcuts -- for example,
instead of introducing a new variable $\ell$ in the induction step, you could
reuse $k$, thus stepping \textquotedblleft from $k$ to $k+1$\textquotedblright%
\ instead of \textquotedblleft from $k=\ell$ to $k=\ell+1$\textquotedblright.
You also don't need to formally state the induction hypothesis, because it is
just a copy of the claim (with $k$ replaced by $\ell$ in our case). Finally,
what we did in our proof was obvious enough that you could just say that
\textquotedblleft Proposition \ref{prop.inverses.A1Ak} follows by a
straightforward induction on $k$, where Proposition \ref{prop.inverses.AB} is
being applied in the induction step\textquotedblright, and declare the proof finished.)
\end{proof}

\begin{remark}
\label{rmk.prop.inverses.A1Ak.k=0}It is common to define the product of $0$
square matrices of size $n\times n$ (an \textquotedblleft empty product of
$n\times n$-matrices\textquotedblright) as the identity matrix $I_{n}$
(similarly to how a product of $0$ numbers is defined to be $1$). With this
convention, Proposition \ref{prop.inverses.A1Ak} holds for $k=0$ too (it then
states that $\left(  I_{n}\right)  ^{-1}=I_{n}$), as long as we agree what
size our non-existing matrices are considered to have (it has to be $n\times
n$ for some $n\in\mathbb{N}$). With this convention, we could have started our
induction (in the above proof of Proposition \ref{prop.inverses.A1Ak}) at
$k=0$ instead of $k=1$.
\end{remark}

\begin{corollary}
\label{cor.inverses.Ak-1}Let $n\in\mathbb{N}$. Let $k\in\mathbb{N}$. Let $A$
be an invertible $n\times n$-matrix. Then, $A^{k}$ is also invertible, and its
inverse is $\left(  A^{k}\right)  ^{-1}=\left(  A^{-1}\right)  ^{k}$.
\end{corollary}

Note that Corollary \ref{cor.inverses.Ak-1} is not obvious! You cannot argue
that $\left(  A^{k}\right)  ^{-1}=\left(  A^{-1}\right)  ^{k}$ because both
sides simplify to $A^{-k}$; this argument makes no sense unless you have
defined $A^{-k}$ (and we have not defined $A^{-k}$) and proved that standard
rules of exponentiation (such as $\left(  A^{u}\right)  ^{v}=A^{uv}$) apply to matrices.

\begin{proof}
[Proof of Corollary \ref{cor.inverses.Ak-1}.]Recall that $A^{0}=I_{n}$ (by the
definition of $A^{0}$). Hence, in the case when $k=0$, Corollary
\ref{cor.inverses.Ak-1} says that $I_{n}$ is invertible, and its inverse is
$\left(  I_{n}\right)  ^{-1}=I_{n}$. This follows from Proposition
\ref{prop.inverses.In}. Thus, Corollary \ref{cor.inverses.Ak-1} is proven in
the case when $k=0$. Therefore, we can WLOG\footnote{\textquotedblleft
WLOG\textquotedblright\ is shorthand for \textquotedblleft without loss of
generality\textquotedblright. See, for example,
\href{https://en.wikipedia.org/wiki/Without_loss_of_generality}{the Wikipedia
article for \textquotedblleft WLOG\textquotedblright} (or any book on
mathematical proofs) for the meaning of this phrase.
\par
(As far as the proof of Corollary \ref{cor.inverses.Ak-1} is concerned, the
meaning of \textquotedblleft we can WLOG assume that $k\neq0$%
\textquotedblright\ is the following: \textquotedblleft If we can prove
Corollary \ref{cor.inverses.Ak-1} for $k\neq0$, then we know how to obtain a
proof of Corollary \ref{cor.inverses.Ak-1} for all $k$ (because Corollary
\ref{cor.inverses.Ak-1} is already proven in the case when $k=0$). Thus, it
will suffice to prove Corollary \ref{cor.inverses.Ak-1} for $k\neq0$; hence,
let us assume that $k\neq0$.\textquotedblright)} assume that $k\neq0$. Assume
this. Thus, $k$ is a positive integer. Hence, $A^{k}=\underbrace{AA\cdots
A}_{k\text{ times}}$ and $\left(  A^{-1}\right)  ^{k}=\underbrace{A^{-1}%
A^{-1}\cdots A^{-1}}_{k\text{ times}}$.

Proposition \ref{prop.inverses.A1Ak} (applied to $A,A,\ldots,A$ instead of
$A_{1},A_{2},\ldots,A_{k}$) shows that the matrix $\underbrace{AA\cdots
A}_{k\text{ times}}$ is invertible, and its inverse is%
\[
\left(  \underbrace{AA\cdots A}_{k\text{ times}}\right)  ^{-1}%
=\underbrace{A^{-1}A^{-1}\cdots A^{-1}}_{k\text{ times}}.
\]
In other words, the matrix $A^{k}$ is invertible, and its inverse is $\left(
A^{k}\right)  ^{-1}=\left(  A^{-1}\right)  ^{k}$. This proves Corollary
\ref{cor.inverses.Ak-1}.
\end{proof}

\begin{proposition}
\label{prop.inverses.A-1-1}Let $n\in\mathbb{N}$. Let $A$ be an invertible
$n\times n$-matrix. Then, its inverse $A^{-1}$ is also invertible, and has the
inverse $\left(  A^{-1}\right)  ^{-1}=A$.
\end{proposition}

\begin{proof}
[Proof of Proposition \ref{prop.inverses.A-1-1}.]Since $A^{-1}$ is an inverse
of $A$, we have the two equalities $A^{-1}A=I_{n}$ and $AA^{-1}=I_{n}$. But
these very same equalities show that $A$ is an inverse of $A^{-1}$ (if you do
not trust me, just check with the definition of \textquotedblleft
inverse\textquotedblright). Thus, the matrix $A^{-1}$ is invertible, and its
inverse is $\left(  A^{-1}\right)  ^{-1}=A$. Proposition
\ref{prop.inverses.A-1-1} is proven.
\end{proof}

\begin{proposition}
\label{prop.inverses.lA}Let $n\in\mathbb{N}$. Let $\lambda$ be a nonzero
number. Let $A$ be an invertible $n\times n$-matrix. Then, the matrix $\lambda
A$ is also invertible, and its inverse is $\left(  \lambda A\right)
^{-1}=\lambda^{-1}A^{-1}=\dfrac{1}{\lambda}A^{-1}$.
\end{proposition}

\begin{exercise}
\label{exe.prop.inverses.lA}Prove Proposition \ref{prop.inverses.lA}.
\end{exercise}

\subsection{More on transposes}

How do the matrix operations we have seen above (addition, multiplication,
inversion, etc.) behave with respect to transposes? The answer is
\textquotedblleft fairly nicely\textquotedblright:

\begin{proposition}
\label{prop.transpose.opers}\textbf{(a)} Let $n\in\mathbb{N}$. Then, $\left(
I_{n}\right)  ^{T}=I_{n}$.

\textbf{(b)} Let $n\in\mathbb{N}$ and $m\in\mathbb{N}$. Then, $\left(
0_{n\times m}\right)  ^{T}=0_{m\times n}$.

\textbf{(c)} Let $n\in\mathbb{N}$ and $m\in\mathbb{N}$. Let $A$ be an $n\times
m$-matrix. Let $\lambda$ be a number. Then, $\left(  \lambda A\right)
^{T}=\lambda A^{T}$.

\textbf{(d)} Let $n\in\mathbb{N}$ and $m\in\mathbb{N}$. Let $A$ and $B$ be two
$n\times m$-matrices. Then, $\left(  A+B\right)  ^{T}=A^{T}+B^{T}$.

\textbf{(e)} Let $n\in\mathbb{N}$, $m\in\mathbb{N}$ and $p\in\mathbb{N}$. Let
$A$ be an $n\times m$-matrix. Let $B$ be an $m\times p$-matrix. Then, $\left(
AB\right)  ^{T}=B^{T}A^{T}$.

\textbf{(f)} Let $n\in\mathbb{N}$. Let $A$ be an invertible $n\times
n$-matrix. Then, $A^{T}$ is invertible, and its inverse is $\left(
A^{T}\right)  ^{-1}=\left(  A^{-1}\right)  ^{T}$.
\end{proposition}

Notice that the right hand side in Proposition \ref{prop.transpose.opers}
\textbf{(e)} is $B^{T}A^{T}$, not $A^{T}B^{T}$ (in fact, $A^{T}B^{T}$ does not
always make sense, since the number of columns of $A^{T}$ is not necessarily
the number of rows of $B^{T}$). This is similar to the $B^{-1}A^{-1}$ in
Proposition \ref{prop.inverses.AB}.

Proposition \ref{prop.transpose.opers} is fairly easy to show; let us only
give a proof of part \textbf{(e)}:

\begin{proof}
[Proof of Proposition \ref{prop.transpose.opers} \textbf{(e)}.]The definition
of $A^{T}$ shows that $A^{T}=\left(  A_{j,i}\right)  _{1\leq i\leq m,\ 1\leq
j\leq n}$. Thus,%
\begin{equation}
\left(  A^{T}\right)  _{i,j}=A_{j,i}\ \ \ \ \ \ \ \ \ \ \text{for all }%
i\in\left\{  1,2,\ldots,m\right\}  \text{ and }j\in\left\{  1,2,\ldots
,n\right\}  . \label{pf.prop.transpose.opers.d.AT}%
\end{equation}
Similarly, the definition of $B^{T}$ shows that $B^{T}=\left(  B_{j,i}\right)
_{1\leq i\leq p,\ 1\leq j\leq m}$. Hence,%
\begin{equation}
\left(  B^{T}\right)  _{i,j}=B_{j,i}\ \ \ \ \ \ \ \ \ \ \text{for all }%
i\in\left\{  1,2,\ldots,p\right\}  \text{ and }j\in\left\{  1,2,\ldots
,m\right\}  . \label{pf.prop.transpose.opers.d.BT}%
\end{equation}


Now, $B^{T}$ is a $p\times m$-matrix, while $A^{T}$ is an $m\times n$-matrix.
Hence, $B^{T}A^{T}$ is a $p\times n$-matrix. Also, $\left(  AB\right)  ^{T}$
is a $p\times n$-matrix (since $AB$ is an $n\times p$-matrix). The definition
of $\left(  AB\right)  ^{T}$ shows that $\left(  AB\right)  ^{T}=\left(
\left(  AB\right)  _{j,i}\right)  _{1\leq i\leq p,\ 1\leq j\leq n}$. Hence,
\begin{align}
\left(  \left(  AB\right)  ^{T}\right)  _{i,j}  &  =\left(  AB\right)
_{j,i}\nonumber\\
&  =A_{j,1}B_{1,i}+A_{j,2}B_{2,i}+\cdots+A_{j,m}B_{m,i}%
\label{pf.prop.transpose.opers.d.1}\\
&  \ \ \ \ \ \ \ \ \ \ \left(  \text{by Proposition \ref{prop.matrix-prod.rc}
\textbf{(a)}, applied to }j\text{ and }i\text{ instead of }i\text{ and
}j\right) \nonumber
\end{align}
for all $i\in\left\{  1,2,\ldots,p\right\}  $ and $j\in\left\{  1,2,\ldots
,n\right\}  $.

On the other hand, we can apply Proposition \ref{prop.matrix-prod.rc}
\textbf{(a)} to $p$, $m$, $n$, $B^{T}$ and $A^{T}$ instead of $n$, $m$, $p$,
$A$ and $B$. We thus conclude that%
\begin{equation}
\left(  B^{T}A^{T}\right)  _{i,j}=\left(  B^{T}\right)  _{i,1}\left(
A^{T}\right)  _{1,j}+\left(  B^{T}\right)  _{i,2}\left(  A^{T}\right)
_{2,j}+\cdots+\left(  B^{T}\right)  _{i,m}\left(  A^{T}\right)  _{m,j}
\label{pf.prop.transpose.opers.d.2}%
\end{equation}
for all $i\in\left\{  1,2,\ldots,p\right\}  $ and $j\in\left\{  1,2,\ldots
,n\right\}  $.

But for every $k\in\left\{  1,2,\ldots,m\right\}  $, we have%
\begin{equation}
\underbrace{\left(  B^{T}\right)  _{i,k}}_{\substack{=B_{k,i}\\\text{(by
(\ref{pf.prop.transpose.opers.d.BT}), applied}\\\text{to }k\text{ instead of
}j\text{)}}}\underbrace{\left(  A^{T}\right)  _{k,j}}_{\substack{=A_{j,k}%
\\\text{(by (\ref{pf.prop.transpose.opers.d.AT}), applied}\\\text{to }k\text{
instead of }i\text{)}}}=B_{k,i}A_{j,k}=A_{j,k}B_{k,i}.
\label{pf.prop.transpose.opers.d.3}%
\end{equation}


Hence, for all $i\in\left\{  1,2,\ldots,p\right\}  $ and $j\in\left\{
1,2,\ldots,n\right\}  $, we have
\begin{align*}
\left(  B^{T}A^{T}\right)  _{i,j}  &  =\underbrace{\left(  B^{T}\right)
_{i,1}\left(  A^{T}\right)  _{1,j}}_{\substack{=A_{j,1}B_{1,i}\\\text{(by
(\ref{pf.prop.transpose.opers.d.3}), applied}\\\text{to }k=1\text{)}%
}}+\underbrace{\left(  B^{T}\right)  _{i,2}\left(  A^{T}\right)  _{2,j}%
}_{\substack{=A_{j,2}B_{2,i}\\\text{(by (\ref{pf.prop.transpose.opers.d.3}),
applied}\\\text{to }k=2\text{)}}}+\cdots+\underbrace{\left(  B^{T}\right)
_{i,m}\left(  A^{T}\right)  _{m,j}}_{\substack{=A_{j,m}B_{m,i}\\\text{(by
(\ref{pf.prop.transpose.opers.d.3}), applied}\\\text{to }k=m\text{)}%
}}\ \ \ \ \ \ \ \ \ \ \left(  \text{by (\ref{pf.prop.transpose.opers.d.2}%
)}\right) \\
&  =A_{j,1}B_{1,i}+A_{j,2}B_{2,i}+\cdots+A_{j,m}B_{m,i}.
\end{align*}
Comparing this with (\ref{pf.prop.transpose.opers.d.1}), we conclude that
$\left(  \left(  AB\right)  ^{T}\right)  _{i,j}=\left(  B^{T}A^{T}\right)
_{i,j}$ for all $i\in\left\{  1,2,\ldots,p\right\}  $ and $j\in\left\{
1,2,\ldots,n\right\}  $. In other words, each entry of the matrix $\left(
AB\right)  ^{T}$ equals the corresponding entry of the matrix $B^{T}A^{T}$.
Thus, $\left(  AB\right)  ^{T}=B^{T}A^{T}$. This proves Proposition
\ref{prop.transpose.opers} \textbf{(e)}.
\end{proof}

\begin{exercise}
\label{exe.prop.transpose.opers.e}Prove Proposition \ref{prop.transpose.opers}
\textbf{(f)}.
\end{exercise}

\subsection{Triangular matrices}

We next discuss some particular classes of matrices: the so-called
\textit{triangular matrices} and some of their variations.

\begin{definition}
\label{def.triangular}Let $n\in\mathbb{N}$. Let $A$ be an $n\times n$-matrix.

\textbf{(a)} We say that the matrix $A$ is \textit{upper-triangular} if and
only if we have%
\[
A_{i,j}=0\ \ \ \ \ \ \ \ \ \ \text{whenever }i>j.
\]
(Of course, \textquotedblleft whenever $i>j$\textquotedblright\ is shorthand
for \textquotedblleft for all $i\in\left\{  1,2,\ldots,n\right\}  $ and
$j\in\left\{  1,2,\ldots,n\right\}  $ satisfying $i>j$\textquotedblright.)

\textbf{(b)} We say that the matrix $A$ is \textit{lower-triangular} if and
only if we have%
\[
A_{i,j}=0\ \ \ \ \ \ \ \ \ \ \text{whenever }i<j.
\]


\textbf{(c)} We say that the matrix $A$ is \textit{diagonal} if and only if we
have%
\[
A_{i,j}=0\ \ \ \ \ \ \ \ \ \ \text{whenever }i\neq j.
\]

\end{definition}

Notice that only square matrices can be upper-triangular or lower-triangular
or diagonal (by definition). Why the name \textquotedblleft
triangular\textquotedblright? Because visually speaking, a matrix is
upper-triangular if and only if all its entries (strictly) below the diagonal
are $0$ (which means that its nonzero entries are concentrated in the
\textbf{triangle} bordered by the diagonal, the upper rim and the right rim).
I hope Example \ref{exam.triangular} will clarify this if it is unclear.
Similarly, a matrix is lower-triangular if and only if all its entries
(strictly) above the diagonal are $0$ (which again means that its nonzero
entries are concentrated in a triangle, this time to the southwest of the
diagonal). Finally, a matrix is diagonal if and only if all its entries except
for the diagonal entries are $0$.

I think the following example should explain this:

\begin{example}
\label{exam.triangular}\textbf{(a)} A $4\times4$-matrix is upper-triangular if
and only if it has the form $\left(
\begin{array}
[c]{cccc}%
a & b & c & d\\
0 & b^{\prime} & c^{\prime} & d^{\prime}\\
0 & 0 & c^{\prime\prime} & d^{\prime\prime}\\
0 & 0 & 0 & d^{\prime\prime\prime}%
\end{array}
\right)  $ for some numbers $a,b,c,d,b^{\prime},c^{\prime},d^{\prime
},c^{\prime\prime},d^{\prime\prime},d^{\prime\prime\prime}$. Notice that we
are making no requirements on these numbers; in particular, they \textbf{can}
be $0$. Upper-triangularity means that $A_{i,j}=0$ whenever $i>j$; it does
\textbf{not} require that $A_{i,j}\neq0$ in all other cases.

\textbf{(b)} A $4\times4$-matrix is lower-triangular if and only if it has the
form $\left(
\begin{array}
[c]{cccc}%
a & 0 & 0 & 0\\
a^{\prime} & b^{\prime} & 0 & 0\\
a^{\prime\prime} & b^{\prime\prime} & c^{\prime\prime} & 0\\
a^{\prime\prime\prime} & b^{\prime\prime\prime} & c^{\prime\prime\prime} &
d^{\prime\prime\prime}%
\end{array}
\right)  $ for some numbers $a,a^{\prime},b^{\prime},a^{\prime\prime
},b^{\prime\prime},c^{\prime\prime},a^{\prime\prime\prime},b^{\prime
\prime\prime},c^{\prime\prime\prime},d^{\prime\prime\prime}$.

\textbf{(c)} A $4\times4$-matrix is diagonal if and only if it has the form
$\left(
\begin{array}
[c]{cccc}%
a & 0 & 0 & 0\\
0 & b^{\prime} & 0 & 0\\
0 & 0 & c^{\prime\prime} & 0\\
0 & 0 & 0 & d^{\prime\prime\prime}%
\end{array}
\right)  $ for some numbers $a,b^{\prime},c^{\prime\prime},d^{\prime
\prime\prime}$.
\end{example}

Here is something obvious:

\begin{proposition}
\label{prop.triangular.obvious}Let $n\in\mathbb{N}$.

\textbf{(a)} An $n\times n$-matrix $A$ is diagonal if and only if $A$ is both
upper-triangular and lower-triangular.

\textbf{(b)} The zero matrix $0_{n\times n}$ and the identity matrix $I_{n}$
are upper-triangular, lower-triangular and diagonal.
\end{proposition}

A less trivial fact is that the product of two upper-triangular matrices is
upper-triangular again. We shall show this, and a little bit more, in the
following theorem:

\begin{theorem}
\label{thm.triangular.prod-up}Let $n\in\mathbb{N}$. Let $A$ and $B$ be two
upper-triangular $n\times n$-matrices.

\textbf{(a)} Then, $AB$ is an upper-triangular $n\times n$-matrix.

\textbf{(b)} The diagonal entries of $AB$ are%
\[
\left(  AB\right)  _{i,i}=A_{i,i}B_{i,i}\ \ \ \ \ \ \ \ \ \ \text{for all
}i\in\left\{  1,2,\ldots,n\right\}  .
\]


\textbf{(c)} Also, $A+B$ is an upper-triangular $n\times n$-matrix.
Furthermore, $\lambda A$ is an upper-triangular matrix whenever $\lambda$ is a number.
\end{theorem}

Note that Theorem \ref{thm.triangular.prod-up} \textbf{(b)} says that each
diagonal entry of $AB$ is the product of the corresponding diagonal entries of
$A$ and of $B$. Thus, \textbf{in this specific case}, the product $AB$ does
behave as if matrices were multiplied entry by entry (but only for its
diagonal entries). Before I prove Theorem \ref{thm.triangular.prod-up}, let me
give an example:

\begin{example}
Let $A=\left(
\begin{array}
[c]{ccc}%
a & b & c\\
0 & b^{\prime} & c^{\prime}\\
0 & 0 & c^{\prime\prime}%
\end{array}
\right)  $ and $B=\left(
\begin{array}
[c]{ccc}%
x & y & z\\
0 & y^{\prime} & z^{\prime}\\
0 & 0 & z^{\prime\prime}%
\end{array}
\right)  $ be two upper-triangular $3\times3$-matrices. Then,%
\[
AB=\left(
\begin{array}
[c]{ccc}%
a & b & c\\
0 & b^{\prime} & c^{\prime}\\
0 & 0 & c^{\prime\prime}%
\end{array}
\right)  \left(
\begin{array}
[c]{ccc}%
x & y & z\\
0 & y^{\prime} & z^{\prime}\\
0 & 0 & z^{\prime\prime}%
\end{array}
\right)  =\left(
\begin{array}
[c]{ccc}%
ax & ay+by^{\prime} & az+bz^{\prime}+cz^{\prime\prime}\\
0 & b^{\prime}y^{\prime} & b^{\prime}z^{\prime}+c^{\prime}z^{\prime\prime}\\
0 & 0 & c^{\prime\prime}z^{\prime\prime}%
\end{array}
\right)  .
\]
Thus, $AB$ is again upper-triangular (as Theorem \ref{thm.triangular.prod-up}
\textbf{(a)} predicts), and the diagonal entries $ax,b^{\prime}y^{\prime
},c^{\prime\prime}z^{\prime\prime}$ of $AB$ are the products of the respective
entries of $A$ and of $B$ (as Theorem \ref{thm.triangular.prod-up}
\textbf{(b)} predicts).
\end{example}

\begin{proof}
[Proof of Theorem \ref{thm.triangular.prod-up}.]The matrix $A$ is
upper-triangular. In other words,%
\begin{equation}
A_{i,j}=0\ \ \ \ \ \ \ \ \ \ \text{whenever }i>j
\label{pf.thm.triangular.prod-up.A-tri}%
\end{equation}
(because this is what it means for $A$ to be upper-triangular). Similarly,%
\begin{equation}
B_{i,j}=0\ \ \ \ \ \ \ \ \ \ \text{whenever }i>j
\label{pf.thm.triangular.prod-up.B-tri}%
\end{equation}
(because $B$, too, is upper-triangular).

Now, fix two elements $i$ and $j$ of $\left\{  1,2,\ldots,n\right\}  $
satisfying $i>j$. We shall prove that for every $k\in\left\{  1,2,\ldots
,n\right\}  $, we have
\begin{equation}
A_{i,k}B_{k,j}=0. \label{pf.thm.triangular.prod-up.AB0}%
\end{equation}


[\textit{Proof of (\ref{pf.thm.triangular.prod-up.AB0}):} Let $k\in\left\{
1,2,\ldots,n\right\}  $. Then, we are in one of the following two cases:

\begin{statement}
\textit{Case 1:} We have $i\leq k$.
\end{statement}

\begin{statement}
\textit{Case 2:} We have $i>k$.
\end{statement}

We shall prove (\ref{pf.thm.triangular.prod-up.AB0}) in each of these two
cases separately:

\begin{enumerate}
\item Let us first consider Case 1. In this case, we have $i\leq k$. Thus,
$k\geq i$, so that $k\geq i>j$. Hence, we can apply
(\ref{pf.thm.triangular.prod-up.B-tri}) to $k$ instead of $i$. As a result, we
obtain $B_{k,j}=0$. Hence, $A_{i,k}\underbrace{B_{k,j}}_{=0}=A_{i,k}0=0$.
Thus, (\ref{pf.thm.triangular.prod-up.AB0}) is proven in Case 1.

\item Let us now consider Case 2. In this case, we have $i>k$. Hence, we can
apply (\ref{pf.thm.triangular.prod-up.A-tri}) to $k$ instead of $j$. As a
result, we obtain $A_{i,k}=0$. Hence, $\underbrace{A_{i,k}}_{=0}%
B_{k,j}=0B_{k,j}=0$. Thus, (\ref{pf.thm.triangular.prod-up.AB0}) is proven in
Case 2.
\end{enumerate}

We have now proven (\ref{pf.thm.triangular.prod-up.AB0}) in both Cases 1 and
2. Thus, (\ref{pf.thm.triangular.prod-up.AB0}) is proven.]

Now, Proposition \ref{prop.matrix-prod.rc} \textbf{(a)} shows that%
\begin{align*}
\left(  AB\right)  _{i,j}  &  =\underbrace{A_{i,1}B_{1,j}}%
_{\substack{=0\\\text{(by (\ref{pf.thm.triangular.prod-up.AB0}),}%
\\\text{applied to }k=1\text{)}}}+\underbrace{A_{i,2}B_{2,j}}%
_{\substack{=0\\\text{(by (\ref{pf.thm.triangular.prod-up.AB0}),}%
\\\text{applied to }k=2\text{)}}}+\cdots+\underbrace{A_{i,n}B_{n,j}%
}_{\substack{=0\\\text{(by (\ref{pf.thm.triangular.prod-up.AB0}),}%
\\\text{applied to }k=n\text{)}}}\\
&  =0+0+\cdots+0=0.
\end{align*}


Now, forget that we fixed $i$ and $j$. We thus have shown that%
\begin{equation}
\left(  AB\right)  _{i,j}=0\ \ \ \ \ \ \ \ \ \ \text{whenever }i>j.
\label{pf.thm.triangular.prod-up.AB0a}%
\end{equation}
But this says precisely that $AB$ is upper-triangular. Thus, Theorem
\ref{thm.triangular.prod-up} \textbf{(a)} is proven.

\textbf{(b)} Let $i\in\left\{  1,2,\ldots,n\right\}  $. We must prove that
$\left(  AB\right)  _{i,i}=A_{i,i}B_{i,i}$.

In a sense, this is similar to how we proved
(\ref{pf.thm.triangular.prod-up.AB0a}), but a little bit more complicated.

We first observe that, for every $k\in\left\{  1,2,\ldots,n\right\}  $
satisfying $k\neq i$, we have%
\begin{equation}
A_{i,k}B_{k,i}=0. \label{pf.thm.triangular.prod-up.AB1}%
\end{equation}


The proof of this will be very similar to the proof of
(\ref{pf.thm.triangular.prod-up.AB0}), with $j$ replaced by $i$:

[\textit{Proof of (\ref{pf.thm.triangular.prod-up.AB1}):} Let $k\in\left\{
1,2,\ldots,n\right\}  $ be such that $k\neq i$. Then, we are in one of the
following two cases:

\begin{statement}
\textit{Case 1:} We have $i\leq k$.
\end{statement}

\begin{statement}
\textit{Case 2:} We have $i>k$.
\end{statement}

We shall prove (\ref{pf.thm.triangular.prod-up.AB1}) in each of these two
cases separately:

\begin{enumerate}
\item Let us first consider Case 1. In this case, we have $i\leq k$. Thus,
$k\geq i$, so that $k>i$ (because $k\neq i$). Hence, we can apply
(\ref{pf.thm.triangular.prod-up.B-tri}) to $k$ and $i$ instead of $i$ and $j$.
As a result, we obtain $B_{k,i}=0$. Hence, $A_{i,k}\underbrace{B_{k,i}}%
_{=0}=A_{i,k}0=0$. Thus, (\ref{pf.thm.triangular.prod-up.AB1}) is proven in
Case 1.

\item Let us now consider Case 2. In this case, we have $i>k$. Hence, we can
apply (\ref{pf.thm.triangular.prod-up.A-tri}) to $k$ instead of $j$. As a
result, we obtain $A_{i,k}=0$. Hence, $\underbrace{A_{i,k}}_{=0}%
B_{k,i}=0B_{k,i}=0$. Thus, (\ref{pf.thm.triangular.prod-up.AB1}) is proven in
Case 2.
\end{enumerate}

We have now proven (\ref{pf.thm.triangular.prod-up.AB1}) in both Cases 1 and
2. Thus, (\ref{pf.thm.triangular.prod-up.AB1}) is proven.]

Now, Proposition \ref{prop.matrix-prod.rc} \textbf{(a)} (applied to $j=i$)
shows that%
\begin{align*}
&  \left(  AB\right)  _{i,i}\\
&  =A_{i,1}B_{1,i}+A_{i,2}B_{2,i}+\cdots+A_{i,n}B_{n,i}\\
&  =\left(  \text{the sum of the terms }A_{i,k}B_{k,i}\text{ for all }%
k\in\left\{  1,2,\ldots,n\right\}  \right) \\
&  =A_{i,i}B_{i,i}+\underbrace{\left(  \text{the sum of the terms }%
A_{i,k}B_{k,i}\text{ for all }k\in\left\{  1,2,\ldots,n\right\}  \text{
satisfying }k\neq i\right)  }_{\substack{=0\\\text{(because
(\ref{pf.thm.triangular.prod-up.AB1}) shows that all of these terms are
}0\text{)}}}\\
&  \ \ \ \ \ \ \ \ \ \ \left(  \text{here, we have taken the term }%
A_{i,i}B_{i,i}\text{ out of the sum}\right) \\
&  =A_{i,i}B_{i,i}+0=A_{i,i}B_{i,i}.
\end{align*}
This proves Theorem \ref{thm.triangular.prod-up} \textbf{(b)}.

\textbf{(c)} Theorem \ref{thm.triangular.prod-up} \textbf{(c)} is
straightforward to check (due to the simple definitions of $A+B$ and $\lambda
A$); the details are left to the reader.
\end{proof}

The natural analogue of Theorem \ref{thm.triangular.prod-up} for
lower-triangular matrices also holds:

\begin{theorem}
\label{thm.triangular.prod-down}Let $n\in\mathbb{N}$. Let $A$ and $B$ be two
lower-triangular $n\times n$-matrices.

\textbf{(a)} Then, $AB$ is a lower-triangular $n\times n$-matrix.

\textbf{(b)} The diagonal entries of $AB$ are%
\[
\left(  AB\right)  _{i,i}=A_{i,i}B_{i,i}\ \ \ \ \ \ \ \ \ \ \text{for all
}i\in\left\{  1,2,\ldots,n\right\}  .
\]


\textbf{(c)} Also, $A+B$ is a lower-triangular $n\times n$-matrix.
Furthermore, $\lambda A$ is a lower-triangular matrix whenever $\lambda$ is a number.
\end{theorem}

The proof of Theorem \ref{thm.triangular.prod-down} is analogous to that of
Theorem \ref{thm.triangular.prod-up}, and the changes required are fairly
straightforward (change some inequality signs). Let me pose this as an exercise:

\begin{exercise}
\label{exe.thm.triangular.prod-down}Prove Theorem
\ref{thm.triangular.prod-down} \textbf{(a)}. (Feel free to repeat my proof of
Theorem \ref{thm.triangular.prod-up} \textbf{(a)}, changing only what little
needs to be changed. This is not plagiarism for the purpose of this exercise!)

(Similarly, you can prove Theorem \ref{thm.triangular.prod-down} \textbf{(b)}
and \textbf{(c)}, but you don't need to write it up.)
\end{exercise}

The following result is an analogue of Theorem \ref{thm.triangular.prod-up}
and Theorem \ref{thm.triangular.prod-down} for diagonal matrices:

\begin{theorem}
\label{thm.triangular.prod-diag}Let $n\in\mathbb{N}$. Let $A$ and $B$ be two
diagonal $n\times n$-matrices.

\textbf{(a)} Then, $AB$ is a diagonal $n\times n$-matrix.

\textbf{(b)} The diagonal entries of $AB$ are%
\[
\left(  AB\right)  _{i,i}=A_{i,i}B_{i,i}\ \ \ \ \ \ \ \ \ \ \text{for all
}i\in\left\{  1,2,\ldots,n\right\}  .
\]
Thus, diagonal matrices actually \textbf{are} multiplied entry by entry!

\textbf{(c)} Also, $A+B$ is a diagonal $n\times n$-matrix. Furthermore,
$\lambda A$ is a diagonal matrix whenever $\lambda$ is a number.
\end{theorem}

\begin{exercise}
\label{exe.thm.triangular.prod-diag}Prove Theorem
\ref{thm.triangular.prod-diag}.
\end{exercise}

Lower-triangular and upper-triangular matrices are not only analogues of each
other; they are also closely related:

\begin{proposition}
\label{prop.triangular.UT=D}Let $n\in\mathbb{N}$. Let $A$ be an $n\times
n$-matrix. Then, $A$ is upper-triangular if and only if $A^{T}$ is lower-triangular.
\end{proposition}

\begin{proof}
[Proof of Proposition \ref{prop.triangular.UT=D}.]The definition of $A^{T}$
shows that $A^{T}=\left(  A_{j,i}\right)  _{1\leq i\leq n,\ 1\leq j\leq n}$.
Thus,%
\begin{equation}
\left(  A^{T}\right)  _{i,j}=A_{j,i}\ \ \ \ \ \ \ \ \ \ \text{for all }%
i\in\left\{  1,2,\ldots,n\right\}  \text{ and }j\in\left\{  1,2,\ldots
,n\right\}  . \label{pf.prop.triangular.UT=D.1}%
\end{equation}


Now, consider the following chain of equivalent statements\footnote{After each
equivalence, we give a justification for why it is an equivalence.}:%
\begin{align*}
&  \ \left(  A\text{ is upper-triangular}\right) \\
&  \Longleftrightarrow\ \left(  A_{i,j}=0\text{ whenever }i>j\right) \\
&  \ \ \ \ \ \ \ \ \ \ \left(  \text{because this is how \textquotedblleft
upper-triangular\textquotedblright\ is defined}\right) \\
&  \Longleftrightarrow\ \left(  A_{j,i}=0\text{ whenever }j>i\right) \\
&  \ \ \ \ \ \ \ \ \ \ \left(  \text{here, we have just renamed }i\text{ and
}j\text{ as }j\text{ and }i\right) \\
&  \Longleftrightarrow\ \left(  A_{j,i}=0\text{ whenever }i<j\right) \\
&  \ \ \ \ \ \ \ \ \ \ \left(  \text{because }j>i\text{ is equivalent to
}i<j\right) \\
&  \Longleftrightarrow\ \left(  \left(  A^{T}\right)  _{i,j}=0\text{ whenever
}i<j\right) \\
&  \ \ \ \ \ \ \ \ \ \ \left(  \text{here, we have replaced }A_{j,i}\text{ by
}\left(  A^{T}\right)  _{i,j}\text{, because of
(\ref{pf.prop.triangular.UT=D.1})}\right) \\
&  \Longleftrightarrow\ \left(  A^{T}\text{ is lower-triangular}\right) \\
&  \ \ \ \ \ \ \ \ \ \ \left(  \text{because this is how \textquotedblleft
lower-triangular\textquotedblright\ is defined}\right)  .
\end{align*}
Thus, Proposition \ref{prop.triangular.UT=D} holds.
\end{proof}

There are a few special classes of triangular matrices worth giving names:

\begin{definition}
\label{def.triangular.uni-up}Let $n\in\mathbb{N}$. Let $A$ be an $n\times n$-matrix.

\textbf{(a)} The matrix $A$ is said to be \textit{upper-unitriangular} if and
only if it is upper-triangular and all its diagonal entries are $1$ (that is,
$A_{i,i}=1$ for all $i$).

\textbf{(b)} The matrix $A$ is said to be \textit{invertibly upper-triangular}
if and only if it is upper-triangular and all its diagonal entries are nonzero
(that is, $A_{i,i}\neq0$ for all $i$).

\textbf{(c)} The matrix $A$ is said to be \textit{strictly upper-triangular}
if and only if it is upper-triangular and all its diagonal entries are $0$
(that is, $A_{i,i}=0$ for all $i$).

Similar notions can be defined with the word \textquotedblleft
lower\textquotedblright\ instead of \textquotedblleft upper\textquotedblright:

\textbf{(d)} The matrix $A$ is said to be \textit{lower-unitriangular} if and
only if it is lower-triangular and all its diagonal entries are $1$ (that is,
$A_{i,i}=1$ for all $i$).

\textbf{(e)} The matrix $A$ is said to be \textit{invertibly lower-triangular}
if and only if it is lower-triangular and all its diagonal entries are nonzero
(that is, $A_{i,i}\neq0$ for all $i$).

\textbf{(f)} The matrix $A$ is said to be \textit{strictly lower-triangular}
if and only if it is lower-triangular and all its diagonal entries are $0$
(that is, $A_{i,i}=0$ for all $i$).
\end{definition}

The words we have just defined are not as important as the word
\textquotedblleft upper-triangular\textquotedblright\ (you certainly don't
need to learn them by heart); but these notions appear from time to time in
mathematics, and it helps if you know how to recognize them.

\begin{example}
\textbf{(a)} A $3\times3$-matrix is upper-unitriangular if and only if it has
the form $\left(
\begin{array}
[c]{ccc}%
1 & b & c\\
0 & 1 & c^{\prime}\\
0 & 0 & 1
\end{array}
\right)  $ for some $b,c,c^{\prime}$.

\textbf{(b)} A $3\times3$-matrix is invertibly upper-triangular if and only if
it has the form $\left(
\begin{array}
[c]{ccc}%
a & b & c\\
0 & b^{\prime} & c^{\prime}\\
0 & 0 & c^{\prime\prime}%
\end{array}
\right)  $ for some $a,b,c,b^{\prime},c^{\prime},c^{\prime\prime}$ with
$a\neq0$, $b^{\prime}\neq0$ and $c^{\prime\prime}\neq0$.

\textbf{(c)} A $3\times3$-matrix is strictly upper-triangular if and only if
it has the form $\left(
\begin{array}
[c]{ccc}%
0 & b & c\\
0 & 0 & c^{\prime}\\
0 & 0 & 0
\end{array}
\right)  $ for some $b,c,c^{\prime}$.
\end{example}

Olver and Shakiban (in \cite{OlvSha06}) use the word \textquotedblleft special
upper triangular\textquotedblright\ instead of \textquotedblleft
upper-unitriangular\textquotedblright. But I prefer \textquotedblleft
upper-unitriangular\textquotedblright, since the word \textquotedblleft
uni\textquotedblright\ hints directly to the definition (namely, the $1$'s on
the diagonal), whereas the word \textquotedblleft special\textquotedblright%
\ can mean pretty much anything.

The word \textquotedblleft invertibly upper-triangular\textquotedblright\ is
my invention. I have chosen it because an upper-triangular matrix is
invertible if and only if it is invertibly upper-triangular. (This is not
obvious. In Theorem \ref{thm.triangular.inverse-up.inv}, we will prove the
\textquotedblleft if\textquotedblright\ direction. The \textquotedblleft only
if\textquotedblright\ direction is also true.)

Here is a simple fact to connect the above definitions:

\begin{proposition}
\label{prop.triangular.uni-inv}\textbf{(a)} Each upper-unitriangular matrix is
invertibly upper-triangular.

\textbf{(b)} Let $n \in\mathbb{N}$. Let $A$ be an $n \times n$-matrix. Then,
$A$ is upper-unitriangular if and only if $I_{n} - A$ is strictly upper-triangular.
\end{proposition}

We shall give a proof of this proposition in Section
\ref{sect.gauss.triang-proof1} (but mainly as an example of how to write a
proof; the mathematics itself is trivial).

Strictly upper-triangular $n\times n$-matrices can also be characterized as
the $n\times n$-matrices $A$ which satisfy%
\[
A_{i,j}=0\ \ \ \ \ \ \ \ \ \ \text{whenever }i\geq j.
\]
Notice the weak inequality \textquotedblleft$i\geq j$\textquotedblright\ (as
opposed to the strict inequality \textquotedblleft$i>j$\textquotedblright\ in
the definition of upper-triangular matrices).

\begin{exercise}
\label{exe.inverses.3x3-ut}Let $A=\left(
\begin{array}
[c]{ccc}%
a & b & c\\
0 & b^{\prime} & c^{\prime}\\
0 & 0 & c^{\prime\prime}%
\end{array}
\right)  $ be an invertibly upper-triangular $3\times3$-matrix. Show that $A$
is invertible by explicitly computing the inverse of $A$ (in terms of
$a,b,c,b^{\prime},c^{\prime},c^{\prime\prime}$).

[\textbf{Hint:} In order to find a right inverse of $A$, it is enough to find
three column vectors $u,v,w$ (each of size $3$) satisfying the equations%
\[
Au=\left(
\begin{array}
[c]{c}%
1\\
0\\
0
\end{array}
\right)  ,\ \ \ \ \ \ \ \ \ \ Av=\left(
\begin{array}
[c]{c}%
0\\
1\\
0
\end{array}
\right)  ,\ \ \ \ \ \ \ \ \ \ Aw=\left(
\begin{array}
[c]{c}%
0\\
0\\
1
\end{array}
\right)  .
\]
In fact, once these vectors are found, assembling them into a matrix yields a
right inverse of $A$ (why?). Find these $u,v,w$. Then, check that the
resulting right inverse of $A$ is also a left inverse.]
\end{exercise}

\begin{corollary}
\label{cor.triangular.uni*uni}Let $n\in\mathbb{N}$. Let $A$ and $B$ be two
upper-unitriangular $n\times n$-matrices. Then, $AB$ is also an
upper-unitriangular $n\times n$-matrix.
\end{corollary}

\begin{proof}
[Proof of Corollary \ref{cor.triangular.uni*uni}.]The matrices $A$ and $B$ are
upper-triangular (since they are upper-unitriangular). Hence, Theorem
\ref{thm.triangular.prod-up} \textbf{(a)} shows that $AB$ is an
upper-triangular $n\times n$-matrix. Moreover, Theorem
\ref{thm.triangular.prod-up} \textbf{(b)} shows that
\[
\left(  AB\right)  _{i,i}=\underbrace{A_{i,i}}_{\substack{=1\\\text{(since
}A\text{ is}\\\text{unitriangular)}}}\underbrace{B_{i,i}}%
_{\substack{=1\\\text{(since }B\text{ is}\\\text{unitriangular)}}}=1\cdot1=1
\]
for all $i\in\left\{  1,2,\ldots,n\right\}  $. Thus, the matrix $AB$ is
upper-unitriangular (since we already know that $AB$ is upper-triangular).
Corollary \ref{cor.triangular.uni*uni} is thus proven.
\end{proof}

\begin{corollary}
\label{cor.triangular.uni*uni*uni}Let $n\in\mathbb{N}$. If $A_{1},A_{2}%
,\ldots,A_{k}$ (for some $k\in\mathbb{N}$) are upper-unitriangular $n\times
n$-matrices, then $A_{1}A_{2}\cdots A_{k}$ is also an upper-unitriangular
$n\times n$-matrix.
\end{corollary}

\begin{proof}
[Proof of Corollary \ref{cor.triangular.uni*uni*uni}.]This can be proven in a
straightforward way by induction over $k$, similarly to how we proved
Proposition \ref{prop.inverses.A1Ak}. The important differences are:

\begin{itemize}
\item We should now use $k=0$ as an induction base (because we have not
required $k$ to be positive). For $k=0$, the product $A_{1}A_{2}\cdots A_{k}$
is an empty product (i.e., a product with no factors), and thus equals $I_{n}$
(because we have \textbf{defined} an empty product of $n\times n$-matrices to
equal $I_{n}$). We thus need to prove that $I_{n}$ is an upper-unitriangular
matrix. But this is clear by inspection.

\item In the induction step, instead of using Proposition
\ref{prop.inverses.AB}, we need to use Corollary \ref{cor.triangular.uni*uni}.
\end{itemize}
\end{proof}

Analogues of Corollary \ref{cor.triangular.uni*uni} and Corollary
\ref{cor.triangular.uni*uni*uni} hold for invertibly upper-triangular matrices:

\begin{corollary}
\label{cor.triangular.inv*inv}Let $n\in\mathbb{N}$. Let $A$ and $B$ be two
invertibly upper-triangular $n\times n$-matrices. Then, $AB$ is also an
invertibly upper-triangular $n\times n$-matrix.
\end{corollary}

\begin{corollary}
\label{cor.triangular.inv*inv*inv}Let $n\in\mathbb{N}$. If $A_{1},A_{2}%
,\ldots,A_{k}$ (for some $k\in\mathbb{N}$) are invertibly upper-triangular
$n\times n$-matrices, then $A_{1}A_{2}\cdots A_{k}$ is also an invertibly
upper-triangular $n\times n$-matrix.
\end{corollary}

\begin{exercise}
\label{exe.cor.triangular.inv*inv}Prove Corollary \ref{cor.triangular.inv*inv}
and Corollary \ref{cor.triangular.inv*inv*inv}.
\end{exercise}

Similarly, analogues of the above-mentioned results hold for lower-triangular
matrices. For example, the following analogues of Corollary
\ref{cor.triangular.uni*uni*uni} and of Corollary
\ref{cor.triangular.inv*inv*inv} hold:

\begin{corollary}
\label{cor.triangular.uni*uni*uni.lower}Let $n\in\mathbb{N}$. If $A_{1}%
,A_{2},\ldots,A_{k}$ (for some $k\in\mathbb{N}$) are lower-unitriangular
$n\times n$-matrices, then $A_{1}A_{2}\cdots A_{k}$ is also a
lower-unitriangular $n\times n$-matrix.
\end{corollary}

\begin{corollary}
\label{cor.triangular.inv*inv*inv.lower}Let $n\in\mathbb{N}$. If $A_{1}%
,A_{2},\ldots,A_{k}$ (for some $k\in\mathbb{N}$) are invertibly
lower-triangular $n\times n$-matrices, then $A_{1}A_{2}\cdots A_{k}$ is also
an invertibly lower-triangular $n\times n$-matrix.
\end{corollary}

As usual, the proofs of these analogues can be obtained from the proofs of the
original versions through minor (and straightforward) modifications.

The reader can easily check that the following analogue of Proposition
\ref{prop.triangular.UT=D} holds:

\begin{proposition}
\label{prop.triangular.UT=D.uni}Let $n\in\mathbb{N}$. Let $A$ be an $n\times
n$-matrix. Then:

\textbf{(a)} The matrix $A$ is upper-unitriangular if and only if $A^{T}$ is lower-unitriangular.

\textbf{(b)} The matrix $A$ is invertibly upper-triangular if and only if
$A^{T}$ is invertibly lower-triangular.

\textbf{(c)} The matrix $A$ is strictly upper-triangular if and only if
$A^{T}$ is strictly lower-triangular.
\end{proposition}

\subsection{\label{sect.gauss.triang-proof1}(*) Proof of Proposition
\ref{prop.triangular.uni-inv}}

Here is a detailed proof of Proposition \ref{prop.triangular.uni-inv}:

\begin{proof}
[Proof of Proposition \ref{prop.triangular.uni-inv}.]\textbf{(a)} Let $A$ be
an upper-unitriangular $n \times n$-matrix. We must then prove that $A$ is
invertibly upper-triangular.

We have assumed that $A$ is upper-unitriangular. In other words, $A$ is
upper-triangular and all its diagonal entries are $1$ (because this is what
``upper-unitriangular'' means). Now, all diagonal entries of $A$ are $1$, and
thus are nonzero (since $1$ is nonzero). So we know that $A$ is
upper-triangular and all its diagonal entries are nonzero. In other words, $A$
is invertibly upper-triangular (by the definition of ``invertibly
upper-triangular''). This proves Proposition \ref{prop.triangular.uni-inv}
\textbf{(a)}.

\textbf{(b)} The statement of Proposition \ref{prop.triangular.uni-inv}
\textbf{(b)} is an ``if and only if'' statement. Thus, it splits into the
following two claims:

\begin{statement}
\textit{Claim 1:} If $A$ is upper-unitriangular, then $I_{n} - A$ is strictly upper-triangular.
\end{statement}

\begin{statement}
\textit{Claim 2:} If $I_{n} - A$ is strictly upper-triangular, then $A$ is upper-unitriangular.
\end{statement}

We are going to prove both of these claims. But first, let us make a simple
observation: Recall that $I_{n} = \left(  \delta_{i,j}\right)  _{1\leq i\leq
n, \ 1\leq j\leq n}$. Hence, $\left(  I_{n}\right)  _{i, j} = \delta_{i, j}$
for all $i \in\left\{  1,2,\ldots,n\right\}  $ and $j \in\left\{
1,2,\ldots,n\right\}  $. In particular, for all $i \in\left\{  1,2,\ldots
,n\right\}  $ and $j \in\left\{  1,2,\ldots,n\right\}  $ satisfying $i > j$,
we have
\begin{equation}
\left(  I_{n}\right)  _{i, j} = \delta_{i, j} = 0
\label{pf.prop.triangular.uni-inv.b.1}%
\end{equation}
(since $i \neq j$ (because $i > j$)). Also, the diagonal entries of the matrix
$I_{n}$ are $1$; in other words, every $i \in\left\{  1,2,\ldots,n\right\}  $
satisfies
\begin{equation}
\left(  I_{n}\right)  _{i, i} = 1. \label{pf.prop.triangular.uni-inv.b.2}%
\end{equation}


\textit{Proof of Claim 1:} Assume that $A$ is upper-unitriangular. We must
show that $I_{n} - A$ is strictly upper-triangular.

We have assumed that $A$ is upper-unitriangular. In other words, $A$ is
upper-triangular and all its diagonal entries are $1$ (because this is what
``upper-unitriangular'' means). Since $A$ is upper-triangular, we have
\begin{equation}
A_{i, j} = 0 \ \ \ \ \ \ \ \ \ \ \text{whenever } i > j
\label{pf.prop.triangular.uni-inv.b.to.1}%
\end{equation}
(by the definition of ``upper-triangular''). Since all diagonal entries of $A$
are $1$, we have
\begin{equation}
A_{i, i} = 1 \ \ \ \ \ \ \ \ \ \ \text{for each } i \in\left\{  1,2,\ldots
,n\right\}  . \label{pf.prop.triangular.uni-inv.b.to.2}%
\end{equation}


Now, recall that matrices are subtracted entry by entry. Hence, for all $i
\in\left\{  1,2,\ldots,n\right\}  $ and $j \in\left\{  1,2,\ldots,n\right\}  $
satisfying $i > j$, we have
\[
\left(  I_{n} - A\right)  _{i, j} = \underbrace{\left(  I_{n}\right)  _{i,j}%
}_{\substack{= 0 \\\text{(by (\ref{pf.prop.triangular.uni-inv.b.1}))}}} -
\underbrace{A_{i,j}}_{\substack{= 0 \\\text{(by
(\ref{pf.prop.triangular.uni-inv.b.to.1}))}}} = 0 - 0 = 0.
\]
In other words, $\left(  I_{n} - A\right)  _{i, j} = 0$ whenever $i > j$. This
means that $I_{n} - A$ is upper-triangular (by the definition of ``upper-triangular'').

Recall again that matrices are subtracted entry by entry. Hence, every $i
\in\left\{  1, 2, \ldots, n\right\}  $ satisfies
\[
\left(  I_{n} - A\right)  _{i, i} = \underbrace{\left(  I_{n}\right)  _{i,i}%
}_{\substack{= 1 \\\text{(by (\ref{pf.prop.triangular.uni-inv.b.2}))}}} -
\underbrace{A_{i,i}}_{\substack{= 1 \\\text{(by
(\ref{pf.prop.triangular.uni-inv.b.to.2}))}}} = 1 - 1 = 0.
\]
In other words, all diagonal entries of the matrix $I_{n} - A$ are $0$.

So we have shown that the matrix $I_{n} - A$ is upper-triangular, and that all
its diagonal entries are $0$. In other words, the matrix $I_{n} - A$ is
strictly upper-triangular (by the definition of ``strictly
upper-triangular''). This proves Claim 1.

\textit{Proof of Claim 2:} Assume that $I_{n} - A$ is strictly
upper-triangular. We must show that $A$ is upper-unitriangular.

We have assumed that $I_{n} - A$ is strictly upper-triangular. In other words,
$I_{n} - A$ is upper-triangular and all its diagonal entries are $0$ (because
this is what ``strictly upper-triangular'' means). Since $I_{n} - A$ is
upper-triangular, we have
\begin{equation}
\left(  I_{n} - A\right)  _{i, j} = 0 \ \ \ \ \ \ \ \ \ \ \text{whenever } i >
j \label{pf.prop.triangular.uni-inv.b.from.1}%
\end{equation}
(by the definition of ``upper-triangular''). Since all diagonal entries of
$I_{n} - A$ are $0$, we have
\begin{equation}
\left(  I_{n} - A\right)  _{i, i} = 0 \ \ \ \ \ \ \ \ \ \ \text{for each } i
\in\left\{  1,2,\ldots,n\right\}  .
\label{pf.prop.triangular.uni-inv.b.from.2}%
\end{equation}


Now, recall that matrices are subtracted entry by entry. Hence, for all $i
\in\left\{  1,2,\ldots,n\right\}  $ and $j \in\left\{  1,2,\ldots,n\right\}  $
satisfying $i > j$, we have
\[
\left(  I_{n} - A\right)  _{i, j} = \underbrace{\left(  I_{n}\right)  _{i,j}%
}_{\substack{= 0 \\\text{(by (\ref{pf.prop.triangular.uni-inv.b.1}))}}} -
A_{i, j} = 0 - A_{i, j} = - A_{i, j}%
\]
and therefore
\[
A_{i, j} = - \underbrace{\left(  I_{n} - A\right)  _{i,j}}_{\substack{= 0
\\\text{(by (\ref{pf.prop.triangular.uni-inv.b.from.1}))}}} = - 0 = 0 .
\]
In other words, $A_{i, j} = 0$ whenever $i > j$. This means that $A$ is
upper-triangular (by the definition of ``upper-triangular'').

Recall again that matrices are subtracted entry by entry. Hence, every $i
\in\left\{  1, 2, \ldots, n\right\}  $ satisfies
\[
\left(  I_{n} - A\right)  _{i, i} = \underbrace{\left(  I_{n}\right)  _{i,i}%
}_{\substack{= 1 \\\text{(by (\ref{pf.prop.triangular.uni-inv.b.2}))}}} -
A_{i, i} = 1 - A_{i, i}%
\]
and therefore
\[
A_{i, i} = 1 - \underbrace{\left(  I_{n} - A\right)  _{i,i}}_{\substack{= 0
\\\text{(by (\ref{pf.prop.triangular.uni-inv.b.from.2}))}}} = 1 - 0 = 1.
\]
In other words, all diagonal entries of the matrix $A$ are $1$.

So we have shown that the matrix $A$ is upper-triangular, and that all its
diagonal entries are $1$. In other words, the matrix $A$ is
upper-unitriangular. This proves Claim 2.

Now, both Claim 1 and Claim 2 are proven, so the proof of Proposition
\ref{prop.triangular.uni-inv} \textbf{(b)} is complete.
\end{proof}

As you might have noticed, I have written down the above proof at an unusually
high level of detail (whereas most textbooks would have only sketched it, or
even left it to the reader to fill in). The reason for that is that I wanted
to demonstrate the structure of such proofs. An experienced writer (writing
for experienced readers) would have been able to shorten the above proof
considerably in the following way:

\begin{itemize}
\item Our proof of Proposition \ref{prop.triangular.uni-inv} \textbf{(a)} was
really obvious; most of it was boilerplate (writing down the assumptions,
writing down the claims, etc.).

\item The way we proved Proposition \ref{prop.triangular.uni-inv} \textbf{(b)}
is a typical way how ``if and only if'' statements are proven.\footnote{But
not the only way: our proof of Proposition \ref{prop.triangular.UT=D} was
organized differently.} What we called Claim 1 in this proof would normally be
called the ``$\Longrightarrow$ direction''\footnote{also known as the ``only
if direction'' or the ``$\Longrightarrow$ part''} of Proposition
\ref{prop.triangular.uni-inv} \textbf{(b)} (because, rewritten in logical
symbols, it says that ``$\left(  A \text{ is upper-unitriangular} \right)
\Longrightarrow\left(  I_{n} - A \text{ is strictly upper-triangular}\right)
$''), while our Claim 2 would be called the ``$\Longleftarrow$
direction''\footnote{also known as the ``if direction'' or the
``$\Longleftarrow$ part''} of Proposition \ref{prop.triangular.uni-inv}
\textbf{(b)} (for similar reasons). In general, if you have an assertion of
the form ``$X$ holds if and only if $Y$ holds'', then the ``$\Longrightarrow$
direction'' of this assertion says ``if $X$ holds, then $Y$ holds'', whereas
the ``$\Longleftarrow$ direction'' of this assertion says ``if $Y$ holds, then
$X$ holds''. In order to prove the assertion, it suffices to prove both its
$\Longrightarrow$ direction and its $\Longleftarrow$ direction; these two
directions can often be proven separately. It is customary to mark the proof
of the $\Longrightarrow$ direction by a single ``$\Longrightarrow$:'' at the
beginning of this proof (instead of writing ``Proof of Claim 1:'' as we did),
and to mark the proof of the $\Longleftarrow$ direction by a single
``$\Longleftarrow$:'' at the beginning of this proof (instead of writing
``Proof of Claim 2:'' as we did). Furthermore, explicitly stating Claim 1 and
Claim 2 (like I did above) is not necessary: They are just the
$\Longrightarrow$ direction and the $\Longleftarrow$ direction of Proposition
\ref{prop.triangular.uni-inv} \textbf{(b)}, respectively; this is enough to
fully characterize them.

\item Our proof of the $\Longrightarrow$ direction (i.e., of Claim 1) was
straightforward: it was following the most obvious route from the givens
(i.e., from the assumption that $A$ is upper-unitriangular) to the goal (i.e.,
to the claim that $I_{n} - A$ is strictly upper-triangular). In fact, once you
have unraveled the definitions of ``upper-unitriangular'' and of
``triangular'', the assumption translates into ``$A_{i,j} = 0$ whenever $i >
j$, and $A_{i,i} = 1$ for all $i$''. Similarly, once you have unraveled the
definitions of ``strictly upper-triangular'' and ``triangular'', the claim
translates into ``$\left(  I_{n}-A\right)  _{i,j} = 0$ whenever $i > j$, and
$\left(  I_{n}-A\right)  _{i,i} = 0$ for all $i$''. Thus, in order to get from
the assumption to the goal, you need to find a way to process knowledge about
entries of $A$ into knowledge about entries of $I_{n} - A$. But there is one
obvious way to do that: Observe that $\left(  I_{n} - A\right)  _{i,j} =
\left(  I_{n}\right)  _{i,j} - A_{i,j}$, and recall the formula for $\left(
I_{n}\right)  _{i,j}$ (which is clear from the definition of $I_{n}$). The
rest is simple arithmetic.

\item Our proof of the $\Longleftarrow$ direction (i.e., of Claim 2) was
essentially the proof of the $\Longrightarrow$ direction (i.e., of Claim 1)
``read backwards'' (the assumption and the claim have switched places, so we
are taking the same argument but in reverse). To read an argument backwards
means, whenever necessary, to switch reasons with consequences (for example,
instead of deriving $\left(  I_{n} - A\right)  _{i, j} = 0$ from $A_{i, j} =
0$, we now derive $A_{i, j} = 0$ from $\left(  I_{n} - A\right)  _{i, j} =
0$). This is not possible (after all, not every valid statement remains valid
if we switch the assumption and the claim!), but when it is, it is not a
difficult task, so it can safely be left to the reader. (And it does work in
our case.)
\end{itemize}

Altogether, we can thus shrink down our above proof of Proposition
\ref{prop.triangular.uni-inv} to the following short form:

\begin{proof}
[Proof of Proposition \ref{prop.triangular.uni-inv} (sketched).]\textbf{(a)}
This follows from the definitions, since $1$ is nonzero.

\textbf{(b)} $\Longrightarrow$: Each entry of the matrix $I_{n} - A$ equals
the corresponding entry of $I_{n}$ minus the corresponding entry of $A$. Thus,
any statement about entries of $I_{n} - A$ can be reduced to a statement about
corresponding entries of $A$. Using this observation, the $\Longrightarrow$
direction of Proposition \ref{prop.triangular.uni-inv} \textbf{(b)} becomes straightforward.

$\Longleftarrow$: To obtain a proof of the $\Longleftarrow$ direction, read
the proof of the $\Longrightarrow$ direction backwards.
\end{proof}

(Or we could have left the whole proof to the reader, seeing that pretty much
all of it was straightforward walking from the assumptions to the goals.)

\subsection{The standard matrix units $E_{u,v}$}

We shall now define a certain family of matrices which are (in a sense) the
building blocks of all matrices:

\begin{definition}
\label{def.Euv}Let $n\in\mathbb{N}$ and $m\in\mathbb{N}$. Let $u\in\left\{
1,2,\ldots,n\right\}  $ and $v\in\left\{  1,2,\ldots,m\right\}  $. Then,
$E_{u,v,n,m}$ shall denote the $n\times m$-matrix whose $\left(  u,v\right)
$-th entry is $1$ and whose all other entries are $0$. We shall abbreviate
$E_{u,v,n,m}$ as $E_{u,v}$ when the values of $n$ and $m$ are clear from the
context (for example, when we say \textquotedblleft the $2\times5$-matrix
$E_{1,4}$\textquotedblright, it is clear that $n=2$ and $m=5$).

The matrices $E_{u,v}$ (for varying $u$ and $v$) are called the
\textit{standard matrix units}.
\end{definition}

\begin{example}
The $2\times3$-matrix $E_{1,3}$ (also known as $E_{1,3,2,3}$) is $\left(
\begin{array}
[c]{ccc}%
0 & 0 & 1\\
0 & 0 & 0
\end{array}
\right)  $. As per its definition, its $\left(  1,3\right)  $-th entry is $1$
and its all other entries are $0$.

The $3\times2$-matrix $E_{2,2}$ (also known as $E_{2,2,3,2}$) is $\left(
\begin{array}
[c]{cc}%
0 & 0\\
0 & 1\\
0 & 0
\end{array}
\right)  $.
\end{example}

What happens when a matrix is multiplied by $E_{u,v}$ ? There are two cases:
either the $E_{u,v}$ is on the left (so we are talking of a product $E_{u,v}%
C$) or the $E_{u,v}$ is on the right (so we are talking of a product
$CE_{u,v}$). Let us see what each of these looks like:

\begin{proposition}
\label{prop.Euv.laction}Let $n\in\mathbb{N}$, $m\in\mathbb{N}$ and
$p\in\mathbb{N}$. Let $u\in\left\{  1,2,\ldots,n\right\}  $ and $v\in\left\{
1,2,\ldots,m\right\}  $. Let $C$ be an $m\times p$-matrix. Then, $E_{u,v}C$ is
the $n\times p$-matrix whose $u$-th row is the $v$-th row of $C$, and whose
all other rows are filled with zeroes. (Here, again, $E_{u,v}$ means
$E_{u,v,n,m}$.)
\end{proposition}

\begin{example}
Let $n=2$, $m=3$ and $p=3$. Let $C=\left(
\begin{array}
[c]{ccc}%
a & b & c\\
a^{\prime} & b^{\prime} & c^{\prime}\\
a^{\prime\prime} & b^{\prime\prime} & c^{\prime\prime}%
\end{array}
\right)  $ be a $3\times3$-matrix. Proposition \ref{prop.Euv.laction} (applied
to $u=1$ and $v=2$) claims that $E_{1,2}C$ is the $2\times3$-matrix whose
$1$-st row is the $2$-nd row of $C$, and whose all other rows are filled with
zeroes. In other words, it claims that%
\[
E_{1,2}C=\left(
\begin{array}
[c]{ccc}%
a^{\prime} & b^{\prime} & c^{\prime}\\
0 & 0 & 0
\end{array}
\right)  .
\]
We can verify this by actually doing the multiplication:%
\begin{align*}
E_{1,2}C  &  =\left(
\begin{array}
[c]{ccc}%
0 & 1 & 0\\
0 & 0 & 0
\end{array}
\right)  \left(
\begin{array}
[c]{ccc}%
a & b & c\\
a^{\prime} & b^{\prime} & c^{\prime}\\
a^{\prime\prime} & b^{\prime\prime} & c^{\prime\prime}%
\end{array}
\right) \\
&  =\left(
\begin{array}
[c]{ccc}%
0a+1a^{\prime}+0a^{\prime\prime} & 0b+1b^{\prime}+0b^{\prime\prime} &
0c+1c^{\prime}+0c^{\prime\prime}\\
0a+0a^{\prime}+0a^{\prime\prime} & 0b+0b^{\prime}+0b^{\prime\prime} &
0c+0c^{\prime}+0c^{\prime\prime}%
\end{array}
\right) \\
&  =\left(
\begin{array}
[c]{ccc}%
a^{\prime} & b^{\prime} & c^{\prime}\\
0 & 0 & 0
\end{array}
\right)  .
\end{align*}
The matrix unit $E_{1,2}=\left(
\begin{array}
[c]{ccc}%
0 & 1 & 0\\
0 & 0 & 0
\end{array}
\right)  $ acts as a sort of mask which, when multiplied by $C$, moves the
$v$-th row into the $u$-th row of the product, while destroying all other rows.
\end{example}

We shall give a formal proof of Proposition \ref{prop.Euv.laction} in Section
\ref{sect.gauss.Euv-more}; but the example above should have given you a good
intuition for it.

So much for products of the form $E_{u,v}C$. What about $CE_{u,v}$ ?

\begin{proposition}
\label{prop.Euv.raction}Let $n\in\mathbb{N}$, $m\in\mathbb{N}$ and
$p\in\mathbb{N}$. Let $u\in\left\{  1,2,\ldots,n\right\}  $ and $v\in\left\{
1,2,\ldots,m\right\}  $. Let $C$ be an $p\times n$-matrix. Then, $CE_{u,v}$ is
the $p\times m$-matrix whose $v$-th column is the $u$-th column of $C$, and
whose all other columns are filled with zeroes. (Here, again, $E_{u,v}$ means
$E_{u,v,n,m}$.)
\end{proposition}

Notice that the numbers $u$ and $v$ play different parts in Proposition
\ref{prop.Euv.raction} as in Proposition \ref{prop.Euv.laction}.

\begin{example}
Let us demonstrate Proposition \ref{prop.Euv.raction} on a more spartanic
example: Let $n=2$, $m=2$ and $p=1$. Let $C=\left(
\begin{array}
[c]{cc}%
a & b
\end{array}
\right)  $ be a $1\times2$-matrix. Proposition \ref{prop.Euv.raction} (applied
to $u=1$ and $v=2$) claims that $CE_{1,2}$ is the $1\times2$-matrix whose
$2$-nd column is the $1$-st column of $C$, and whose all other columns are
filled with zeroes. In other words, it claims that%
\[
CE_{1,2}=\left(
\begin{array}
[c]{cc}%
0 & a
\end{array}
\right)  .
\]
Again, we can verify this by actually doing the multiplication:%
\[
CE_{1,2}=\left(
\begin{array}
[c]{cc}%
a & b
\end{array}
\right)  \left(
\begin{array}
[c]{cc}%
0 & 1\\
0 & 0
\end{array}
\right)  =\left(
\begin{array}
[c]{cc}%
a\cdot0+b\cdot0 & a\cdot1+b\cdot0
\end{array}
\right)  =\left(
\begin{array}
[c]{cc}%
0 & a
\end{array}
\right)  .
\]

\end{example}

\subsection{\label{sect.gauss.Euv-more}(*) A bit more on the standard matrix
units}

Let us get some practice by rewriting the definition of the matrices $E_{u,v}$:

\begin{proposition}
\label{prop.Euv.deltas}Let $n\in\mathbb{N}$ and $m\in\mathbb{N}$. Let
$u\in\left\{  1,2,\ldots,n\right\}  $ and $v\in\left\{  1,2,\ldots,m\right\}
$. Let $i\in\left\{  1,2,\ldots,n\right\}  $ and $j\in\left\{  1,2,\ldots
,m\right\}  $. Then,%
\begin{equation}
\left(  E_{u,v}\right)  _{i,j}=\delta_{\left(  i,j\right)  ,\left(
u,v\right)  }=\delta_{i,u}\delta_{j,v} \label{eq.prop.Euv.deltas.1}%
\end{equation}
(where $E_{u,v}$ is short for $E_{u,v,n,m}$). (Recall that the meaning of the
symbols $\delta_{\left(  i,j\right)  ,\left(  u,v\right)  }$, $\delta_{i,u}$
and $\delta_{j,v}$ is defined as in (\ref{eq.def.In.eq}).)
\end{proposition}

\begin{proof}
[Proof of Proposition \ref{prop.Euv.deltas}.]We have defined $E_{u,v}$ as the
$n\times m$-matrix whose $\left(  u,v\right)  $-th entry is $1$ and whose all
other entries are $0$. Hence, the $\left(  i,j\right)  $-th entry of the
matrix $E_{u,v}$ is $1$ if $\left(  i,j\right)  =\left(  u,v\right)  $ and $0$
otherwise. In formulas, this says that%
\begin{equation}
\left(  E_{u,v}\right)  _{i,j}=%
\begin{cases}
1, & \text{if }\left(  i,j\right)  =\left(  u,v\right)  ;\\
0, & \text{if }\left(  i,j\right)  \neq\left(  u,v\right)
\end{cases}
. \label{pf.prop.Euv.deltas.1}%
\end{equation}


On the other hand, the definition of $\delta_{\left(  i,j\right)  ,\left(
u,v\right)  }$ yields%
\begin{equation}
\delta_{\left(  i,j\right)  ,\left(  u,v\right)  }=%
\begin{cases}
1, & \text{if }\left(  i,j\right)  =\left(  u,v\right)  ;\\
0, & \text{if }\left(  i,j\right)  \neq\left(  u,v\right)
\end{cases}
. \label{pf.prop.Euv.deltas.2}%
\end{equation}
Comparing this with (\ref{pf.prop.Euv.deltas.1}), we immediately obtain
$\left(  E_{u,v}\right)  _{i,j}=\delta_{\left(  i,j\right)  ,\left(
u,v\right)  }$.

Let us now show that
\begin{equation}
\delta_{\left(  i,j\right)  ,\left(  u,v\right)  }=\delta_{i,u}\delta_{j,v}.
\label{pf.prop.Euv.deltas.5}%
\end{equation}


[\textit{Proof of (\ref{pf.prop.Euv.deltas.5}):} We are in one of the
following three cases:

\begin{statement}
\textit{Case 1:} We have $i\neq u$.
\end{statement}

\begin{statement}
\textit{Case 2:} We have $j\neq v$.
\end{statement}

\begin{statement}
\textit{Case 3:} We have neither $i\neq u$ nor $j\neq v$.
\end{statement}

(In fact, it is possible that we are in Case 1 and Case 2 simultaneously. But
this does not invalidate our proof; it is perfectly fine if the cases
\textquotedblleft overlap\textquotedblright, as long as every possible
situation is covered by at least one case.)

We shall prove (\ref{pf.prop.Euv.deltas.5}) in each of the three cases:

\begin{enumerate}
\item Let us first consider Case 1. In this case, we have $i\neq u$. Hence,
$\left(  i,j\right)  \neq\left(  u,v\right)  $. Thus, $\delta_{\left(
i,j\right)  ,\left(  u,v\right)  }=0$. Comparing this with $\underbrace{\delta
_{i,u}}_{\substack{=0\\\text{(since }i\neq u\text{)}}}\delta_{j,v}%
=0\delta_{j,v}=0$, we find $\delta_{\left(  i,j\right)  ,\left(  u,v\right)
}=\delta_{i,u}\delta_{j,v}$. Hence, (\ref{pf.prop.Euv.deltas.5}) is proven in
Case 1.

\item Let us next consider Case 2. In this case, we have $j\neq v$. Hence,
$\left(  i,j\right)  \neq\left(  u,v\right)  $. Thus, $\delta_{\left(
i,j\right)  ,\left(  u,v\right)  }=0$. Comparing this with $\delta
_{i,u}\underbrace{\delta_{j,v}}_{\substack{=0\\\text{(since }j\neq v\text{)}%
}}=\delta_{i,u}0=0$, we find $\delta_{\left(  i,j\right)  ,\left(  u,v\right)
}=\delta_{i,u}\delta_{j,v}$. Hence, (\ref{pf.prop.Euv.deltas.5}) is proven in
Case 2.

\item Let us finally consider Case 3. In this case, we have neither $i\neq u$
and $j\neq v$. Hence, $i=u$ (since not $i\neq u$) and $j=v$ (since not $j\neq
v$). As a consequence, $\left(  i,j\right)  =\left(  u,v\right)  $, so that
$\delta_{\left(  i,j\right)  ,\left(  u,v\right)  }=1$. Comparing this with
$\underbrace{\delta_{i,u}}_{\substack{=1\\\text{(since }i=u\text{)}%
}}\underbrace{\delta_{j,v}}_{\substack{=1\\\text{(since }j=v\text{)}}%
}=1\cdot1=1$, we find $\delta_{\left(  i,j\right)  ,\left(  u,v\right)
}=\delta_{i,u}\delta_{j,v}$. Hence, (\ref{pf.prop.Euv.deltas.5}) is proven in
Case 3.
\end{enumerate}

We have now proven (\ref{pf.prop.Euv.deltas.5}) in all three Cases; this
finishes our proof of (\ref{pf.prop.Euv.deltas.5}).]

Now, we have shown both $\left(  E_{u,v}\right)  _{i,j}=\delta_{\left(
i,j\right)  ,\left(  u,v\right)  }$ and $\delta_{\left(  i,j\right)  ,\left(
u,v\right)  }=\delta_{i,u}\delta_{j,v}$. Combining these two equalities gives
us (\ref{eq.prop.Euv.deltas.1}), and so Proposition \ref{prop.Euv.deltas} is proven.
\end{proof}

We can now easily prove Proposition \ref{prop.Euv.laction}:

\begin{proof}
[Proof of Proposition \ref{prop.Euv.laction}.]We need to prove the following
two claims:

\begin{statement}
\textit{Claim 1:} The $u$-th row of the $n\times p$-matrix $E_{u,v}C$ is the
$v$-th row of $C$.
\end{statement}

\begin{statement}
\textit{Claim 2:} For each $i\in\left\{  1,2,\ldots,n\right\}  $ satisfying
$i\neq u$, the $i$-th row of the $n\times p$-matrix $E_{u,v}C$ is filled with zeroes.
\end{statement}

Before we do so, let us derive a few formulas. First of all, we have%
\begin{equation}
\left(  E_{u,v}\right)  _{i,j}=\delta_{i,u}\delta_{j,v}
\label{pf.prop.Euv.laction.1}%
\end{equation}
for all $i\in\left\{  1,2,\ldots,n\right\}  $ and $j\in\left\{  1,2,\ldots
,m\right\}  $ (according to Proposition \ref{prop.Euv.deltas}). Furthermore,
for any $i\in\left\{  1,2,\ldots,n\right\}  $ and $j\in\left\{  1,2,\ldots
,p\right\}  $, we have%
\begin{align}
\left(  E_{u,v}C\right)  _{i,j}  &  =\sum_{k=1}^{m}\underbrace{\left(
E_{u,v}\right)  _{i,k}}_{\substack{=\delta_{i,u}\delta_{k,v}\\\text{(by
(\ref{pf.prop.Euv.laction.1}), applied to}\\k\text{ instead of }j\text{)}%
}}C_{k,j}\ \ \ \ \ \ \ \ \ \ \left(
\begin{array}
[c]{c}%
\text{by Proposition \ref{prop.AB},}\\
\text{applied to }A=E_{u,v}\text{ and }B=C
\end{array}
\right) \nonumber\\
&  =\sum_{k=1}^{m}\delta_{i,u}\delta_{k,v}C_{k,j}=\sum_{k=1}^{m}\delta
_{i,u}C_{k,j}\delta_{k,v}=\delta_{i,u}C_{v,j} \label{pf.prop.Euv.laction.2}%
\end{align}
(by Proposition \ref{prop.sum.delta}, applied to $1$, $m$, $v$ and
$\delta_{i,u}C_{k,j}$ instead of $p$, $q$, $r$ and $a_{k}$). Now, we can prove
both claims easily:

\textit{Proof of Claim 1:} For every $j\in\left\{  1,2,\ldots,p\right\}  $, we
have%
\begin{align*}
\left(  E_{u,v}C\right)  _{u,j}  &  =\underbrace{\delta_{u,u}}%
_{\substack{=1\\\text{(since }u=u\text{)}}}C_{v,j}\ \ \ \ \ \ \ \ \ \ \left(
\text{by (\ref{pf.prop.Euv.laction.2}), applied to }i=u\right) \\
&  =C_{v,j}.
\end{align*}
In other words, for every $j\in\left\{  1,2,\ldots,p\right\}  $, the $\left(
u,j\right)  $-th entry of the matrix $E_{u,v}C$ equals the $\left(
v,j\right)  $-th entry of the matrix $C$. In other words, each entry of the
$u$-th row of $E_{u,v}C$ equals the corresponding entry of the $v$-th row of
$C$. In other words, the $u$-th row of the $n\times p$-matrix $E_{u,v}C$ is
the $v$-th row of $C$. This proves Claim 1.

\textit{Proof of Claim 2:} Let $i\in\left\{  1,2,\ldots,n\right\}  $ be such
that $i\neq u$. We must prove that the $i$-th row of the $n\times p$-matrix
$E_{u,v}C$ is filled with zeroes.

For every $j\in\left\{  1,2,\ldots,p\right\}  $, we have%
\begin{align*}
\left(  E_{u,v}C\right)  _{i,j}  &  =\underbrace{\delta_{i,u}}%
_{\substack{=0\\\text{(since }i\neq u\text{)}}}C_{v,j}%
\ \ \ \ \ \ \ \ \ \ \left(  \text{by (\ref{pf.prop.Euv.laction.2})}\right) \\
&  =0C_{v,j}=0.
\end{align*}
In other words, for every $j\in\left\{  1,2,\ldots,p\right\}  $, the $\left(
i,j\right)  $-th entry of the matrix $E_{u,v}C$ is $0$. In other words, each
entry of the $i$-th row of $E_{u,v}C$ is $0$. In other words, the $i$-th row
of the $n\times p$-matrix $E_{u,v}C$ is filled with zeroes. This proves Claim 2.

Now, both Claim 1 and Claim 2 are proven, and so we are finished proving
Proposition \ref{prop.Euv.laction}.
\end{proof}

The proof of Proposition \ref{prop.Euv.raction} is similar, and is left to the reader.

Here is another simple property of matrix units:

\begin{proposition}
\label{prop.Euv.transpose}Let $n\in\mathbb{N}$ and $m\in\mathbb{N}$. Let
$u\in\left\{  1,2,\ldots,n\right\}  $ and $v\in\left\{  1,2,\ldots,m\right\}
$. Then, $\left(  E_{u,v}\right)  ^{T}=E_{v,u}$. (More precisely: $\left(
E_{u,v,n,m}\right)  ^{T}=E_{v,u,m,n}$.)
\end{proposition}

The proof of Proposition \ref{prop.Euv.transpose} is easy enough that it could
be left as an exercise at this point, but let me nevertheless give it for the
sake of completeness:

\begin{proof}
[Proof of Proposition \ref{prop.Euv.transpose}.]We first notice that both
$\left(  E_{u,v,n,m}\right)  ^{T}$ and $E_{v,u,m,n}$ are $m\times n$-matrices.

For every $i\in\left\{  1,2,\ldots,n\right\}  $ and $j\in\left\{
1,2,\ldots,m\right\}  $, we have%
\begin{equation}
\left(  E_{u,v,n,m}\right)  _{i,j}=\delta_{\left(  i,j\right)  ,\left(
u,v\right)  }=\delta_{i,u}\delta_{j,v}. \label{pf.prop.Euv.transpose.1}%
\end{equation}
(Indeed, this is just a more precise way to state (\ref{eq.prop.Euv.deltas.1}%
), because the \textquotedblleft$E_{u,v}$\textquotedblright\ in
(\ref{eq.prop.Euv.deltas.1}) is a shorthand for \textquotedblleft$E_{u,v,n,m}%
$\textquotedblright.) Thus, in particular, every $i\in\left\{  1,2,\ldots
,n\right\}  $ and $j\in\left\{  1,2,\ldots,m\right\}  $ satisfy%
\begin{equation}
\left(  E_{u,v,n,m}\right)  _{i,j}=\delta_{i,u}\delta_{j,v}.
\label{pf.prop.Euv.transpose.2}%
\end{equation}
(This is an immediate consequence of (\ref{pf.prop.Euv.transpose.1}).)

The definition of the transpose $\left(  E_{u,v,n,m}\right)  ^{T}$ now yields
$\left(  E_{u,v,n,m}\right)  ^{T}=\left(  \left(  E_{u,v,n,m}\right)
_{j,i}\right)  _{1\leq i\leq m,\ 1\leq j\leq n}$. Hence, for every
$i\in\left\{  1,2,\ldots,m\right\}  $ and $j\in\left\{  1,2,\ldots,n\right\}
$, we have%
\begin{equation}
\left(  \left(  E_{u,v,n,m}\right)  ^{T}\right)  _{i,j}=\left(  E_{u,v,n,m}%
\right)  _{j,i}=\delta_{j,u}\delta_{i,v} \label{pf.prop.Euv.transpose.3}%
\end{equation}
(by (\ref{pf.prop.Euv.transpose.2}), applied to $j$ and $i$ instead of $i$ and
$j$).

On the other hand, the same reasoning that we used to obtain
(\ref{pf.prop.Euv.transpose.2}) can be applied to $m$, $n$, $v$ and $u$
instead of $n$, $m$, $u$ and $v$. As a result of this reasoning, we then find
that every $i\in\left\{  1,2,\ldots,m\right\}  $ and $j\in\left\{
1,2,\ldots,n\right\}  $ satisfy%
\begin{equation}
\left(  E_{v,u,m,n}\right)  _{i,j}=\delta_{i,v}\delta_{j,u}.
\label{pf.prop.Euv.transpose.4}%
\end{equation}
Hence, for every $i\in\left\{  1,2,\ldots,m\right\}  $ and $j\in\left\{
1,2,\ldots,n\right\}  $, we have%
\begin{align*}
\left(  \left(  E_{u,v,n,m}\right)  ^{T}\right)  _{i,j}  &  =\delta
_{j,u}\delta_{i,v}\ \ \ \ \ \ \ \ \ \ \left(  \text{by
(\ref{pf.prop.Euv.transpose.3})}\right) \\
&  =\delta_{i,v}\delta_{j,u}=\left(  E_{v,u,m,n}\right)  _{i,j}%
\ \ \ \ \ \ \ \ \ \ \left(  \text{by (\ref{pf.prop.Euv.transpose.4})}\right)
.
\end{align*}
In other words, for every $i\in\left\{  1,2,\ldots,m\right\}  $ and
$j\in\left\{  1,2,\ldots,n\right\}  $, the $\left(  i,j\right)  $-th entry of
the matrix $\left(  E_{u,v,n,m}\right)  ^{T}$ equals the $\left(  i,j\right)
$-th entry of the matrix $E_{v,u,m,n}$. In other words, each entry of the
matrix $\left(  E_{u,v,n,m}\right)  ^{T}$ equals the corresponding entry of
the matrix $E_{v,u,m,n}$. Hence, $\left(  E_{u,v,n,m}\right)  ^{T}%
=E_{v,u,m,n}$. Using our shorthand notations, this rewrites as $\left(
E_{u,v}\right)  ^{T}=E_{v,u}$. Thus, Proposition \ref{prop.Euv.transpose} is proven.
\end{proof}

As I have said, the standard matrix units are building blocks for matrices:
every matrix can be obtained from them by scaling and adding. More precisely:

\begin{proposition}
\label{prop.Euv.A=sum}Let $n\in\mathbb{N}$ and $m\in\mathbb{N}$. Let $A$ be an
$n\times m$-matrix. Then,%
\[
A=\sum_{u=1}^{n}\sum_{v=1}^{m}A_{u,v}E_{u,v}.
\]
Here, we are using the $\sum$ symbol introduced in Section
\ref{sect.intro.sum-matrices}. (Of course, $E_{u,v}$ means $E_{u,v,n,m}$ here.)
\end{proposition}

\begin{example}
In the case when $n=2$ and $m=2$, Proposition \ref{prop.Euv.A=sum} says that%
\[
A=A_{1,1}E_{1,1}+A_{1,2}E_{1,2}+A_{2,1}E_{2,1}+A_{2,2}E_{2,2}.
\]
It is easy to check this directly:%
\begin{align*}
&  A_{1,1}E_{1,1}+A_{1,2}E_{1,2}+A_{2,1}E_{2,1}+A_{2,2}E_{2,2}\\
&  =A_{1,1}\left(
\begin{array}
[c]{cc}%
1 & 0\\
0 & 0
\end{array}
\right)  +A_{1,2}\left(
\begin{array}
[c]{cc}%
0 & 1\\
0 & 0
\end{array}
\right)  +A_{2,1}\left(
\begin{array}
[c]{cc}%
0 & 0\\
1 & 0
\end{array}
\right)  +A_{2,2}\left(
\begin{array}
[c]{cc}%
0 & 0\\
0 & 1
\end{array}
\right) \\
&  =\left(
\begin{array}
[c]{cc}%
A_{1,1} & 0\\
0 & 0
\end{array}
\right)  +\left(
\begin{array}
[c]{cc}%
0 & A_{1,2}\\
0 & 0
\end{array}
\right)  +\left(
\begin{array}
[c]{cc}%
0 & 0\\
A_{2,1} & 0
\end{array}
\right)  +\left(
\begin{array}
[c]{cc}%
0 & 0\\
0 & A_{2,2}%
\end{array}
\right) \\
&  =\left(
\begin{array}
[c]{cc}%
A_{1,1} & A_{1,2}\\
A_{2,1} & A_{2,2}%
\end{array}
\right)  =A.
\end{align*}
This should make the truth of Proposition \ref{prop.Euv.A=sum} obvious (even
in the general case). The proof below is just for the fans of formal reasoning.
\end{example}

\begin{proof}
[Proof of Proposition \ref{prop.Euv.A=sum}.]For every $i\in\left\{
1,2,\ldots,n\right\}  $ and $j\in\left\{  1,2,\ldots,m\right\}  $, we have%
\begin{align*}
&  \left(  \sum_{u=1}^{n}\sum_{v=1}^{m}A_{u,v}E_{u,v}\right)  _{i,j}\\
&  =\sum_{u=1}^{n}\underbrace{\left(  \sum_{v=1}^{m}A_{u,v}E_{u,v}\right)
_{i,j}}_{\substack{=\sum_{v=1}^{m}\left(  A_{u,v}E_{u,v}\right)
_{i,j}\\\text{(by an application of}\\\text{Proposition
\ref{prop.sum.matrices.multi})}}}\ \ \ \ \ \ \ \ \ \ \left(  \text{by an
application of Proposition \ref{prop.sum.matrices.multi}}\right) \\
&  =\sum_{u=1}^{n}\sum_{v=1}^{m}\underbrace{\left(  A_{u,v}E_{u,v}\right)
_{i,j}}_{\substack{=A_{u,v}\left(  E_{u,v}\right)  _{i,j}\\\text{(since
matrices are scaled entry by entry)}}}\\
&  =\sum_{u=1}^{n}\sum_{v=1}^{m}A_{u,v}\underbrace{\left(  E_{u,v}\right)
_{i,j}}_{\substack{=\delta_{i,u}\delta_{j,v}\\\text{(by
(\ref{eq.prop.Euv.deltas.1}))}}}=\sum_{u=1}^{n}\sum_{v=1}^{m}A_{u,v}%
\underbrace{\delta_{i,u}}_{=\delta_{u,i}}\underbrace{\delta_{j,v}}%
_{=\delta_{v,j}}\\
&  =\sum_{u=1}^{n}\underbrace{\sum_{v=1}^{m}A_{u,v}\delta_{u,i}\delta_{v,j}%
}_{\substack{=\sum_{k=1}^{m}A_{u,k}\delta_{u,i}\delta_{k,j}\\=A_{u,j}%
\delta_{u,i}\\\text{(by Proposition \ref{prop.sum.delta}, applied
to}\\p=1\text{, }q=m\text{, }r=j\text{ and }a_{k}=A_{u,k}\delta_{u,i}\text{)}%
}}=\sum_{u=1}^{n}A_{u,j}\delta_{u,i}=\sum_{k=1}^{n}A_{k,j}\delta_{k,i}\\
&  =A_{i,j}\ \ \ \ \ \ \ \ \ \ \left(  \text{by Proposition
\ref{prop.sum.delta}, applied to }p=1\text{, }q=n\text{, }r=i\text{ and }%
a_{k}=A_{k,j}\right)  .
\end{align*}
In other words, each entry of the matrix $\sum_{u=1}^{n}\sum_{v=1}^{m}%
A_{u,v}E_{u,v}$ equals the corresponding entry of the matrix $A$. Thus,
$\sum_{u=1}^{n}\sum_{v=1}^{m}A_{u,v}E_{u,v}=A$. This proves Proposition
\ref{prop.Euv.A=sum}.
\end{proof}

Continuing the flow of boring and straightforward little propositions, let us
see how standard matrix units multiply:

\begin{proposition}
\label{prop.Euv.prod}Let $n\in\mathbb{N}$, $m\in\mathbb{N}$ and $p\in
\mathbb{N}$. Let $u\in\left\{  1,2,\ldots,n\right\}  $, $v\in\left\{
1,2,\ldots,m\right\}  $, $x\in\left\{  1,2,\ldots,m\right\}  $ and
$y\in\left\{  1,2,\ldots,p\right\}  $. Then,
\[
E_{u,v,n,m}E_{x,y,m,p}=\delta_{v,x}E_{u,y,n,p}.
\]
(We shall write this equality as $E_{u,v}E_{x,y}=\delta_{v,x}E_{u,y}$, since
we hope that the sizes of the matrices will be clear from the context.)
\end{proposition}

It might be a good exercise to devise at least two examples for this
proposition (one with $v=x$ and one with $v\neq x$), and to prove it.
Nevertheless, let me give a proof, because noone ever seems to do so in writing:

\begin{proof}
[Proof of Proposition \ref{prop.Euv.prod}.]There are two ways to prove
Proposition \ref{prop.Euv.prod}: One is to apply Proposition
\ref{prop.Euv.laction} to $C=E_{x,y}$. Another is to obstinately applying the
definitions. We opt for the second way, because the first is too simple.

From (\ref{eq.prop.Euv.deltas.1}), we obtain%
\begin{equation}
\left(  E_{u,v,n,m}\right)  _{i,j}=\delta_{i,u}\delta_{j,v}
\label{pf.prop.Euv.prod.1}%
\end{equation}
for all $i\in\left\{  1,2,\ldots,n\right\}  $ and $j\in\left\{  1,2,\ldots
,m\right\}  $. (If this doesn't look like (\ref{eq.prop.Euv.deltas.1}) to you,
do remember that $E_{u,v,n,m}$ was abbreviated as $E_{u,v}$ in
(\ref{eq.prop.Euv.deltas.1}).)

The same reasoning (applied to $m$, $p$, $x$ and $y$ instead of $n$, $m$, $u$
and $v$) shows that%
\begin{equation}
\left(  E_{x,y,m,p}\right)  _{i,j}=\delta_{i,x}\delta_{j,y}
\label{pf.prop.Euv.prod.2}%
\end{equation}
for all $i\in\left\{  1,2,\ldots,m\right\}  $ and $j\in\left\{  1,2,\ldots
,p\right\}  $.

The same reasoning that gave us (\ref{pf.prop.Euv.prod.1}) can also be applied
to $n$, $p$, $u$ and $y$ instead of $n$, $m$, $u$ and $v$. As a result, we
find%
\begin{equation}
\left(  E_{u,y,n,p}\right)  _{i,j}=\delta_{i,u}\delta_{j,y}
\label{pf.prop.Euv.prod.3}%
\end{equation}
for all $i\in\left\{  1,2,\ldots,n\right\}  $ and $j\in\left\{  1,2,\ldots
,p\right\}  $.

Now, let $i\in\left\{  1,2,\ldots,n\right\}  $ and $j\in\left\{
1,2,\ldots,p\right\}  $. Then, Proposition \ref{prop.AB} (applied to
$A=E_{u,v,n,m}$ and $B=E_{x,y,m,p}$) shows that%
\begin{align*}
\left(  E_{u,v,n,m}E_{x,y,m,p}\right)  _{i,j}  &  =\sum_{k=1}^{m}%
\underbrace{\left(  E_{u,v,n,m}\right)  _{i,k}}_{\substack{=\delta_{i,u}%
\delta_{k,v}\\\text{(by (\ref{pf.prop.Euv.prod.1}), applied to }%
k\\\text{instead of }j\text{)}}}\underbrace{\left(  E_{x,y,m,p}\right)
_{k,j}}_{\substack{=\delta_{k,x}\delta_{j,y}\\\text{(by
(\ref{pf.prop.Euv.prod.2}), applied to }k\\\text{instead of }i\text{)}}}\\
&  =\sum_{k=1}^{m}\delta_{i,u}\delta_{k,v}\delta_{k,x}\delta_{j,y}=\sum
_{k=1}^{m}\delta_{i,u}\delta_{k,x}\delta_{j,y}\delta_{k,v}\\
&  =\delta_{i,u}\delta_{v,x}\delta_{j,y}%
\end{align*}
(by Proposition \ref{prop.sum.delta}, applied to $1$, $m$, $v$ and
$\delta_{i,u}\delta_{k,x}\delta_{j,y}$ instead of $p$, $q$, $r$ and $a_{k}$).
Comparing this with%
\begin{align*}
\left(  \delta_{v,x}E_{u,y,n,p}\right)  _{i,j}  &  =\delta_{v,x}%
\underbrace{\left(  E_{u,y,n,p}\right)  _{i,j}}_{\substack{=\delta_{i,u}%
\delta_{j,y}\\\text{(by (\ref{pf.prop.Euv.prod.3}))}}}\\
&  \ \ \ \ \ \ \ \ \ \ \left(  \text{since matrices are scaled entry by
entry}\right) \\
&  =\delta_{v,x}\delta_{i,u}\delta_{j,y}=\delta_{i,u}\delta_{v,x}\delta_{j,y},
\end{align*}
we find $\left(  E_{u,v,n,m}E_{x,y,m,p}\right)  _{i,j}=\left(  \delta
_{v,x}E_{u,y,n,p}\right)  _{i,j}$.

Now, forget that we fixed $i$ and $j$. We thus have shown that $\left(
E_{u,v,n,m}E_{x,y,m,p}\right)  _{i,j}=\left(  \delta_{v,x}E_{u,y,n,p}\right)
_{i,j}$ for all $i\in\left\{  1,2,\ldots,n\right\}  $ and $j\in\left\{
1,2,\ldots,p\right\}  $. In other words, each entry of the matrix
$E_{u,v,n,m}E_{x,y,m,p}$ equals the corresponding entry of the matrix
$\delta_{v,x}E_{u,y,n,p}$. Thus, $E_{u,v,n,m}E_{x,y,m,p}=\delta_{v,x}%
E_{u,y,n,p}$. This proves Proposition \ref{prop.Euv.prod}.
\end{proof}

\subsection{\label{sect.gauss.Alamuv}The $\lambda$-addition matrices
$A_{u,v}^{\lambda}$}

Now, we come to another important class of matrices (that can also be seen as
building blocks of a kind): the $\lambda$\textit{-addition matrices}. Those
are square matrices, and are defined as follows:

\begin{definition}
\label{def.Alamuv}Let $n\in\mathbb{N}$. Let $u$ and $v$ be two distinct
elements of $\left\{  1,2,\ldots,n\right\}  $. Let $\lambda$ be a number.
Then, $A_{u,v}^{\lambda}$ shall denote the $n\times n$-matrix $I_{n}+\lambda
E_{u,v}$ (where $E_{u,v}$ means the $n\times n$-matrix $E_{u,v}$, that is,
$E_{u,v,n,n}$).

A few remarks about the notation:

\textbf{(a)} The superscript $\lambda$ in the notation \textquotedblleft%
$A_{u,v}^{\lambda}$\textquotedblright\ is not an exponent; i.e., the matrix
$A_{u,v}^{\lambda}$ is not the $\lambda$-th power of some matrix $A_{u,v}$.
Instead, it is just an argument that we have chosen to write as a superscript
instead of a subscript. So the role of the $\lambda$ in \textquotedblleft%
$A_{u,v}^{\lambda}$\textquotedblright\ is completely different from the role
of the $-1$ in \textquotedblleft$A_{1}^{-1}$\textquotedblright\ in Proposition
\ref{prop.inverses.A1Ak}.

\textbf{(b)} To be really precise, we ought to denote $A_{u,v}^{\lambda}$ by
$A_{u,v,n}^{\lambda}$, because it depends on $n$. (This is similar to how we
ought to denote $E_{u,v}$ by $E_{u,v,n,n}$.) But the $n$ will be really clear
from the context almost every time we deal with these matrices, so we shall
keep it out of our notation.

\textbf{(c)} The notation $A_{u,v}^{\lambda}$ is not standard in the
literature, but I will use this notation in the following.
\end{definition}

\begin{example}
\label{exam.Alamuv}Let $n=4$. Then,%
\begin{align*}
A_{1,3}^{\lambda}  &  =I_{n}+\lambda E_{1,3}=\left(
\begin{array}
[c]{cccc}%
1 & 0 & \lambda & 0\\
0 & 1 & 0 & 0\\
0 & 0 & 1 & 0\\
0 & 0 & 0 & 1
\end{array}
\right)  \ \ \ \ \ \ \ \ \ \ \text{and}\\
A_{4,2}^{\lambda}  &  =I_{n}+\lambda E_{4,2}=\left(
\begin{array}
[c]{cccc}%
1 & 0 & 0 & 0\\
0 & 1 & 0 & 0\\
0 & 0 & 1 & 0\\
0 & \lambda & 0 & 1
\end{array}
\right)  .
\end{align*}

\end{example}

The pattern that you see on these examples is true in general:

\begin{proposition}
\label{prop.Alamuv.entries}Let $n\in\mathbb{N}$. Let $u$ and $v$ be two
distinct elements of $\left\{  1,2,\ldots,n\right\}  $. Let $\lambda$ be a
number. Then, the matrix $A_{u,v}^{\lambda}$ has the following entries:

\begin{itemize}
\item All its diagonal entries are $1$.

\item Its $\left(  u,v\right)  $-th entry is $\lambda$.

\item All its remaining entries are $0$.
\end{itemize}
\end{proposition}

\begin{proof}
[Proof of Proposition \ref{prop.Alamuv.entries}.]Recall that $E_{u,v}$ is the
$n\times n$-matrix whose $\left(  u,v\right)  $-th entry is $1$ and whose all
other entries are $0$ (indeed, this is how $E_{u,v}$ was defined). Since
matrices are scaled entry by entry, we can therefore conclude how $\lambda
E_{u,v}$ looks like: Namely, $\lambda E_{u,v}$ is the $n\times n$-matrix whose
$\left(  u,v\right)  $-th entry is $\lambda\cdot1=\lambda$ and whose all other
entries are $\lambda\cdot0=0$. Thus, we know the following:

\begin{itemize}
\item The matrix $I_{n}$ is the $n\times n$-matrix whose diagonal entries are
$1$, and whose all other entries are $0$.

\item The matrix $\lambda E_{u,v}$ is the $n\times n$-matrix whose $\left(
u,v\right)  $-th entry is $\lambda$, and whose all other entries are $0$.
\end{itemize}

Since matrices are added entry by entry, we can thus infer how $I_{n}+\lambda
E_{u,v}$ looks like: Namely, the matrix $I_{n}+\lambda E_{u,v}$ is the
$n\times n$-matrix whose diagonal entries\footnote{Here, we are using the fact
that the $\left(  u,v\right)  $-th entry is not a diagonal entry; this is
because $u$ and $v$ are distinct! If $u$ and $v$ were equal, then the $\left(
u,v\right)  $-th entry of $I_{n}+\lambda E_{u,v}$ would be $1+\lambda$
instead.} are $1+0=1$, whose $\left(  u,v\right)  $-th entry is $0+\lambda
=\lambda$, and whose all other entries are $0+0=0$. Since $I_{n}+\lambda
E_{u,v}=A_{u,v}^{\lambda}$, this rewrites as follows: The matrix
$A_{u,v}^{\lambda}$ is the $n\times n$-matrix whose diagonal entries are $1$,
whose $\left(  u,v\right)  $-th entry is $\lambda$, and whose all other
entries are $0$. This proves Proposition \ref{prop.Alamuv.entries}.
\end{proof}

We can next see what happens to a matrix when it is multiplied by
$A_{u,v}^{\lambda}$:

\begin{proposition}
\label{prop.Alamuv.laction}Let $n\in\mathbb{N}$ and $m\in\mathbb{N}$. Let $u$
and $v$ be two distinct elements of $\left\{  1,2,\ldots,n\right\}  $. Let
$\lambda$ be a number. Let $C$ be an $n\times m$-matrix. Then, $A_{u,v}%
^{\lambda}C$ is the $n\times m$-matrix obtained from $C$ by adding
$\lambda\operatorname*{row}\nolimits_{v}C$ to the $u$-th row.

(Recall that $\operatorname*{row}\nolimits_{k}C$ denotes the $k$-th row of $C$
for each $k$. Recall also that the rows of $C$ are row vectors, and thus are
added and scaled entry by entry. Hence, adding $\lambda\operatorname*{row}%
\nolimits_{v}C$ to the $u$-th row means adding $\lambda$ times each entry of
the $v$-th row of $C$ to the corresponding entry of the $u$-th row.)
\end{proposition}

\begin{example}
\label{exam.prop.Alamuv.laction}Let $n=3$ and $m=2$. Let $C$ be the $3\times
2$-matrix $\left(
\begin{array}
[c]{cc}%
a & b\\
a^{\prime} & b^{\prime}\\
a^{\prime\prime} & b^{\prime\prime}%
\end{array}
\right)  $. Let $\lambda$ be a number. Then, Proposition
\ref{prop.Alamuv.laction} (applied to $u=2$ and $v=1$) claims that
$A_{2,1}^{\lambda}C$ is the $3\times2$-matrix obtained from $C$ by adding
$\lambda\operatorname*{row}\nolimits_{1}C$ to the $2$-nd row. A computation
confirms this claim:%
\[
A_{2,1}^{\lambda}C=\left(
\begin{array}
[c]{ccc}%
1 & 0 & 0\\
\lambda & 1 & 0\\
0 & 0 & 1
\end{array}
\right)  \left(
\begin{array}
[c]{cc}%
a & b\\
a^{\prime} & b^{\prime}\\
a^{\prime\prime} & b^{\prime\prime}%
\end{array}
\right)  =\left(
\begin{array}
[c]{cc}%
a & b\\
a^{\prime}+\lambda a & b^{\prime}+\lambda b\\
a^{\prime\prime} & b^{\prime\prime}%
\end{array}
\right)  .
\]

\end{example}

\begin{proposition}
\label{prop.Alamuv.raction}Let $n\in\mathbb{N}$ and $m\in\mathbb{N}$. Let $u$
and $v$ be two distinct elements of $\left\{  1,2,\ldots,n\right\}  $. Let
$\lambda$ be a number. Let $C$ be an $m\times n$-matrix. Then, $CA_{u,v}%
^{\lambda}$ is the $m\times n$-matrix obtained from $C$ by adding
$\lambda\operatorname*{col}\nolimits_{u}C$ to the $v$-th column.
\end{proposition}

Note how Proposition \ref{prop.Alamuv.raction} differs from Proposition
\ref{prop.Alamuv.laction}: not only have rows been replaced by columns, but
also have $u$ and $v$ switched roles.

You might find Example \ref{exam.prop.Alamuv.laction} sufficient to convince
you of the truth of Proposition \ref{prop.Alamuv.laction}. If not, a proof
will be given in Section \ref{sect.gauss.Alamuv.proofs} below.

We shall refer to the matrix $A_{u,v}^{\lambda}$ defined in Definition
\ref{def.Alamuv} as a \textquotedblleft$\lambda$-addition
matrix\textquotedblright; it is one of three kinds of matrices that are called
elementary matrices. We shall learn about the other two kinds below.

Here are a few more properties of $\lambda$-addition matrices:

\begin{proposition}
\label{prop.Alamuv.transpose}Let $n\in\mathbb{N}$. Let $u$ and $v$ be two
distinct elements of $\left\{  1,2,\ldots,n\right\}  $. Let $\lambda$ be a
number. Then, $\left(  A_{u,v}^{\lambda}\right)  ^{T}=A_{v,u}^{\lambda}$.
\end{proposition}

\begin{proposition}
\label{prop.Alamuv.lambda+mu}Let $n\in\mathbb{N}$. Let $u$ and $v$ be two
distinct elements of $\left\{  1,2,\ldots,n\right\}  $.

\textbf{(a)} We have $A_{u,v}^{0}=I_{n}$.

\textbf{(b)} If $\lambda$ and $\mu$ are two numbers, then $A_{u,v}^{\lambda
}A_{u,v}^{\mu}=A_{u,v}^{\lambda+\mu}$.

\textbf{(c)} Let $\lambda$ be a number. Then, the matrix $A_{u,v}^{\lambda}$
is invertible, and its inverse is $\left(  A_{u,v}^{\lambda}\right)
^{-1}=A_{u,v}^{-\lambda}$.
\end{proposition}

A proof of this proposition will also be given in Section
\ref{sect.gauss.Alamuv.proofs}.

\subsection{\label{sect.gauss.Alamuv.proofs}(*) Some proofs about the
$\lambda$-addition matrices}

\begin{proof}
[Proof of Proposition \ref{prop.Alamuv.laction}.]Clearly, $A_{u,v}^{\lambda}C$
is an $n\times m$-matrix.

Proposition \ref{prop.Euv.laction} (applied to $n$ and $m$ instead of $m$ and
$p$) shows that $E_{u,v}C$ is the $n\times m$-matrix whose $u$-th row is the
$v$-th row of $C$, and whose all other rows are filled with zeroes. Thus,%
\begin{equation}
\operatorname*{row}\nolimits_{u}\left(  E_{u,v}C\right)  =\operatorname*{row}%
\nolimits_{v}C \label{pf.prop.Alamuv.laction.1}%
\end{equation}
(since the $u$-the row of $E_{u,v}C$ is the $v$-th row of $C$) and%
\begin{equation}
\operatorname*{row}\nolimits_{i}\left(  E_{u,v}C\right)  =0_{1\times
m}\ \ \ \ \ \ \ \ \ \ \text{for every }i\in\left\{  1,2,\ldots,n\right\}
\text{ satisfying }i\neq u \label{pf.prop.Alamuv.laction.2}%
\end{equation}
(since all other rows of $E_{u,v}C$ are filled with zeroes).

But recall that matrices are added entry by entry. Thus, matrices are also
added by row by row -- i.e., if $U$ and $V$ are two $n\times m$-matrices, then
any row of $U+V$ is the sum of the corresponding rows of $U$ and of $V$. In
other words, if $U$ and $V$ are two $n\times m$-matrices, then%
\begin{equation}
\operatorname*{row}\nolimits_{i}\left(  U+V\right)  =\operatorname*{row}%
\nolimits_{i}U+\operatorname*{row}\nolimits_{i}V\ \ \ \ \ \ \ \ \ \ \text{for
every }i\in\left\{  1,2,\ldots,n\right\}  . \label{pf.prop.Alamuv.laction.5}%
\end{equation}
Also, if $U$ is an $n\times m$-matrix, then%
\begin{equation}
\operatorname*{row}\nolimits_{i}\left(  \lambda U\right)  =\lambda
\operatorname*{row}\nolimits_{i}U\ \ \ \ \ \ \ \ \ \ \text{for every }%
i\in\left\{  1,2,\ldots,n\right\}  \label{pf.prop.Alamuv.laction.6}%
\end{equation}
(since matrices are scaled entry by entry).

We have%
\[
\underbrace{A_{u,v}^{\lambda}}_{=I_{n}+\lambda E_{u,v}}C=\left(  I_{n}+\lambda
E_{u,v}\right)  C=\underbrace{I_{n}C}_{=C}+\lambda E_{u,v}C=C+\lambda
E_{u,v}C.
\]
Hence, for each $i\in\left\{  1,2,\ldots,n\right\}  $, we have%
\begin{align}
\operatorname*{row}\nolimits_{i}\left(  \underbrace{A_{u,v}^{\lambda}%
C}_{=C+\lambda E_{u,v}C}\right)   &  =\operatorname*{row}\nolimits_{i}\left(
C+\lambda E_{u,v}C\right)  =\operatorname*{row}\nolimits_{i}%
C+\underbrace{\operatorname*{row}\nolimits_{i}\left(  \lambda E_{u,v}C\right)
}_{\substack{=\lambda\operatorname*{row}\nolimits_{i}\left(  E_{u,v}C\right)
\\\text{(by (\ref{pf.prop.Alamuv.laction.6}))}}}\nonumber\\
&  \ \ \ \ \ \ \ \ \ \ \left(  \text{by (\ref{pf.prop.Alamuv.laction.5}),
applied to }U=C\text{ and }V=\lambda E_{u,v}C\right) \nonumber\\
&  =\operatorname*{row}\nolimits_{i}C+\lambda\operatorname*{row}%
\nolimits_{i}\left(  E_{u,v}C\right)  . \label{pf.prop.Alamuv.laction.9}%
\end{align}


Now, we must prove that $A_{u,v}^{\lambda}C$ is the $n\times m$-matrix
obtained from $C$ by adding $\lambda\operatorname*{row}\nolimits_{v}C$ to the
$u$-th row. In other words, we must prove the following two claims:

\begin{statement}
\textit{Claim 1:} The $u$-th row of the $n\times m$-matrix $A_{u,v}^{\lambda
}C$ is the sum of $\lambda\operatorname*{row}\nolimits_{v}C$ with the $u$-th
row of $C$.
\end{statement}

\begin{statement}
\textit{Claim 2:} For each $i\in\left\{  1,2,\ldots,n\right\}  $ satisfying
$i\neq u$, the $i$-th row of the $n\times m$-matrix $A_{u,v}^{\lambda}C$
equals the $i$-th row of $C$.
\end{statement}

\textit{Proof of Claim 1:} The $u$-th row of the $n\times m$-matrix
$A_{u,v}^{\lambda}C$ is%
\begin{align*}
\operatorname*{row}\nolimits_{u}\left(  A_{u,v}^{\lambda}C\right)   &
=\operatorname*{row}\nolimits_{u}C+\lambda\underbrace{\operatorname*{row}%
\nolimits_{u}\left(  E_{u,v}C\right)  }_{\substack{=\operatorname*{row}%
\nolimits_{v}C\\\text{(by (\ref{pf.prop.Alamuv.laction.1}))}}%
}\ \ \ \ \ \ \ \ \ \ \left(  \text{by (\ref{pf.prop.Alamuv.laction.9}),
applied to }i=u\right) \\
&  =\operatorname*{row}\nolimits_{u}C+\lambda\operatorname*{row}%
\nolimits_{v}C.
\end{align*}
In other words, the $u$-th row of the $n\times m$-matrix $A_{u,v}^{\lambda}C$
is the sum of $\lambda\operatorname*{row}\nolimits_{v}C$ with the $u$-th row
of $C$. This proves Claim 1.

\textit{Proof of Claim 2:} Let $i\in\left\{  1,2,\ldots,n\right\}  $ be such
that $i\neq u$. Then, the $i$-th row of the $n\times m$-matrix $A_{u,v}%
^{\lambda}C$ is%
\begin{align*}
\operatorname*{row}\nolimits_{i}\left(  A_{u,v}^{\lambda}C\right)   &
=\operatorname*{row}\nolimits_{i}C+\lambda\underbrace{\operatorname*{row}%
\nolimits_{i}\left(  E_{u,v}C\right)  }_{\substack{=0_{1\times m}\\\text{(by
(\ref{pf.prop.Alamuv.laction.2}))}}}\ \ \ \ \ \ \ \ \ \ \left(  \text{by
(\ref{pf.prop.Alamuv.laction.9})}\right) \\
&  =\operatorname*{row}\nolimits_{i}C+\underbrace{\lambda0_{1\times m}%
}_{=0_{1\times m}}=\operatorname*{row}\nolimits_{i}C+0_{1\times m}%
=\operatorname*{row}\nolimits_{i}C.
\end{align*}
In other words, the $i$-th row of the $n\times m$-matrix $A_{u,v}^{\lambda}C$
equals the $i$-th row of $C$. This proves Claim 2.

Now, we have proven both Claim 1 and Claim 2; this completes the proof of
Proposition \ref{prop.Alamuv.laction}.
\end{proof}

The proof of Proposition \ref{prop.Alamuv.raction} is analogous.

\begin{proof}
[Proof of Proposition \ref{prop.Alamuv.transpose}.]Proposition
\ref{prop.Euv.transpose} (applied to $m=n$) yields $\left(  E_{u,v}\right)
^{T}=E_{v,u}$.

We have $A_{u,v}^{\lambda}=I_{n}+\lambda E_{u,v}$ (by the definition of
$A_{u,v}^{\lambda}$) and $A_{v,u}^{\lambda}=I_{n}+\lambda E_{v,u}$ (by the
definition of $A_{v,u}^{\lambda}$). Now,
\begin{align*}
\left(  \underbrace{A_{u,v}^{\lambda}}_{=I_{n}+\lambda E_{u,v}}\right)  ^{T}
&  =\left(  I_{n}+\lambda E_{u,v}\right)  ^{T}=\underbrace{\left(
I_{n}\right)  ^{T}}_{\substack{=I_{n}\\\text{(by Proposition
\ref{prop.transpose.opers} \textbf{(a)})}}}+\underbrace{\left(  \lambda
E_{u,v}\right)  ^{T}}_{\substack{=\lambda\left(  E_{u,v}\right)
^{T}\\\text{(by Proposition \ref{prop.transpose.opers} \textbf{(c)}%
,}\\\text{applied to }m=n\text{ and }A=E_{u,v}\text{)}}}\\
&  \ \ \ \ \ \ \ \ \ \ \left(
\begin{array}
[c]{c}%
\text{by Proposition \ref{prop.transpose.opers} \textbf{(d)},}\\
\text{applied to }m=n\text{, }A=I_{n}\text{ and }B=\lambda E_{u,v}%
\end{array}
\right) \\
&  =I_{n}+\lambda\underbrace{\left(  E_{u,v}\right)  ^{T}}_{=E_{v,u}}%
=I_{n}+\lambda E_{v,u}=A_{v,u}^{\lambda}.
\end{align*}
Thus, Proposition \ref{prop.Alamuv.transpose} is proven.
\end{proof}

\begin{proof}
[Proof of Proposition \ref{prop.Alamuv.lambda+mu}.]\textbf{(a)} The definition
of $A_{u,v}^{0}$ yields $A_{u,v}^{0}=I_{n}+\underbrace{0E_{u,v}}_{=0_{n\times
n}}=I_{n}+0_{n\times n}=I_{n}$. This proves Proposition
\ref{prop.Alamuv.lambda+mu} \textbf{(a)}.

\textbf{(b)} \textit{First proof:} Let $\lambda$ and $\mu$ be two numbers.
Proposition \ref{prop.Alamuv.entries} (applied to $\mu$ instead of $\lambda$)
tells us how the matrix $A_{u,v}^{\mu}$ looks like: Its diagonal entries are
$1$; its $\left(  u,v\right)  $-th entry is $\mu$; all its remaining entries
are $0$. In particular, its $v$-th row is%
\begin{equation}
\operatorname*{row}\nolimits_{v}\left(  A_{u,v}^{\mu}\right)  =\left(
0,0,\ldots,0,1,0,0,\ldots,0\right)  \label{pf.prop.Alamuv.lambda+mu.b.1}%
\end{equation}
(where the lonely $1$ stands in the $v$-th position).

Proposition \ref{prop.Alamuv.laction} (applied to $m=n$ and $C=A_{u,v}^{\mu}$)
shows that $A_{u,v}^{\lambda}A_{u,v}^{\mu}$ is the $n\times n$-matrix obtained
from $A_{u,v}^{\mu}$ by adding $\lambda\operatorname*{row}\nolimits_{v}\left(
A_{u,v}^{\mu}\right)  $ to the $u$-th row. Since
\begin{align*}
\lambda\operatorname*{row}\nolimits_{v}\left(  A_{u,v}^{\mu}\right)   &
=\lambda\left(  0,0,\ldots,0,1,0,0,\ldots,0\right)
\ \ \ \ \ \ \ \ \ \ \left(  \text{by (\ref{pf.prop.Alamuv.lambda+mu.b.1}%
)}\right) \\
&  =\left(  0,0,\ldots,0,\lambda,0,0,\ldots,0\right)
\end{align*}
(where the lonely $\lambda$ stands in the $v$-th position), this shows that
$A_{u,v}^{\lambda}A_{u,v}^{\mu}$ is the $n\times n$-matrix obtained from
$A_{u,v}^{\mu}$ by adding $\left(  0,0,\ldots,0,\lambda,0,0,\ldots,0\right)  $
to the $u$-th row. This addition has the effect that the $\left(  u,v\right)
$-th entry increases by $\lambda$, whereas all the other entries remain
unchanged. The resulting matrix $A_{u,v}^{\lambda}A_{u,v}^{\mu}$ therefore has
its $\left(  u,v\right)  $-th entry equal to $\mu+\lambda=\lambda+\mu$,
whereas all its other entries are the same as in $A_{u,v}^{\mu}$ (that is, the
diagonal entries are $1$ and the remaining entries are $0$). But this is
precisely how the matrix $A_{u,v}^{\lambda+\mu}$ looks like (because of
Proposition \ref{prop.Alamuv.entries}, applied to $\lambda+\mu$ instead of
$\lambda$). Hence, $A_{u,v}^{\lambda}A_{u,v}^{\mu}=A_{u,v}^{\lambda+\mu}$.
This proves Proposition \ref{prop.Alamuv.lambda+mu} \textbf{(b)}.

\textit{Second proof:} We can also prove Proposition
\ref{prop.Alamuv.lambda+mu} \textbf{(b)} easily using Proposition
\ref{prop.Euv.prod}: Indeed, we have $v\neq u$ (since $u$ and $v$ are
distinct), so that $\delta_{v,u}=0$. But Proposition \ref{prop.Euv.prod}
(applied to $m=n$, $p=n$, $x=u$ and $y=v$) yields $E_{u,v,n,n}E_{u,v,n,n}%
=\delta_{v,u}E_{u,v,n,n}$. Since $E_{u,v,n,n}=E_{u,v}$, this rewrites as
$E_{u,v}E_{u,v}=\underbrace{\delta_{v,u}}_{=0}E_{u,v}=0E_{u,v}=0_{n\times n}$.
Now, the definitions of $A_{u,v}^{\lambda}$ and $A_{u,v}^{\mu}$ yield
$A_{u,v}^{\lambda}=I_{n}+\lambda E_{u,v}$ and $A_{u,v}^{\mu}=I_{n}+\mu
E_{u,v}$. Multiplying these two equalities, we find%
\begin{align*}
A_{u,v}^{\lambda}A_{u,v}^{\mu}  &  =\left(  I_{n}+\lambda E_{u,v}\right)
\left(  I_{n}+\mu E_{u,v}\right) \\
&  =\underbrace{I_{n}\left(  I_{n}+\mu E_{u,v}\right)  }_{=I_{n}+\mu E_{u,v}%
}+\underbrace{\lambda E_{u,v}\left(  I_{n}+\mu E_{u,v}\right)  }_{=\lambda
E_{u,v}I_{n}+\lambda E_{u,v}\mu E_{u,v}}\\
&  =I_{n}+\mu E_{u,v}+\lambda\underbrace{E_{u,v}I_{n}}_{=E_{u,v}%
}+\underbrace{\lambda E_{u,v}\mu E_{u,v}}_{=\lambda\mu E_{u,v}E_{u,v}}\\
&  =I_{n}+\underbrace{\mu E_{u,v}+\lambda E_{u,v}}_{=\left(  \mu
+\lambda\right)  E_{u,v}}+\lambda\mu\underbrace{E_{u,v}E_{u,v}}_{=0_{n\times
n}}\\
&  =I_{n}+\underbrace{\left(  \mu+\lambda\right)  }_{=\lambda+\mu}%
E_{u,v}+\underbrace{\lambda\mu0_{n\times n}}_{=0_{n\times n}}=I_{n}+\left(
\lambda+\mu\right)  E_{u,v}+0_{n\times n}=I_{n}+\left(  \lambda+\mu\right)
E_{u,v}.
\end{align*}
Comparing this with
\[
A_{u,v}^{\lambda+\mu}=I_{n}+\left(  \lambda+\mu\right)  E_{u,v}%
\ \ \ \ \ \ \ \ \ \ \left(  \text{by the definition of }A_{u,v}^{\lambda+\mu
}\right)  ,
\]
we obtain $A_{u,v}^{\lambda}A_{u,v}^{\mu}=A_{u,v}^{\lambda+\mu}$. This proves
Proposition \ref{prop.Alamuv.lambda+mu} \textbf{(b)} again.

\textbf{(c)} Proposition \ref{prop.Alamuv.lambda+mu} \textbf{(b)} (applied to
$\mu=-\lambda$) yields $A_{u,v}^{\lambda}A_{u,v}^{-\lambda}=A_{u,v}%
^{\lambda+\left(  -\lambda\right)  }=A_{u,v}^{0}=I_{n}$ (by Proposition
\ref{prop.Alamuv.lambda+mu} \textbf{(a)}).

But we can also apply Proposition \ref{prop.Alamuv.lambda+mu} \textbf{(b)} to
$-\lambda$ and $\lambda$ instead of $\lambda$ and $\mu$. We thus obtain
$A_{u,v}^{-\lambda}A_{u,v}^{\lambda}=A_{u,v}^{\left(  -\lambda\right)
+\lambda}=A_{u,v}^{0}=I_{n}$ (by Proposition \ref{prop.Alamuv.lambda+mu}
\textbf{(a)}).

The two equalities $A_{u,v}^{\lambda}A_{u,v}^{-\lambda}=I_{n}$ and
$A_{u,v}^{-\lambda}A_{u,v}^{\lambda}=I_{n}$ show that $A_{u,v}^{-\lambda}$ is
an inverse of $A_{u,v}^{\lambda}$. This proves Proposition
\ref{prop.Alamuv.lambda+mu} \textbf{(c)}.
\end{proof}

\subsection{Unitriangular matrices are products of $A_{u,v}^{\lambda}$'s}

We have already said that the matrices $A_{u,v}^{\lambda}$ are building blocks
(of a sort). Before we explain what this means in detail, let us define a
convenient word:

\begin{definition}
\label{def.Alamuv.laddition}Let $n\in\mathbb{N}$. A \textit{lower addition
}$n\times n$\textit{-matrix} means a matrix of the form $A_{u,v}^{\lambda}$,
where $\lambda$ is a number, and where $u$ and $v$ are two elements of
$\left\{  1,2,\ldots,n\right\}  $ satisfying $u>v$. When $n$ is clear from the
context, we shall omit the \textquotedblleft$n\times n$-\textquotedblright%
\ and simply say \textquotedblleft lower addition matrix\textquotedblright.
\end{definition}

The name \textquotedblleft lower addition matrix\textquotedblright\ is, again,
not standard, but it will be useful for me in this chapter.

\begin{example}
If $n=3$, then the lower addition $3\times3$-matrices are the matrices of the
form%
\begin{align*}
A_{2,1}^{\lambda}  &  =\left(
\begin{array}
[c]{ccc}%
1 & 0 & 0\\
\lambda & 1 & 0\\
0 & 0 & 1
\end{array}
\right)  ,\ \ \ \ \ \ \ \ \ \ A_{3,1}^{\lambda}=\left(
\begin{array}
[c]{ccc}%
1 & 0 & 0\\
0 & 1 & 0\\
\lambda & 0 & 1
\end{array}
\right)  ,\\
A_{3,2}^{\lambda}  &  =\left(
\begin{array}
[c]{ccc}%
1 & 0 & 0\\
0 & 1 & 0\\
0 & \lambda & 1
\end{array}
\right)
\end{align*}
for all numbers $\lambda$. If $n=2$, then the lower addition $2\times
2$-matrices are the matrices of the form $A_{2,1}^{\lambda}=\left(
\begin{array}
[c]{cc}%
1 & 0\\
\lambda & 1
\end{array}
\right)  $ for all numbers $\lambda$. If $n=1$ or $n=0$, then there are no
lower addition $n\times n$-matrices (because there is nowhere to put the
$\lambda$, speaking visually).
\end{example}

It is clear that each lower addition matrix is
lower-unitriangular\footnote{See Definition \ref{def.triangular.uni-up} for
the meaning of \textquotedblleft lower-unitriangular\textquotedblright.}
(because Proposition \ref{prop.Alamuv.entries} shows that its entries above
the diagonal are $0$, and its diagonal entries are $1$). Thus, every product
of lower addition matrices is a product of lower-unitriangular matrices, and
thus itself must be lower-unitriangular\footnote{because Corollary
\ref{cor.triangular.uni*uni*uni.lower} shows that any product of
lower-unitriangular matrices is lower-unitriangular}. It turns out that the
converse is also true: Every lower-unitriangular matrix is a product of lower
addition matrices! This is a first, simple particular case of Gaussian
elimination; let me state it as a theorem:

\begin{theorem}
\label{thm.triangular.Alamuv}Let $n\in\mathbb{N}$. An $n\times n$-matrix $C$
is lower-unitriangular if and only if $C$ is a product of lower addition matrices.
\end{theorem}

Keep in mind that \textquotedblleft a product of lower addition
matrices\textquotedblright\ (and any other product, unless declared otherwise)
may contain zero factors (in which case it is empty, and thus equals the
identity matrix) or one factor (in which case it equals that factor). Usually,
of course, it will contain more than one factor.

\begin{example}
\label{exam.thm.triangular.Alamuv}\textbf{(a)} The lower-unitriangular
$2\times2$-matrix $\left(
\begin{array}
[c]{cc}%
1 & 0\\
5 & 1
\end{array}
\right)  $ is a product of lower addition matrices: Namely, it equals
$A_{2,1}^{5}$.

\textbf{(b)} The lower-unitriangular $1\times1$-matrix $\left(
\begin{array}
[c]{c}%
1
\end{array}
\right)  $ is a product of lower addition matrices: Namely, it is the empty
product. (Recall that the empty product of $1\times1$-matrices is defined to
be $I_{1}$, and this is precisely our matrix $\left(
\begin{array}
[c]{c}%
1
\end{array}
\right)  $.)

\textbf{(c)} Let $C$ be the lower-unitriangular $3\times3$-matrix $\left(
\begin{array}
[c]{ccc}%
1 & 0 & 0\\
a & 1 & 0\\
b & c & 1
\end{array}
\right)  $. Then, $C$ is a product of lower addition matrices: Namely, it
equals $A_{2,1}^{a}A_{3,1}^{b}A_{3,2}^{c}$.

Let us actually see how this representation of $C$ can be found. We shall
proceed by writing $C$ as a product of one lower addition matrix with a second
matrix $C^{\prime}$, which is still lower-triangular but has one less nonzero
entry than $C$. We then will do the same with $C^{\prime}$, obtaining a third
matrix $C^{\prime\prime}$; then, do the same with $C^{\prime\prime}$, and so
on. At the end, we will be left with an identity matrix. In more detail:

\begin{description}
\item[Step 1:] Let us get rid of the $\left(  2,1\right)  $-th entry of $C$
(that is, turn this entry into a $0$) by subtracting $a\operatorname*{row}%
\nolimits_{1}C$ from row $2$ of $C$. Denote the resulting matrix by
$C^{\prime}$. Thus, $C^{\prime}=\left(
\begin{array}
[c]{ccc}%
1 & 0 & 0\\
0 & 1 & 0\\
b & c & 1
\end{array}
\right)  $. (No entries other than the $\left(  2,1\right)  $-th one have been
changed, because $a\operatorname*{row}\nolimits_{1}C=\left(  a,0,0\right)  $.)
Since $C^{\prime}$ was obtained from $C$ by subtracting $a\operatorname*{row}%
\nolimits_{1}C$ from row $2$ of $C$, we can conversely obtain $C$ from
$C^{\prime}$ by adding $a\operatorname*{row}\nolimits_{1}\left(  C^{\prime
}\right)  $ to row $2$ of $C^{\prime}$. According to Proposition
\ref{prop.Alamuv.laction} (applied to $n$, $2$, $1$, $a$ and $C^{\prime}$
instead of $m$, $u$, $v$, $\lambda$ and $C$), this means that $C=A_{2,1}%
^{a}C^{\prime}$.

\item[Step 2:] Let us get rid of the $\left(  3,1\right)  $-th entry of
$C^{\prime}$ by subtracting $b\operatorname*{row}\nolimits_{1}\left(
C^{\prime}\right)  $ from row $3$ of $C^{\prime}$. Denote the resulting matrix
by $C^{\prime\prime}$. Thus, $C^{\prime\prime}=\left(
\begin{array}
[c]{ccc}%
1 & 0 & 0\\
0 & 1 & 0\\
0 & c & 1
\end{array}
\right)  $. (Again, no entries other than the $\left(  3,1\right)  $-th one
have been changed.) Similarly to how we found that $C=A_{2,1}^{a}C^{\prime}$
in Step 1, we now obtain $C^{\prime}=A_{3,1}^{b}C^{\prime\prime}$.

\item[Step 3:] Let us get rid of the $\left(  3,2\right)  $-th entry of
$C^{\prime\prime}$ by subtracting $c\operatorname*{row}\nolimits_{2}\left(
C^{\prime\prime}\right)  $ from row $3$ of $C^{\prime\prime}$. Denote the
resulting matrix by $C^{\prime\prime\prime}$. Thus, $C^{\prime\prime\prime
}=\left(
\begin{array}
[c]{ccc}%
1 & 0 & 0\\
0 & 1 & 0\\
0 & 0 & 1
\end{array}
\right)  $. (Again, no entries other than the $\left(  3,2\right)  $-th one
have been changed.) Similarly to how we found that $C=A_{2,1}^{a}C^{\prime}$
in Step 1, we now obtain $C^{\prime\prime}=A_{3,2}^{c}C^{\prime\prime\prime}$.
\end{description}

We have now removed all nonzero entries below the diagonal. Our final matrix
$C^{\prime\prime\prime}$ is simply the identity matrix: $C^{\prime\prime
\prime}=I_{3}$. Combining the three equalities we have found, we obtain%
\begin{align}
C  &  =A_{2,1}^{a}\underbrace{C^{\prime}}_{=A_{3,1}^{b}C^{\prime\prime}%
}=A_{2,1}^{a}A_{3,1}^{b}\underbrace{C^{\prime\prime}}_{=A_{3,2}^{c}%
C^{\prime\prime\prime}}=A_{2,1}^{a}A_{3,1}^{b}A_{3,2}^{c}\underbrace{C^{\prime
\prime\prime}}_{=I_{3}}=A_{2,1}^{a}A_{3,1}^{b}A_{3,2}^{c}I_{3}\nonumber\\
&  =A_{2,1}^{a}A_{3,1}^{b}A_{3,2}^{c}.
\label{eq.exam.thm.triangular.Alamuv.c.3}%
\end{align}
Thus we have represented $C$ as a product of lower addition matrices.

Notice that we need to be careful about the order in which we perform the
above steps. We have first gotten rid of the $\left(  2,1\right)  $-st entry,
then gotten rid of the $\left(  3,1\right)  $-th entry, and then gotten rid of
the $\left(  3,2\right)  $-th entry. If we had tried to clean out the entries
in a different order, we might have run into trouble. Namely, if we first get
rid of the $\left(  3,1\right)  $-st entry, and then get rid of the $\left(
3,2\right)  $-th entry, then the $\left(  3,1\right)  $-th entry can become
\textquotedblleft polluted\textquotedblright\ again (i.e., the resulting
matrix might again have a nonzero $\left(  3,1\right)  $-th entry). One way to
avoid this kind of trouble is to clear out entries column by column: first
clear out all entries in the $1$-st column (except for the $1$ on the
diagonal); then clear out all entries in the $2$-nd column (except for the $1$
on the diagonal); and so on, moving left to right. (This is what we have done
in our three steps above.)
\end{example}

The general proof of Theorem \ref{thm.triangular.Alamuv} follows the idea
outlined in Example \ref{exam.thm.triangular.Alamuv} \textbf{(c)}:

\begin{proof}
[Proof of Theorem \ref{thm.triangular.Alamuv}.]$\Longleftarrow$:\footnote{The
symbol \textquotedblleft$\Longleftarrow$\textquotedblright\ means that we are
now going to prove the \textquotedblleft$\Longleftarrow$
direction\textquotedblright\ of Theorem \ref{thm.triangular.Alamuv} (that is,
we are going to prove that if $C$ is a product of lower addition matrices,
then $C$ is lower-unitriangular). See Section \ref{sect.gauss.triang-proof1}
for more about this notation.} We have already proven that every product of
lower addition matrices is lower-unitriangular. Hence, if $C$ is a product of
lower addition matrices, then $C$ is lower-unitriangular. This proves the
$\Longleftarrow$ direction of Theorem \ref{thm.triangular.Alamuv}.

$\Longrightarrow$:\footnote{The symbol \textquotedblleft$\Longrightarrow
$\textquotedblright\ means that we are now going to prove the
\textquotedblleft$\Longrightarrow$ direction\textquotedblright\ of Theorem
\ref{thm.triangular.Alamuv} (that is, we are going to prove that if $C$ is
lower-unitriangular, then $C$ is a product of lower addition matrices). See
Section \ref{sect.gauss.triang-proof1} for more about this notation.} We need
to prove that if $C$ is lower-unitriangular, then $C$ is a product of lower
addition matrices.

So let us assume that $C$ is lower-unitriangular. Our goal is to prove that
$C$ is a product of lower addition matrices.

Let me introduce a notation first: A \textit{downward row addition} shall mean
a transformation that changes an $n\times m$-matrix (for some $m \in
\mathbb{N}$) by adding a scalar multiple\footnote{Recall that a \textit{scalar
multiple} of a matrix $A$ means a matrix of the form $\lambda A$ with
$\lambda$ being a number. Row vectors are matrices (by definition), so we can
talk about a scalar multiple of a row.} of one of its rows to another row
further down. In more formal terms: A \textit{downward row addition} means a
transformation of the form ``add $\lambda$ times the $v$-th row to the $u$-th
row'', for some fixed number $\lambda$ and some fixed integers $u$ and $v$
satisfying $1 \leq v < u \leq n$. As we know from Proposition
\ref{prop.Alamuv.laction}, this transformation amounts to multiplying a matrix
by $A^{\lambda}_{u, v}$ from the left (i.e., this transformation sends any
$n\times m$-matrix $B$ to $A^{\lambda}_{u,v} B$); we shall therefore denote
this transformation itself by $A^{\lambda}_{u,v}$ as well (hoping that the
reader will not confuse the transformation with the matrix).

Here is an example (for $n = 4$): The downward row addition $A^{\lambda}%
_{3,1}$ is the transformation that changes a $4\times m$-matrix by adding
$\lambda$ times the $1$-st row to the $3$-rd row. For example, it transforms
the $4 \times2$-matrix $\left(
\begin{array}
[c]{cc}%
a & b\\
a^{\prime} & b^{\prime}\\
a^{\prime\prime} & b^{\prime\prime}\\
a^{\prime\prime\prime} & b^{\prime\prime\prime}%
\end{array}
\right)  $ into $\left(
\begin{array}
[c]{cc}%
a & b\\
a^{\prime} & b^{\prime}\\
a^{\prime\prime}+ \lambda a & b^{\prime\prime}+ \lambda b\\
a^{\prime\prime\prime} & b^{\prime\prime\prime}%
\end{array}
\right)  $.
%We shall visually represent this fact as follows:
%\[
%\left(\begin{array}{cc} a & b \\ a' & b' \\ a'' & b'' \\ a''' & b''' \end{array}\right)
%\overset{A^\lambda_{3,1}}{\longrightarrow}
%\left(\begin{array}{cc} a & b \\ a' & b' \\ a'' + \lambda a & b'' + \lambda b \\ a''' & b''' \end{array}\right) .
%\]
%(The ``$A^\lambda_{3,1}$'' over the arrow tells us that the transformation
%used is the downward row addition $A^\lambda_{3,1}$.)


Notice that any downward row addition $A_{u,v}^{\lambda}$ is invertible:
Namely, it can be undone by the downward row addition $A_{u,v}^{-\lambda}%
$.\ \ \ \ \footnote{Here are two ways to prove this:
\par
\textit{First proof:} The downward row addition $A_{u,v}^{\lambda}$ transforms
a matrix by adding $\lambda$ times the $v$-th row to the $u$-th row. The
downward row addition $A_{u,v}^{-\lambda}$ transforms a matrix by adding
$-\lambda$ times the $v$-th row to the $u$-th row, i.e., by subtracting
$\lambda$ times the $v$-th row from the $u$-th row. Hence, these two row
additions undo each other (i.e., if we perform one and then the other, then we
arrive back at the matrix we have started with), because (for example) adding
$\lambda$ times the $v$-th row to the $u$-th row and then subtracting it back
recovers the original $u$-th row. (Here, we have tacitly used the fact that
$u\neq v$. If we had $u=v$, then adding $\lambda$ times the $v$-th row to the
$u$-th row would have changed the $v$-th row, and then subtracting it back
would mean subtracting $\lambda$ times a \textbf{changed} $v$-th row from the
$u$-th row.) So we have shown that the downward row addition $A_{u,v}%
^{\lambda}$ can be undone by the downward row addition $A_{u,v}^{-\lambda}$.
Qed.
\par
\textit{Second proof:} Proposition \ref{prop.Alamuv.lambda+mu} \textbf{(c)}
shows that the matrix $A_{u,v}^{-\lambda}$ is the inverse of the matrix
$A_{u,v}^{\lambda}$. Hence, multiplying a matrix by $A_{u,v}^{-\lambda}$
undoes multiplying a matrix by $A_{u,v}^{\lambda}$. In other words, the
downward row addition $A_{u,v}^{-\lambda}$ undoes the downward row addition
$A_{u,v}^{\lambda}$. Qed.}

Notice that, for any downward row addition $A^{\lambda}_{u,v}$, the matrix
$A^{\lambda}_{u,v}$ is a lower addition matrix. This is because $u > v$ (by
the definition of a downward row addition).

I claim that we can transform the lower-unitriangular $n\times n$-matrix $C$
into the identity matrix $I_{n}$ by performing a sequence of downward row
additions. Namely, we should proceed by the following method:\footnote{See the
three-step procedure in Example \ref{exam.thm.triangular.Alamuv} \textbf{(c)}
for an illustration of this method.}

\begin{itemize}
\item At first, our matrix is
\[
C = \left(
\begin{array}
[c]{ccccc}%
1 & 0 & 0 & \cdots & 0\\
C_{2,1} & 1 & 0 & \cdots & 0\\
C_{3,1} & C_{3,2} & 1 & \cdots & 0\\
\vdots & \vdots & \vdots & \ddots & \vdots\\
C_{n,1} & C_{n,2} & C_{n,3} & \cdots & 1
\end{array}
\right)  .
\]
In particular, its $1$-st row is $\left(  1,0,0,\ldots,0\right)  $ (same as
the $1$-st row of $I_{n}$).

\item Now, we perform the downward row addition $A^{\lambda}_{2,1}$ (for an
appropriate choice of $\lambda$, namely for $\lambda= -C_{2,1}$) to clear out
the $\left(  2,1\right)  $-th entry of the matrix (i.e., to put a $0$ where
this entry stood). This does not affect any of the other entries, because the
$1$-st row of the matrix is $\left(  1,0,0,\ldots,0\right)  $ (thus has only
one nonzero entry). Similarly, we then perform the downward row addition
$A^{\lambda}_{3,1}$ (with $\lambda= -C_{3,1}$) to clear out the $\left(
3,1\right)  $-th entry; then, we perform the downward row addition
$A^{\lambda}_{4,1}$ (with $\lambda= -C_{4,1}$) to clear out the $\left(
4,1\right)  $-th entry; and so on, ending with the downward row addition
$A^{\lambda}_{n,1}$ (with $\lambda= -C_{n,1}$) to clear out the $\left(
n,1\right)  $-th entry. As the result, we have cleared out all entries in the
$1$-st column of our matrix, except for the $1$ on the diagonal. In other
words, our matrix now looks as follows:
\[
\left(
\begin{array}
[c]{ccccc}%
1 & 0 & 0 & \cdots & 0\\
0 & 1 & 0 & \cdots & 0\\
0 & C_{3,2} & 1 & \cdots & 0\\
\vdots & \vdots & \vdots & \ddots & \vdots\\
0 & C_{n,2} & C_{n,3} & \cdots & 1
\end{array}
\right)  .
\]
In particular, its $2$-nd row is $\left(  0,1,0,0,\ldots,0\right)  $ (same as
the $2$-nd row of $I_{n}$).

\item Next, we similarly clear out all entries in the $2$-nd column of the
matrix, by performing the downward row additions $A^{\lambda}_{3,2},
A^{\lambda}_{4,2}, \ldots, A^{\lambda}_{n,2}$ (each time picking an
appropriate value of $\lambda$). Again, this does not affect any of the other
entries, because the $2$-nd row of the matrix is $\left(  0,1,0,0,\ldots
,0\right)  $. As the result, we have cleared out all entries in the $2$-nd
column of our matrix, except for the $1$ on the diagonal. In other words, our
matrix now looks as follows:
\[
\left(
\begin{array}
[c]{ccccc}%
1 & 0 & 0 & \cdots & 0\\
0 & 1 & 0 & \cdots & 0\\
0 & 0 & 1 & \cdots & 0\\
\vdots & \vdots & \vdots & \ddots & \vdots\\
0 & 0 & C_{n,3} & \cdots & 1
\end{array}
\right)  .
\]
In particular, its $3$-rd row is $\left(  0,0,1,0,0,\ldots,0\right)  $ (same
as the $3$-rd row of $I_{n}$).

\item Next, we similarly clear out all entries in the $3$-rd column of the
matrix, by performing the downward row additions $A^{\lambda}_{4,3},
A^{\lambda}_{5,3}, \ldots, A^{\lambda}_{n,3}$. As the result, we have cleared
out all entries in the $3$-rd column of our matrix, except for the $1$ on the
diagonal. In other words, our matrix now looks as follows:
\[
\left(
\begin{array}
[c]{ccccc}%
1 & 0 & 0 & \cdots & 0\\
0 & 1 & 0 & \cdots & 0\\
0 & 0 & 1 & \cdots & 0\\
\vdots & \vdots & \vdots & \ddots & \vdots\\
0 & 0 & 0 & \cdots & 1
\end{array}
\right)  .
\]
Don't mistake this for the identity matrix $I_{n}$: There might still be
nonzero entries outside of the diagonal. They are just hidden by the $\cdots$ notation.

\item We continue this process, clearing out column after column, until the
$n$-th column is cleared out. By then, our matrix has become the identity
matrix $I_{n}$.
\end{itemize}

Thus, we have found an algorithm to transform our matrix $C$ into the identity
matrix $I_{n}$ by a sequence of downward row additions. Therefore, we can
conversely transform the identity matrix $I_{n}$ into $C$ by a sequence of
downward row additions\footnote{because (as we have shown) any downward row
addition $A_{u,v}^{\lambda}$ is invertible, and can be undone by another
downward row addition}. Let us denote these downward row additions (used to
transform $I_{n}$ into $C$) by $A_{u_{1},v_{1}}^{\lambda_{1}},A_{u_{2},v_{2}%
}^{\lambda_{2}},\cdots,A_{u_{k},v_{k}}^{\lambda_{k}}$, \textbf{numbered
backwards} (i.e., starting from the one used last). Since each downward row
addition $A_{u,v}^{\lambda}$ amounts to multiplying a matrix by the matrix
$A_{u,v}^{\lambda}$ (that is, it sends any $n\times m$-matrix $B$ to
$A_{u,v}^{\lambda}B$), we thus conclude that
\[
C=A_{u_{1},v_{1}}^{\lambda_{1}}A_{u_{2},v_{2}}^{\lambda_{2}}\cdots
A_{u_{k},v_{k}}^{\lambda_{k}}I_{n}=A_{u_{1},v_{1}}^{\lambda_{1}}A_{u_{2}%
,v_{2}}^{\lambda_{2}}\cdots A_{u_{k},v_{k}}^{\lambda_{k}}.
\]
Thus, $C$ is a product of lower addition matrices (because each of the
matrices $A_{u_{1},v_{1}}^{\lambda_{1}},A_{u_{2},v_{2}}^{\lambda_{2}}%
,\cdots,A_{u_{k},v_{k}}^{\lambda_{k}}$ is a lower addition
matrix\footnote{Here we are using the fact that, for any downward row addition
$A_{u,v}^{\lambda}$, the matrix $A_{u,v}^{\lambda}$ is a lower addition
matrix.}). This is precisely what we had to prove. This proves the
$\Longrightarrow$ direction of Theorem \ref{thm.triangular.Alamuv}. Hence, the
proof of Theorem \ref{thm.triangular.Alamuv} is complete.
\end{proof}

\begin{remark}
Our proof of Theorem \ref{thm.triangular.Alamuv} (specifically, of its
$\Longrightarrow$ direction) actually gives an explicit representation of a
lower-unitriangular $n\times n$-matrix $C$ as a product of lower addition
matrices:%
\begin{align*}
C  &  =\left(  A_{2,1}^{C_{2,1}}A_{3,1}^{C_{3,1}}\cdots A_{n,1}^{C_{n,1}%
}\right)  \left(  A_{3,2}^{C_{3,2}}A_{4,2}^{C_{4,2}}\cdots A_{n,2}^{C_{n,2}%
}\right)  \left(  A_{4,3}^{C_{4,3}}A_{5,3}^{C_{5,3}}\cdots A_{n,3}^{C_{n,3}%
}\right) \\
&  \ \ \ \ \ \ \ \ \ \ \cdots\left(  A_{n-2,n-3}^{C_{n-2,n-3}}A_{n-1,n-3}%
^{C_{n-1,n-3}}A_{n,n-3}^{C_{n,n-3}}\right)  \left(  A_{n-1,n-2}^{C_{n-1,n-2}%
}A_{n,n-2}^{C_{n,n-2}}\right)  \left(  A_{n,n-1}^{C_{n,n-1}}\right)  \left(
I_{n}\right)  .
\end{align*}
This complicated product consists of $n$ factors, each of which is itself a
product. More precisely, the $k$-th factor is the product $A_{k+1,k}%
^{C_{k+1,k}}A_{k+2,k}^{C_{k+2,k}}\cdots A_{n,k}^{C_{n,k}}$ of all matrices
$A_{i,k}^{C_{i,k}}$ with $i>k$; this corresponds to the $n-k$ downward row
additions that clear out the non-diagonal entries in the $k$-th column of the
matrix. In particular, the $n$-th factor is thus an empty product, hence
equals $I_{n}$; we could thus leave it out (but we prefer to keep it in, for
reasons of clarity).

The reason why the above explicit representation works is that in our process
of clearing out zero entries, we have never modified any entry other than the
one we were clearing out. Thus, for each $\left(  i,j\right)  $, the $\left(
i,j\right)  $-th entry of our matrix remained equal to its original value
$C_{i,j}$ up until the moment when we cleared it out.
\end{remark}

\begin{exercise}
In Example \ref{exam.thm.triangular.Alamuv} \textbf{(c)}, we have cleared out
the three sub-diagonal (= below the diagonal) entries of $C$ in a specific
order: first the $\left(  2,1\right)  $-th entry, then the $\left(
3,1\right)  $-th entry, then the $\left(  3,2\right)  $-th entry. We thus
obtained the formula $C=A_{2,1}^{a}A_{3,1}^{b}A_{3,2}^{c}$.

We could also have proceeded differently: for instance, we could have cleared
out the $\left(  3,2\right)  $-th entry first, then the $\left(  3,1\right)
$-th entry, then the $\left(  2,1\right)  $-th entry. This would have resulted
in the formula $C=A_{3,2}^{c}A_{3,1}^{b-ac}A_{2,1}^{a}$. (Be aware of the
$b-ac$ in the $A_{3,1}^{b-ac}$; this is because our first clearing operation
has changed the $\left(  3,1\right)  $-th entry to $b-ac$.)

There is a total of $6$ different orders in which we can try clearing out the
three sub-diagonal entries of our $3\times3$-matrix $C$. The two of them just
shown work; on the other hand, in Example \ref{exam.thm.triangular.Alamuv}
\textbf{(c)}, we have seen one order which does not (namely, starting with the
$\left(  3,1\right)  $-th entry, then doing the $\left(  3,2\right)  $-th and
then the $\left(  2,1\right)  $-th one). Of the remaining three, which ones
work? And what formulas do they result in?

(By \textquotedblleft work\textquotedblright, I mean \textquotedblleft work
for every lower-unitriangular matrix $C$\textquotedblright.)
\end{exercise}

\subsection{The inverse of a lower-unitriangular matrix}

Theorem \ref{thm.triangular.Alamuv} has the following neat consequence:

\begin{theorem}
\label{thm.triangular.inverse}Let $n\in\mathbb{N}$. Let $A$ be a
lower-unitriangular $n\times n$-matrix. Then, $A$ is invertible, and its
inverse $A^{-1}$ is again lower-unitriangular.
\end{theorem}

\begin{example}
\label{exam.thm.triangular.inverse}Let $A$ be the lower-unitriangular
$3\times3$-matrix $\left(
\begin{array}
[c]{ccc}%
1 & 0 & 0\\
-2 & 1 & 0\\
1 & 6 & 1
\end{array}
\right)  $. Theorem \ref{thm.triangular.inverse} (applied to $n=3$) then
claims that $A$ is invertible, and that its inverse $A^{-1}$ is again
lower-unitriangular. This can easily be checked: We have $A^{-1}=\left(
\begin{array}
[c]{ccc}%
1 & 0 & 0\\
2 & 1 & 0\\
-13 & -6 & 1
\end{array}
\right)  $.
\end{example}

\begin{proof}
[Proof of Theorem \ref{thm.triangular.inverse}.]Theorem
\ref{thm.triangular.Alamuv} (applied to $C=A$) shows that $A$ is
lower-unitriangular if and only if $A$ is a product of lower addition
matrices. Hence, $A$ is a product of lower addition matrices (since $A$ is
lower-unitriangular). In other words, $A$ has the form $A=A_{1}A_{2}\cdots
A_{k}$ for some $k\in\mathbb{N}$ and some $k$ lower addition matrices
$A_{1},A_{2},\ldots,A_{k}$. Consider these $k$ and $A_{1},A_{2},\ldots,A_{k}$.

We are in one of the following two cases:

\begin{statement}
\textit{Case 1:} We have $k\neq0$.
\end{statement}

\begin{statement}
\textit{Case 2:} We have $k=0$.
\end{statement}

Let us deal with Case 1. In this case, we have $k\neq0$; thus, $k$ is a
positive integer.

For each $i\in\left\{  1,2,\ldots,k\right\}  $, the matrix $A_{i}$ is
invertible\footnote{\textit{Proof.} Let $i\in\left\{  1,2,\ldots,k\right\}  $.
We must show that the matrix $A_{i}$ is invertible.
\par
We know that $A_{i}$ is a lower addition matrix (since $A_{1},A_{2}%
,\ldots,A_{k}$ are lower addition matrices). In other words, $A_{i}$ has the
form $A_{i}=A_{u,v}^{\lambda}$, where $\lambda$ is a number, and where $u$ and
$v$ are two elements of $\left\{  1,2,\ldots,n\right\}  $ satisfying $u>v$ (by
the definition of a \textquotedblleft lower addition matrix\textquotedblright%
). Consider these $\lambda$, $u$ and $v$. Proposition
\ref{prop.Alamuv.lambda+mu} \textbf{(c)} shows that the matrix $A_{u,v}%
^{\lambda}$ is invertible. In other words, the matrix $A_{i}$ is invertible
(since $A_{i}=A_{u,v}^{\lambda}$). This completes our proof.}, and its inverse
$A_{i}^{-1}$ is again a lower addition matrix\footnote{\textit{Proof.} Let
$i\in\left\{  1,2,\ldots,k\right\}  $. We must show that $A_{i}^{-1}$ is a
lower addition matrix.
\par
We know that $A_{i}$ is a lower addition matrix (since $A_{1},A_{2}%
,\ldots,A_{k}$ are lower addition matrices). In other words, $A_{i}$ has the
form $A_{i}=A_{u,v}^{\lambda}$, where $\lambda$ is a number, and where $u$ and
$v$ are two elements of $\left\{  1,2,\ldots,n\right\}  $ satisfying $u>v$ (by
the definition of a \textquotedblleft lower addition matrix\textquotedblright%
). Consider these $\lambda$, $u$ and $v$. Proposition
\ref{prop.Alamuv.lambda+mu} \textbf{(c)} shows that the matrix $A_{u,v}%
^{\lambda}$ is invertible, and that its inverse is $\left(  A_{u,v}^{\lambda
}\right)  ^{-1}=A_{u,v}^{-\lambda}$.
\par
But $A_{u,v}^{-\lambda}$ is a lower addition matrix (since $-\lambda$ is a
number, and since $u>v$). In other words, $A_{i}^{-1}$ is a lower addition
matrix (since $\left(  \underbrace{A_{i}}_{=A_{u,v}^{\lambda}}\right)
^{-1}=\left(  A_{u,v}^{\lambda}\right)  ^{-1}=A_{u,v}^{-\lambda}$). This
completes our proof.}. In other words, the matrices $A_{1},A_{2},\ldots,A_{k}$
are invertible, and their inverses $A_{1}^{-1},A_{2}^{-1},\ldots,A_{k}^{-1}$
are again lower addition matrices. In other words, $A_{k}^{-1},A_{k-1}%
^{-1},\ldots,A_{1}^{-1}$ are lower addition matrices.

Now, Proposition \ref{prop.inverses.A1Ak} shows that the matrix $A_{1}%
A_{2}\cdots A_{k}$ is invertible, and its inverse is $\left(  A_{1}A_{2}\cdots
A_{k}\right)  ^{-1}=A_{k}^{-1}A_{k-1}^{-1}\cdots A_{1}^{-1}$. Since
$A=A_{1}A_{2}\cdots A_{k}$, this rewrites as follows: The matrix $A$ is
invertible, and its inverse is $A^{-1}=A_{k}^{-1}A_{k-1}^{-1}\cdots A_{1}%
^{-1}$.

The equality $A^{-1}=A_{k}^{-1}A_{k-1}^{-1}\cdots A_{1}^{-1}$ shows that
$A^{-1}$ is a product of lower addition matrices (since $A_{k}^{-1}%
,A_{k-1}^{-1},\ldots,A_{1}^{-1}$ are lower addition matrices). But Theorem
\ref{thm.triangular.Alamuv} (applied to $C=A^{-1}$) shows that $A^{-1}$ is
lower-unitriangular if and only if $A^{-1}$ is a product of lower addition
matrices. Hence, $A^{-1}$ is lower-unitriangular (since $A^{-1}$ is a product
of lower addition matrices). This completes the proof of Theorem
\ref{thm.triangular.inverse} in Case 1.

Case 2 is trivial (indeed, $A=I_{n}$ in this case) and is left to the
reader.\footnote{Alternatively, our proof for Case 1 can be made to work in
Case 2 as well, because Proposition \ref{prop.inverses.A1Ak} holds for $k=0$
(as long as we define the empty product to be $I_{n}$). See Remark
\ref{rmk.prop.inverses.A1Ak.k=0} for the details.} Thus, Theorem
\ref{thm.triangular.inverse} is proven in both Cases 1 and 2; this shows that
Theorem \ref{thm.triangular.inverse} is always valid.
\end{proof}

A similar result holds for upper-triangular matrices:

\begin{theorem}
\label{thm.triangular.inverse-up}Let $n\in\mathbb{N}$. Let $A$ be an
upper-unitriangular $n\times n$-matrix. Then, $A$ is invertible, and its
inverse $A^{-1}$ is again upper-unitriangular.
\end{theorem}

We could prove Theorem \ref{thm.triangular.inverse-up} by modifying our above
proof of Theorem \ref{thm.triangular.inverse} (replacing \textquotedblleft
lower\textquotedblright\ by \textquotedblleft upper\textquotedblright%
\ everywhere); of course, this would necessitate an analogue for Theorem
\ref{thm.triangular.Alamuv} concerning upper-unitriangular instead of
lower-unitriangular matrices (and upper addition matrices instead of lower
addition matrices\footnote{Of course, these upper addition matrices are
defined exactly as you would expect: They are the matrices of the form
$A_{u,v}^{\lambda}$, where $\lambda$ is a number, and where $u$ and $v$ are
two elements of $\left\{  1,2,\ldots,n\right\}  $ satisfying $u<v$.}). The
proof of this analogue would then proceed similarly to our proof of Theorem
\ref{thm.triangular.Alamuv}, but again with some changes (e.g., instead of
clearing out the columns from the first to the last, we would have to clear
out the columns from the last to the first, and we would achieve this using
\textquotedblleft upward row additions\textquotedblright).

However, we can also quickly derive Theorem \ref{thm.triangular.inverse-up}
from Theorem \ref{thm.triangular.inverse} using transposes:

\begin{proof}
[Proof of Theorem \ref{thm.triangular.inverse-up}.]We know that $A$ is
upper-unitriangular. Hence, by Proposition \ref{prop.triangular.UT=D.uni}
\textbf{(a)}, we can conclude that $A^{T}$ is lower-unitriangular. Therefore,
we can apply Theorem \ref{thm.triangular.inverse} to $A^{T}$ instead of $A$.
As a result, we see that $A^{T}$ is invertible, and its inverse $\left(
A^{T}\right)  ^{-1}$ is again lower-unitriangular.

Hence, we can apply Proposition \ref{prop.transpose.opers} \textbf{(f)} to
$A^{T}$ instead of $A$. Thus, we conclude that the matrix $\left(
A^{T}\right)  ^{T}$ is invertible, and its inverse is $\left(  \left(
A^{T}\right)  ^{T}\right)  ^{-1}=\left(  \left(  A^{T}\right)  ^{-1}\right)
^{T}$. Since $\left(  A^{T}\right)  ^{T}=A$ (by Proposition
\ref{prop.transpose.invol}, applied to $m=n$), this rewrites as follows: The
matrix $A$ is invertible, and its inverse is $A^{-1}=\left(  \left(
A^{T}\right)  ^{-1}\right)  ^{T}$.

It remains to prove that $A^{-1}$ is upper-unitriangular. This is again quite
easy: We have%
\[
\left(  \underbrace{A^{-1}}_{=\left(  \left(  A^{T}\right)  ^{-1}\right)
^{T}}\right)  ^{T}=\left(  \left(  \left(  A^{T}\right)  ^{-1}\right)
^{T}\right)  ^{T}=\left(  A^{T}\right)  ^{-1}%
\]
(by Proposition \ref{prop.transpose.invol}, applied to $n$ and $\left(
A^{T}\right)  ^{-1}$ instead of $m$ and $A$). Hence, $\left(  A^{-1}\right)
^{T}$ is lower-unitriangular (because we already know that $\left(
A^{T}\right)  ^{-1}$ is lower-unitriangular). But Proposition
\ref{prop.triangular.UT=D.uni} \textbf{(a)} (applied to $A^{-1}$ instead of
$A$) shows that $A^{-1}$ is upper-unitriangular if and only if $\left(
A^{-1}\right)  ^{T}$ is lower-unitriangular. Since $\left(  A^{-1}\right)
^{T}$ is lower-unitriangular, we thus conclude that $A^{-1}$ is
upper-unitriangular. This completes our proof of Theorem
\ref{thm.triangular.inverse-up}.
\end{proof}

\subsection{(*) Products of strictly triangular matrices}

In this section, I shall give a different proof of Theorem
\ref{thm.triangular.inverse}, which is somewhat of a digression from our road
towards Gaussian elimination, but has the advantage of showing off some other
ideas. The proof is motivated by the following example:

\begin{example}
\label{exam.triangular.strict.nilp}Let $A$ be a strictly lower-triangular
$4\times4$-matrix. Thus, $A$ has the form $\left(
\begin{array}
[c]{cccc}%
0 & 0 & 0 & 0\\
a & 0 & 0 & 0\\
b & c & 0 & 0\\
d & e & f & 0
\end{array}
\right)  $ for some numbers $a,b,c,d,e,f$. How do the powers of $A$ look like?

Well, we can just compute the first few of them and see what happens:%
\begin{align*}
A^{1}  &  =\left(
\begin{array}
[c]{cccc}%
0 & 0 & 0 & 0\\
a & 0 & 0 & 0\\
b & c & 0 & 0\\
d & e & f & 0
\end{array}
\right)  ,\ \ \ \ \ \ \ \ \ \ A^{2}=\left(
\begin{array}
[c]{cccc}%
0 & 0 & 0 & 0\\
0 & 0 & 0 & 0\\
ac & 0 & 0 & 0\\
ae+bf & cf & 0 & 0
\end{array}
\right)  ,\\
A^{3}  &  =\left(
\begin{array}
[c]{cccc}%
0 & 0 & 0 & 0\\
0 & 0 & 0 & 0\\
0 & 0 & 0 & 0\\
acf & 0 & 0 & 0
\end{array}
\right)  ,\ \ \ \ \ \ \ \ \ \ A^{4}=\left(
\begin{array}
[c]{cccc}%
0 & 0 & 0 & 0\\
0 & 0 & 0 & 0\\
0 & 0 & 0 & 0\\
0 & 0 & 0 & 0
\end{array}
\right)  .
\end{align*}
Two things strike the eye:

\begin{itemize}
\item We have $A^{4}=0_{4\times4}$. As a consequence, every $n\geq4$ satisfies
$A^{n}=\underbrace{A^{4}}_{=0_{4\times4}}A^{n-4}=0_{4\times4}A^{n-4}%
=0_{4\times4}$. So we actually know all the powers of $A$.

\item Every time we pass from one power of $A$ to the next (for example, from
$A^{2}$ to $A^{3}$), the nonzero entries recede one level further towards the
bottom-left corner. At the step from $A^{3}$ to $A^{4}$, they finally recede
beyond that corner, so that only zeroes are left in the matrix.
\end{itemize}
\end{example}

The pattern seen in Example \ref{exam.triangular.strict.nilp} actually
generalizes: The powers of a strictly lower-triangular $n\times n$-matrix
behave similarly, except that this time we have $A^{n}=0_{n\times n}$ instead
of $A^{4}=0_{4\times4}$. Moreover, the same rule holds more generally if we
multiply several strictly lower-triangular $n\times n$-matrices (instead of
taking powers of one such matrix). In order to explore this more rigorously, I
shall introduce a (nonstandard) notion:

\begin{definition}
\label{def.triangular.strict.k}Let $n\in\mathbb{N}$. Let $A$ be an $n\times
n$-matrix. Let $k\in\mathbb{Z}$. We say that the matrix $A$ is $k$%
\textit{-lower-triangular} if and only if we have%
\[
A_{i,j}=0\ \ \ \ \ \ \ \ \ \ \text{whenever }i<j+k.
\]

\end{definition}

\begin{example}
Visually speaking, a square matrix $A$ is $k$-lower-triangular if and only if
its nonzero entries begin no earlier than $k$ levels below the main diagonal.
For example:

\begin{itemize}
\item Any $4\times4$-matrix is $k$-lower-triangular for $k\leq-3$.

\item A $4\times4$-matrix is $\left(  -1\right)  $-lower-triangular if and
only if it has the form $\left(
\begin{array}
[c]{cccc}%
a & b & 0 & 0\\
c & d & e & 0\\
f & g & h & i\\
j & k & l & m
\end{array}
\right)  $ for some numbers $a,b,c,d,e,f,g,h,i,j,k,l,m$.

\item A $4\times4$-matrix is $0$-lower-triangular if and only if it has the
form $\left(
\begin{array}
[c]{cccc}%
a & 0 & 0 & 0\\
b & c & 0 & 0\\
d & e & f & 0\\
g & h & i & j
\end{array}
\right)  $ for some numbers $a,b,c,d,e,f,g,h,i,j$. This is, of course, the
same as being lower-triangular.

\item A $4\times4$-matrix is $1$-lower-triangular if and only if it has the
form $\left(
\begin{array}
[c]{cccc}%
0 & 0 & 0 & 0\\
a & 0 & 0 & 0\\
b & c & 0 & 0\\
d & e & f & 0
\end{array}
\right)  $ for some numbers $a,b,c,d,e,f$. This is the same as being strictly lower-triangular.

\item A $4\times4$-matrix is $2$-lower-triangular if and only if it has the
form $\left(
\begin{array}
[c]{cccc}%
0 & 0 & 0 & 0\\
0 & 0 & 0 & 0\\
a & 0 & 0 & 0\\
b & c & 0 & 0
\end{array}
\right)  $ for some numbers $a,b,c$.

\item A $4\times4$-matrix is $3$-lower-triangular if and only if it has the
form $\left(
\begin{array}
[c]{cccc}%
0 & 0 & 0 & 0\\
0 & 0 & 0 & 0\\
0 & 0 & 0 & 0\\
a & 0 & 0 & 0
\end{array}
\right)  $ for some number $a$.

\item A $4\times4$-matrix is $4$-lower-triangular if and only if it has the
form $\left(
\begin{array}
[c]{cccc}%
0 & 0 & 0 & 0\\
0 & 0 & 0 & 0\\
0 & 0 & 0 & 0\\
0 & 0 & 0 & 0
\end{array}
\right)  $ (i.e., if and only if it is the zero matrix $0_{4\times4}$). The
same holds for $5$-lower-triangular matrices and higher on.
\end{itemize}
\end{example}

We state some simple facts (which should have been clear from the example already):

\begin{proposition}
\label{prop.triangular.strict.k.values}Let $n\in\mathbb{N}$. Let $A$ be an
$n\times n$-matrix.

\textbf{(a)} The matrix $A$ is $k$-lower-triangular for every integer $k$
satisfying $k\leq-n+1$.

\textbf{(b)} The matrix $A$ is $0$-lower-triangular if and only if $A$ is lower-triangular.

\textbf{(c)} The matrix $A$ is $1$-lower-triangular if and only if $A$ is
strictly lower-triangular.

\textbf{(d)} Let $k$ be an integer such that $k\geq n$. The matrix $A$ is
$k$-lower-triangular if and only if $A=0_{n\times n}$.
\end{proposition}

\begin{proof}
[Proof of Proposition \ref{prop.triangular.strict.k.values}.]\textbf{(a)} Let
$k$ be an integer satisfying $k\leq-n+1$. We shall show that the matrix $A$ is
$k$-lower-triangular. In order to do so, we must prove that%
\begin{equation}
A_{i,j}=0\ \ \ \ \ \ \ \ \ \ \text{whenever }i<j+k
\label{pf.prop.triangular.strict.k.values.a.goal}%
\end{equation}
(because this is how \textquotedblleft$k$-lower-triangular\textquotedblright%
\ was defined).

This is an example of a \textquotedblleft vacuously true\textquotedblright%
\ statement. Let me explain the concept: A logical statement of the form
\textquotedblleft if $\mathcal{A}$, then $\mathcal{B}$\textquotedblright\ is
said to be \textit{vacuously true} if $\mathcal{A}$ never holds. For example,
the statement \textquotedblleft if a positive integer $n$ is negative, then
$n=15$\textquotedblright\ is vacuously true, since a positive integer $n$ will
never be negative in the first place. Similarly, the statement
\textquotedblleft we have $n=m$ for any two integers $n$ and $m$ satisfying
$n=m+\dfrac{1}{2}$\textquotedblright\ is also vacuously true, since two
integers $n$ and $m$ will never satisfy $n=m+\dfrac{1}{2}$. (I have not worded
this statement as an \textquotedblleft if $\mathcal{A}$, then $\mathcal{B}%
$\textquotedblright\ statement, but I could easily have done so, by rewriting
it as \textquotedblleft if two integers $n$ and $m$ satisfy $n=m+\dfrac{1}{2}%
$, then $n=m$\textquotedblright. The wording doesn't matter much.) Finally,
the statement \textquotedblleft every element of the empty set is a prime
number\textquotedblright\ is also vacuously true, since there is no element of
the empty set. (Again, you can rewrite this statement as \textquotedblleft if
$a$ is an element of the empty set, then $a$ is a prime
number\textquotedblright\ in order to bring it into the \textquotedblleft if
$\mathcal{A}$, then $\mathcal{B}$\textquotedblright\ form.)

As the name suggests, mathematicians consider vacuously true statements to be
true. The reasoning here is that, as long as you say nothing (and vacuously
true statements say nothing, in a sense), you remain truthful.

We are now going to prove that the statement
(\ref{pf.prop.triangular.strict.k.values.a.goal}) is vacuously true. In other
words, we are going to prove that two elements $i$ and $j$ of $\left\{
1,2,\ldots,n\right\}  $ never satisfy $i<j+k$ in the first place.

In fact, let $i$ and $j$ be two elements of $\left\{  1,2,\ldots,n\right\}  $
that satisfy $i<j+k$. Then, $i\geq1$ (since $i\in\left\{  1,2,\ldots
,n\right\}  $) and $j\leq n$ (since $j\in\left\{  1,2,\ldots,n\right\}  $), so
that $\underbrace{j}_{\leq n}+\underbrace{k}_{\leq-n+1}\leq n+\left(
-n+1\right)  =1\leq i$ (since $i\geq1$). This contradicts $i<j+k$.

Now, forget that we fixed $i$ and $j$. We thus have found a contradiction
\textbf{for any two elements }$i$ and $j$ of $\left\{  1,2,\ldots,n\right\}  $
that satisfy $i<j+k$. This shows that there exist no two elements $i$ and $j$
of $\left\{  1,2,\ldots,n\right\}  $ that satisfy $i<j+k$. In other words, two
elements $i$ and $j$ of $\left\{  1,2,\ldots,n\right\}  $ never satisfy
$i<j+k$. Thus, the statement (\ref{pf.prop.triangular.strict.k.values.a.goal})
is vacuously true, and therefore true. In other words, the matrix $A$ is
$k$-lower-triangular (by the definition of \textquotedblleft$k$%
-lower-triangular\textquotedblright). This proves Proposition
\ref{prop.triangular.strict.k.values} \textbf{(a)}.

(Again, we shall be a lot briefer in proofs like this in the future.)

\textbf{(b)} Behold the following chain of equivalent
statements\footnote{After each equivalence, we give a justification for why it
is an equivalence.}:%
\begin{align*}
&  \ \left(  A\text{ is }0\text{-lower-triangular}\right) \\
&  \Longleftrightarrow\ \left(  A_{i,j}=0\text{ whenever }i<j+0\right) \\
&  \ \ \ \ \ \ \ \ \ \ \left(  \text{because this is how \textquotedblleft%
}0\text{-lower-triangular\textquotedblright\ is defined}\right) \\
&  \Longleftrightarrow\ \left(  A_{i,j}=0\text{ whenever }i<j\right) \\
&  \ \ \ \ \ \ \ \ \ \ \left(  \text{here, we have replaced }j+0\text{ by
}j\text{, since every }j\text{ satisfies }j+0=j\right) \\
&  \Longleftrightarrow\ \left(  A\text{ is lower-triangular}\right) \\
&  \ \ \ \ \ \ \ \ \ \ \left(  \text{because this is how \textquotedblleft
lower-triangular\textquotedblright\ is defined}\right)  .
\end{align*}
This proves Proposition \ref{prop.triangular.strict.k.values} \textbf{(b)}.

\textbf{(c)} $\Longrightarrow:$ We must prove that if $A$ is $1$%
-lower-triangular, then $A$ is strictly lower-triangular.

Indeed, assume that $A$ is $1$-lower-triangular. In other words,%
\begin{equation}
A_{i,j}=0\ \ \ \ \ \ \ \ \ \ \text{whenever }i<j+1
\label{pf.prop.triangular.strict.k.values.c.1}%
\end{equation}
(because this is how \textquotedblleft$1$-lower-triangular\textquotedblright%
\ is defined).

Now, we have $A_{i,j}=0$ whenever $i<j$\ \ \ \ \footnote{\textit{Proof.} Let
$i$ and $j$ be two elements of $\left\{  1,2,\ldots,n\right\}  $ such that
$i<j$. Then, $i<j<j+1$. Hence, (\ref{pf.prop.triangular.strict.k.values.c.1})
shows that $A_{i,j}=0$. Qed.}. In other words, $A$ is lower-triangular (by the
definition of \textquotedblleft lower-triangular\textquotedblright). Moreover,
every $i\in\left\{  1,2,\ldots,n\right\}  $ satisfies $i<i+1$ and thus
$A_{i,i}=0$ (by (\ref{pf.prop.triangular.strict.k.values.c.1}), applied to
$j=i$). In other words, all diagonal entries of $A$ are $0$.

So we have shown that the matrix $A$ is lower-triangular, and that all its
diagonal entries are $0$. In other words, $A$ is strictly lower-triangular (by
the definition of \textquotedblleft strictly
lower-triangular\textquotedblright). This proves the $\Longrightarrow$
direction of Proposition \ref{prop.triangular.strict.k.values} \textbf{(c)}.

$\Longleftarrow:$ We must prove that if $A$ is strictly lower-triangular, then
$A$ is $1$-lower-triangular.

Indeed, assume that $A$ is strictly lower-triangular. According to the
definition of \textquotedblleft strictly lower-triangular\textquotedblright,
this means that the matrix $A$ is lower-triangular, and that all its diagonal
entries are $0$.

The matrix $A$ is lower-triangular; in other words,%
\begin{equation}
A_{i,j}=0\ \ \ \ \ \ \ \ \ \ \text{whenever }i<j
\label{pf.prop.triangular.strict.k.values.c.6}%
\end{equation}
(according to the definition of \textquotedblleft
lower-triangular\textquotedblright). Also, all diagonal entries of $A$ are
$0$; in other words,%
\begin{equation}
A_{i,i}=0\ \ \ \ \ \ \ \ \ \ \text{for each }i\in\left\{  1,2,\ldots
,n\right\}  . \label{pf.prop.triangular.strict.k.values.c.7}%
\end{equation}


Now, we can easily see that%
\[
A_{i,j}=0\ \ \ \ \ \ \ \ \ \ \text{whenever }i<j+1
\]
\footnote{\textit{Proof.} Let $i$ and $j$ be two elements of $\left\{
1,2,\ldots,n\right\}  $ such that $i<j+1$. We must prove that $A_{i,j}=0$.
\par
We are in one of the following two cases:
\par
\textit{Case 1:} We have $i=j$.
\par
\textit{Case 2:} We have $i\neq j$.
\par
Let us first consider Case 1. In this case, we have $i=j$. Hence, $j=i$, so
that $A_{i,j}=A_{i,i}=0$ (by (\ref{pf.prop.triangular.strict.k.values.c.7})).
Hence, $A_{i,j}=0$ is proven in Case 1.
\par
Let us now consider Case 2. In this case, we have $i\neq j$. On the other
hand, $i<j+1$, so that $i\leq\left(  j+1\right)  -1$ (since $i$ and $j+1$ are
integers). Thus, $i\leq\left(  j+1\right)  -1=j$. Combining this with $i\neq
j$, we obtain $i<j$. Hence, (\ref{pf.prop.triangular.strict.k.values.c.6})
shows that $A_{i,j}=0$. Thus, $A_{i,j}=0$ is proven in Case 2.
\par
We now have proven $A_{i,j}=0$ in each of the two Cases 1 and 2. Thus,
$A_{i,j}=0$ always holds, qed.}. But this means precisely that $A$ is
$1$-lower-triangular (because this is how \textquotedblleft$1$%
-lower-triangular\textquotedblright\ is defined). Thus, we have shown that $A$
is $1$-lower-triangular. This proves the $\Longleftarrow$ direction of
Proposition \ref{prop.triangular.strict.k.values} \textbf{(c)}.

Now, the proof of Proposition \ref{prop.triangular.strict.k.values}
\textbf{(c)} is complete (since both its $\Longrightarrow$ and its
$\Longleftarrow$ directions are proven).

\textbf{(d)} $\Longrightarrow:$ We must prove that if $A$ is $k$%
-lower-triangular, then $A=0_{n\times n}$.

Indeed, assume that the matrix $A$ is $k$-lower-triangular. In other words,%
\begin{equation}
A_{i,j}=0\ \ \ \ \ \ \ \ \ \ \text{whenever }i<j+k
\label{pf.prop.triangular.strict.k.values.d.1}%
\end{equation}
(by the definition of \textquotedblleft$k$-lower-triangular\textquotedblright).

Now, let $i\in\left\{  1,2,\ldots,n\right\}  $ and $j\in\left\{
1,2,\ldots,n\right\}  $ be arbitrary. Then, $j\geq1$ (since $j\in\left\{
1,2,\ldots,n\right\}  $) and $i\leq n$ (since $i\in\left\{  1,2,\ldots
,n\right\}  $). Now, $\underbrace{j}_{\geq1}+\underbrace{k}_{\geq n}\geq
1+n>n$, so that $n<j+k$ and thus $i\leq n<j+k$. Hence, $A_{i,j}=0$ (by
(\ref{pf.prop.triangular.strict.k.values.d.1})). Comparing this with $\left(
0_{n\times n}\right)  _{i,j}=0$ (since each entry of the matrix $0_{n\times
n}$ is $0$), we obtain $A_{i,j}=\left(  0_{n\times n}\right)  _{i,j}$.

Now, let us forget that we fixed $i$ and $j$. We thus have shown that
$A_{i,j}=\left(  0_{n\times n}\right)  _{i,j}$ for each $i\in\left\{
1,2,\ldots,n\right\}  $ and $j\in\left\{  1,2,\ldots,n\right\}  $. In other
words, each entry of the matrix $A$ equals the corresponding entry of the
matrix $0_{n\times n}$. In other words, $A=0_{n\times n}$. This proves the
$\Longrightarrow$ direction of Proposition
\ref{prop.triangular.strict.k.values} \textbf{(d)}.

$\Longleftarrow:$ We must prove that if $A=0_{n\times n}$, then $A$ is $k$-lower-triangular.

If $i$ and $j$ are two elements of $\left\{  1,2,\ldots,n\right\}  $
satisfying $i<j+k$, then%
\begin{align*}
A_{i,j}  &  =\left(  0_{n\times n}\right)  _{i,j}\ \ \ \ \ \ \ \ \ \ \left(
\text{since }A=0_{n\times n}\right) \\
&  =0\ \ \ \ \ \ \ \ \ \ \left(  \text{since each entry of the matrix
}0_{n\times n}\text{ is }0\right)  .
\end{align*}
In other words, $A_{i,j}=0$ whenever $i<j+k$. But this means precisely that
the matrix $A$ is $k$-lower-triangular (by the definition of \textquotedblleft%
$k$-lower-triangular\textquotedblright). Hence, we have shown that $A$ is
$k$-lower-triangular. This proves the $\Longleftarrow$ direction of
Proposition \ref{prop.triangular.strict.k.values} \textbf{(d)}.

Now, Proposition \ref{prop.triangular.strict.k.values} \textbf{(d)} is proven
(since both its $\Longrightarrow$ and its $\Longleftarrow$ directions are proven).
\end{proof}

Next, we state a fact which is crucial for our argument:

\begin{proposition}
\label{prop.triangular.strict.k+l}Let $n\in\mathbb{N}$. Let $p$ and $q$ be two
integers. Let $A$ be a $p$-lower-triangular $n\times n$-matrix. Let $B$ be a
$q$-lower-triangular $n\times n$-matrix. Then, $AB$ is a $\left(  p+q\right)
$-lower-triangular $n\times n$-matrix.
\end{proposition}

\begin{remark}
Proposition \ref{prop.triangular.strict.k+l} generalizes Theorem
\ref{thm.triangular.prod-down} \textbf{(a)}. In fact, recall that an $n\times
n$-matrix is $0$-lower-triangular if and only if it is lower-triangular (by
Proposition \ref{prop.triangular.strict.k.values} \textbf{(b)}). Hence,
applying Proposition \ref{prop.triangular.strict.k+l} to $p=0$ and $q=0$, we
obtain precisely Theorem \ref{thm.triangular.prod-down} \textbf{(a)}.
\end{remark}

\begin{proof}
[Proof of Proposition \ref{prop.triangular.strict.k+l}.]We shall imitate our
above proof of Theorem \ref{thm.triangular.prod-down} \textbf{(a)} as well as
we can.

The matrix $A$ is $p$-lower-triangular. In other words,%
\begin{equation}
A_{i,j}=0\ \ \ \ \ \ \ \ \ \ \text{whenever }i<j+p
\label{pf.prop.triangular.strict.k+l.A-tri}%
\end{equation}
(because this is what it means for $A$ to be $p$-lower-triangular).

The matrix $B$ is $q$-lower-triangular. In other words,%
\begin{equation}
B_{i,j}=0\ \ \ \ \ \ \ \ \ \ \text{whenever }i<j+q
\label{pf.prop.triangular.strict.k+l.B-tri}%
\end{equation}
(because this is what it means for $B$ to be $q$-lower-triangular).

Now, fix two elements $i$ and $j$ of $\left\{  1,2,\ldots,n\right\}  $
satisfying $i<j+\left(  p+q\right)  $. We shall prove that for every
$k\in\left\{  1,2,\ldots,n\right\}  $, we have
\begin{equation}
A_{i,k}B_{k,j}=0. \label{pf.prop.triangular.strict.k+l.AB0}%
\end{equation}


[\textit{Proof of (\ref{pf.prop.triangular.strict.k+l.AB0}):} Let
$k\in\left\{  1,2,\ldots,n\right\}  $. Then, we are in one of the following
two cases:

\begin{statement}
\textit{Case 1:} We have $i\geq k+p$.
\end{statement}

\begin{statement}
\textit{Case 2:} We have $i<k+p$.
\end{statement}

We shall prove (\ref{pf.prop.triangular.strict.k+l.AB0}) in each of these two
cases separately:

\begin{enumerate}
\item Let us first consider Case 1. In this case, we have $i\geq k+p$. Thus,
$k+p\leq i<j+\left(  p+q\right)  =\left(  j+q\right)  +p$. Subtracting $p$
from both sides of this inequality, we obtain $k<j+q$. Hence, we can apply
(\ref{pf.prop.triangular.strict.k+l.B-tri}) to $k$ instead of $i$. As a
result, we obtain $B_{k,j}=0$. Hence, $A_{i,k}\underbrace{B_{k,j}}%
_{=0}=A_{i,k}0=0$. Thus, (\ref{pf.prop.triangular.strict.k+l.AB0}) is proven
in Case 1.

\item Let us now consider Case 2. In this case, we have $i<k+p$. Hence, we can
apply (\ref{pf.prop.triangular.strict.k+l.A-tri}) to $k$ instead of $j$. As a
result, we obtain $A_{i,k}=0$. Hence, $\underbrace{A_{i,k}}_{=0}%
B_{k,j}=0B_{k,j}=0$. Thus, (\ref{pf.prop.triangular.strict.k+l.AB0}) is proven
in Case 2.
\end{enumerate}

We have now proven (\ref{pf.prop.triangular.strict.k+l.AB0}) in both Cases 1
and 2. Thus, (\ref{pf.prop.triangular.strict.k+l.AB0}) is proven.]

Now, Proposition \ref{prop.matrix-prod.rc} \textbf{(a)} shows that%
\begin{align*}
\left(  AB\right)  _{i,j}  &  =\underbrace{A_{i,1}B_{1,j}}%
_{\substack{=0\\\text{(by (\ref{pf.prop.triangular.strict.k+l.AB0}%
),}\\\text{applied to }k=1\text{)}}}+\underbrace{A_{i,2}B_{2,j}}%
_{\substack{=0\\\text{(by (\ref{pf.prop.triangular.strict.k+l.AB0}%
),}\\\text{applied to }k=2\text{)}}}+\cdots+\underbrace{A_{i,n}B_{n,j}%
}_{\substack{=0\\\text{(by (\ref{pf.prop.triangular.strict.k+l.AB0}%
),}\\\text{applied to }k=n\text{)}}}\\
&  =0+0+\cdots+0=0.
\end{align*}


Now, forget that we fixed $i$ and $j$. We thus have shown that%
\[
\left(  AB\right)  _{i,j}=0\ \ \ \ \ \ \ \ \ \ \text{whenever }i<j+\left(
p+q\right)  .
\]
But this says precisely that the matrix $AB$ is $\left(  p+q\right)
$-lower-triangular (by the definition of \textquotedblleft$\left(  p+q\right)
$-lower-triangular\textquotedblright). Thus, Proposition
\ref{prop.triangular.strict.k+l} is proven.
\end{proof}

Using Proposition \ref{prop.triangular.strict.k+l}, we can show the following fact:

\begin{corollary}
\label{cor.triangular.strict.kprod}Let $n\in\mathbb{N}$. Let $A_{1}%
,A_{2},\ldots,A_{k}$ be $k$ strictly lower-triangular $n\times n$-matrices
(where $k\in\mathbb{N}$). Then, the $n\times n$-matrix $A_{1}A_{2}\cdots
A_{k}$ is $k$-lower-triangular.
\end{corollary}

This is proven by induction similarly to how we proved Proposition
\ref{prop.inverses.A1Ak} (we are actually copying the structure of that proof):

\begin{proof}
[Proof of Corollary \ref{cor.triangular.strict.kprod}.]We prove Corollary
\ref{cor.triangular.strict.kprod} by induction on $k$:

\textit{Induction base:} If $k=0$, then Corollary
\ref{cor.triangular.strict.kprod} says that the $n\times n$-matrix $A_{1}%
A_{2}\cdots A_{0}$ is $0$-lower-triangular. But this is indeed
true\footnote{\textit{Proof.} The product $A_{1}A_{2}\cdots A_{0}$ is an empty
product of $n\times n$-matrices, and thus equals $I_{n}$ (by definition).
\par
But Proposition \ref{prop.triangular.strict.k.values} \textbf{(b)} (applied to
$A=I_{n}$) shows that the matrix $I_{n}$\textbf{ }is $0$-lower-triangular if
and only if $I_{n}$ is lower-triangular. Hence, the matrix $I_{n}$ is
$0$-lower-triangular (since $I_{n}$ is lower-triangular). In other words, the
$n\times n$-matrix $A_{1}A_{2}\cdots A_{0}$ is $0$-lower-triangular (since
$A_{1}A_{2}\cdots A_{0}=I_{n}$). Qed.}. Hence, Corollary
\ref{cor.triangular.strict.kprod} holds for $k=0$. This completes the
induction base.

\textit{Induction step:} Let $\ell$ be a positive integer. Assume (as our
\textit{induction hypothesis}) that Corollary
\ref{cor.triangular.strict.kprod} holds for $k=\ell$. In other words, for any
$\ell$ strictly lower-triangular $n\times n$-matrices $A_{1},A_{2}%
,\ldots,A_{\ell}$, the $n\times n$-matrix $A_{1}A_{2}\cdots A_{\ell}$ is
$\ell$-lower-triangular.

We must now show that Corollary \ref{cor.triangular.strict.kprod} also holds
for $k=\ell+1$. So let us fix $\ell+1$ strictly lower-triangular $n\times
n$-matrices $A_{1},A_{2},\ldots,A_{\ell+1}$. We must then show that the
$n\times n$-matrix $A_{1}A_{2}\cdots A_{\ell+1}$ is $\left(  \ell+1\right)  $-lower-triangular.

Clearly, $A_{1},A_{2},\ldots,A_{\ell}$ are $\ell$ strictly lower-triangular
$n\times n$-matrices (since $A_{1},A_{2},\ldots,A_{\ell+1}$ are $\ell+1$
strictly lower-triangular $n\times n$-matrices). Thus, we can apply our
induction hypothesis, and conclude that the $n\times n$-matrix $A_{1}%
A_{2}\cdots A_{\ell}$ is $\ell$-lower-triangular.

But the $n\times n$-matrix $A_{\ell+1}$ is $1$%
-lower-triangular\footnote{\textit{Proof.} We know that $A_{1},A_{2}%
,\ldots,A_{\ell+1}$ are $\ell+1$ strictly lower-triangular $n\times
n$-matrices. In particular, $A_{\ell+1}$ is a strictly lower-triangular
$n\times n$-matrix.
\par
But Proposition \ref{prop.triangular.strict.k.values} \textbf{(c)} (applied to
$A=A_{\ell+1}$) shows that the matrix $A_{\ell+1}$ is $1$-lower-triangular if
and only if $A_{\ell+1}$ is strictly lower-triangular. Hence, the matrix
$A_{\ell+1}$ is $1$-lower-triangular (since $A_{\ell+1}$ is strictly
lower-triangular). Qed.}. Thus, we can apply Proposition
\ref{prop.triangular.strict.k+l} to $p=\ell$, $q=1$, $A=A_{1}A_{2}\cdots
A_{\ell}$ and $B=A_{\ell+1}$. As a result, we conclude that $\left(
A_{1}A_{2}\cdots A_{\ell}\right)  A_{\ell+1}$ is an $\left(  \ell+1\right)
$-lower-triangular $n\times n$-matrix. Since $\left(  A_{1}A_{2}\cdots
A_{\ell}\right)  A_{\ell+1}=A_{1}A_{2}\cdots A_{\ell+1}$, this rewrites as
follows: $A_{1}A_{2}\cdots A_{\ell+1}$ is an $\left(  \ell+1\right)
$-lower-triangular $n\times n$-matrix. In other words, the $n\times n$-matrix
$A_{1}A_{2}\cdots A_{\ell+1}$ is $\left(  \ell+1\right)  $-lower-triangular.
This is precisely what we wanted to show! Thus, Corollary
\ref{cor.triangular.strict.kprod} holds for $k=\ell+1$. This completes the
induction step. Thus, Corollary \ref{cor.triangular.strict.kprod} is proven by induction.
\end{proof}

As a consequence of Corollary \ref{cor.triangular.strict.kprod}, we obtain the
following fact, which we experimentally observed right after Example
\ref{exam.triangular.strict.nilp}:

\begin{corollary}
\label{cor.triangular.strict.nprod}Let $n\in\mathbb{N}$. Let $A_{1}%
,A_{2},\ldots,A_{n}$ be $n$ strictly lower-triangular $n\times n$-matrices.
Then, $A_{1}A_{2}\cdots A_{n}=0_{n\times n}$.
\end{corollary}

\begin{proof}
[Proof of Corollary \ref{cor.triangular.strict.nprod}.]Corollary
\ref{cor.triangular.strict.kprod} (applied to $k=n$) shows that the $n\times
n$-matrix $A_{1}A_{2}\cdots A_{n}$ is $n$-lower-triangular. But we have $n\geq
n$. Hence, Proposition \ref{prop.triangular.strict.k.values} \textbf{(d)}
(applied to $k=n$ and $A=A_{1}A_{2}\cdots A_{n}$) shows that the matrix
$A_{1}A_{2}\cdots A_{n}$ is $n$-lower-triangular if and only if $A_{1}%
A_{2}\cdots A_{n}=0_{n\times n}$. Hence, $A_{1}A_{2}\cdots A_{n}=0_{n\times
n}$ (since we know that the matrix $A_{1}A_{2}\cdots A_{n}$ is $n$%
-lower-triangular). Corollary \ref{cor.triangular.strict.nprod} is thus proven.
\end{proof}

The following corollary is obtained as a particular case of Corollary
\ref{cor.triangular.strict.nprod} when we set all the $n$ matrices
$A_{1},A_{2},\ldots,A_{n}$ equal to one and the same matrix $A$:

\begin{corollary}
\label{cor.triangular.strict.npow}Let $n\in\mathbb{N}$. Let $A$ be a strictly
lower-triangular $n\times n$-matrix. Then, $A^{n}=0_{n\times n}$.
\end{corollary}

\begin{proof}
[Proof of Corollary \ref{cor.triangular.strict.npow}.]Clearly,
$\underbrace{A,A,\ldots,A}_{n\text{ times}}$ are $n$ strictly lower-triangular
$n\times n$-matrices. Thus, Corollary \ref{cor.triangular.strict.nprod}
(applied to $A_{i}=A$) shows that $\underbrace{AA\cdots A}_{n\text{ times}%
}=0_{n\times n}$. Now, $A^{n}=\underbrace{AA\cdots A}_{n\text{ times}%
}=0_{n\times n}$. Corollary \ref{cor.triangular.strict.npow} is proven.
\end{proof}

Another corollary of the preceding results is the following:

\begin{corollary}
\label{cor.triangular.strict.kpow-weak}Let $n\in\mathbb{N}$. Let $k$ be a
positive integer. Let $A$ be a strictly lower-triangular $n\times n$-matrix.
Then, the $n\times n$-matrix $A^{k}$ is strictly lower-triangular.
\end{corollary}

\begin{proof}
[Proof of Corollary \ref{cor.triangular.strict.kpow-weak}.]Clearly,
$\underbrace{A,A,\ldots,A}_{k\text{ times}}$ are $k$ strictly lower-triangular
$n\times n$-matrices. Thus, Corollary \ref{cor.triangular.strict.kprod}
(applied to $A_{i}=A$) shows that the $n\times n$-matrix $\underbrace{AA\cdots
A}_{k\text{ times}}$ is $k$-lower-triangular. Since $\underbrace{AA\cdots
A}_{k\text{ times}}=A^{k}$, this rewrites as follows: The $n\times n$-matrix
$A^{k}$ is $k$-lower-triangular. In other words,%
\begin{equation}
\left(  A^{k}\right)  _{i,j}=0\ \ \ \ \ \ \ \ \ \ \text{whenever }i<j+k
\label{pf.cor.triangular.strict.kpow-weak.1}%
\end{equation}
(according to the definition of \textquotedblleft$k$%
-lower-triangular\textquotedblright).

But $k$ is a positive integer. Thus, $1\leq k$, so that $j+\underbrace{1}%
_{\leq k}\leq j+k$ for every $j\in\left\{  1,2,\ldots,n\right\}  $. Hence,
every $i\in\left\{  1,2,\ldots,n\right\}  $ and $j\in\left\{  1,2,\ldots
,n\right\}  $ satisfying $i<j+1$ must also satisfy $i<j+k$ (since $i<j+1\leq
j+k$) and consequently $\left(  A^{k}\right)  _{i,j}=0$ (by
(\ref{pf.cor.triangular.strict.kpow-weak.1})). In other words,
\[
\left(  A^{k}\right)  _{i,j}=0\ \ \ \ \ \ \ \ \ \ \text{whenever }i<j+1.
\]
In other words, the matrix $A^{k}$ is $1$-lower-triangular (by the definition
of \textquotedblleft$1$-lower-triangular\textquotedblright).

But Proposition \ref{prop.triangular.strict.k.values} \textbf{(c)} (applied to
$A^{k}$ instead of $A$) shows that the matrix $A^{k}$ is $1$-lower-triangular
if and only if $A^{k}$ is strictly lower-triangular. Hence, $A^{k}$ is
strictly lower-triangular (since $A^{k}$ is $1$-lower-triangular). This proves
Corollary \ref{cor.triangular.strict.kpow-weak}.
\end{proof}

So much for products of strictly lower-triangular matrices. What about their sums?

\begin{proposition}
\label{prop.triangular.strict.sum-of-stricts}Let $n\in\mathbb{N}$. Let
$A_{1},A_{2},\ldots,A_{k}$ be $k$ strictly lower-triangular $n\times
n$-matrices (where $k\in\mathbb{N}$). Then, $A_{1}+A_{2}+\cdots+A_{k}$ is a
strictly lower-triangular $n\times n$-matrix.
\end{proposition}

\begin{proof}
[Proof of Proposition \ref{prop.triangular.strict.sum-of-stricts}.]This is
left to the reader. (What makes this proof easy is that matrices are added
entry by entry.)
\end{proof}

Just as trivial is the following fact:

\begin{proposition}
\label{prop.triangular.strict.-strict}Let $n\in\mathbb{N}$. Let $A$ be a
strictly lower-triangular $n\times n$-matrix. Then, $-A$ is a strictly
lower-triangular $n\times n$-matrix.
\end{proposition}

\begin{proof}
[Proof of Proposition \ref{prop.triangular.strict.-strict}.]Left to the reader.
\end{proof}

Let us also state an analogue of Proposition \ref{prop.triangular.uni-inv} for
lower-triangular matrices:

\begin{proposition}
\label{prop.triangular.uni-inv.lower}\textbf{(a)} Each lower-unitriangular
matrix is invertibly lower-triangular.

\textbf{(b)} Let $n\in\mathbb{N}$. Let $A$ be an $n\times n$-matrix. Then, $A$
is lower-unitriangular if and only if $I_{n}-A$ is strictly lower-triangular.
\end{proposition}

\begin{proof}
[Proof of Proposition \ref{prop.triangular.uni-inv.lower}.]This is proven in
the same way as we proved Proposition \ref{prop.triangular.uni-inv} (once the
obvious modifications are made).
\end{proof}

Next, we show another simple fact:

\begin{proposition}
\label{prop.triangular.strict.geoseries}Let $n\in\mathbb{N}$. Let $A$ be an
$n\times n$-matrix such that $A^{n}=0_{n\times n}$. Then, the matrix $I_{n}-A$
is invertible, and its inverse is $\left(  I_{n}-A\right)  ^{-1}=A^{0}%
+A^{1}+\cdots+A^{n-1}$.
\end{proposition}

\begin{proof}
[Proof of Proposition \ref{prop.triangular.strict.geoseries}.]Let $B$ be the
$n\times n$-matrix $A^{0}+A^{1}+\cdots+A^{n-1}$.

Multiplying the equalities $A=A$ and $B=A^{0}+A^{1}+\cdots+A^{n-1}$, we obtain%
\begin{align*}
AB  &  =A\left(  A^{0}+A^{1}+\cdots+A^{n-1}\right)  =\underbrace{AA^{0}%
}_{=A^{1}}+\underbrace{AA^{1}}_{=A^{2}}+\cdots+\underbrace{AA^{n-1}}_{=A^{n}%
}\\
&  =A^{1}+A^{2}+\cdots+A^{n}=\left(  A^{1}+A^{2}+\cdots+A^{n-1}\right)
+\underbrace{A^{n}}_{=0_{n\times n}}=A^{1}+A^{2}+\cdots+A^{n-1}.
\end{align*}
Now,%
\begin{align*}
\left(  I_{n}-A\right)  B  &  =\underbrace{I_{n}B}_{=B=A^{0}+A^{1}%
+\cdots+A^{n-1}}-\underbrace{AB}_{=A^{1}+A^{2}+\cdots+A^{n-1}}\\
&  =\left(  A^{0}+A^{1}+\cdots+A^{n-1}\right)  -\left(  A^{1}+A^{2}%
+\cdots+A^{n-1}\right)  =A^{0}=I_{n}.
\end{align*}
A similar argument shows that $B\left(  I_{n}-A\right)  =I_{n}$. (To be more
precise: This is proven by multiplying the equalities $B=A^{0}+A^{1}%
+\cdots+A^{n-1}$ and $A=A$, as opposed to $A=A$ and $B=A^{0}+A^{1}%
+\cdots+A^{n-1}$.)

The two equalities $\left(  I_{n}-A\right)  B=I_{n}$ and $B\left(
I_{n}-A\right)  =I_{n}$ show that the matrix $B$ is an inverse of $I_{n}-A$.
Thus, the matrix $I_{n}-A$ is invertible, and its inverse is $\left(
I_{n}-A\right)  ^{-1}=B$. Hence, $\left(  I_{n}-A\right)  ^{-1}=B=A^{0}%
+A^{1}+\cdots+A^{n-1}$. The proof of Proposition
\ref{prop.triangular.strict.geoseries} is thus complete.
\end{proof}

\begin{remark}
\textbf{(a)} The above proof of Proposition
\ref{prop.triangular.strict.geoseries} might look like a slick and artful
trick. However, it is actually an incarnation of a well-known idea: the same
idea that enters in the proof of the infinite-sum formula%
\[
\dfrac{1}{1-a}=a^{0}+a^{1}+a^{2}+\cdots\ \ \ \ \ \ \ \ \ \ \text{for any real
number }a\text{ with }-1<a<1.
\]
The main difference here is that we are working with a matrix $A$ instead of a
real number $a$, and that the infinite sum $a^{0}+a^{1}+a^{2}+\cdots$ is
replaced by a \textbf{finite} sum $A^{0}+A^{1}+\cdots+A^{n-1}$ (because all
the higher powers $A^{n},A^{n+1},A^{n+2},\ldots$ equal the zero matrix).

\textbf{(b)} Proposition \ref{prop.triangular.strict.geoseries} requires an
$n\times n$-matrix $A$ satisfying $A^{n}=0_{n\times n}$. How do we find such matrices?

Corollary \ref{cor.triangular.strict.npow} shows that every strictly
lower-triangular $n\times n$-matrix $A$ has this property; this gives us an
infinite supply of such matrices (at least for $n\geq2$). Similarly, every
strictly upper-triangular $n\times n$-matrix $A$ has this property. But there
are also others: For example, if $A$ is the $2\times2$-matrix $\left(
\begin{array}
[c]{cc}%
1 & 1\\
-1 & -1
\end{array}
\right)  $, then $A$ also satisfies $A^{2}=0_{2\times2}$, despite not being triangular.
\end{remark}

We can now prove Theorem \ref{thm.triangular.inverse} again:

\begin{proof}
[Second proof of Theorem \ref{thm.triangular.inverse}.]Proposition
\ref{prop.triangular.uni-inv.lower} \textbf{(b)} shows that $A$ is
lower-unitriangular if and only if $I_{n}-A$ is strictly lower-triangular.
Hence, $I_{n}-A$ is strictly lower-triangular (because $A$ is lower-unitriangular).

Let $C=I_{n}-A$. Thus, $C$ is strictly lower-triangular (since $I_{n}-A$ is
strictly lower-triangular). Hence, Corollary \ref{cor.triangular.strict.npow}
(applied to $C$ instead of $A$) shows that $C^{n}=0_{n\times n}$. Proposition
\ref{prop.triangular.strict.geoseries} (applied to $C$ instead of $A$) thus
shows that the matrix $I_{n}-C$ is invertible, and that its inverse is
$\left(  I_{n}-C\right)  ^{-1}=C^{0}+C^{1}+\cdots+C^{n-1}$.

We have $I_{n}-\underbrace{C}_{=I_{n}-A}=I_{n}-\left(  I_{n}-A\right)  =A$.
Now, we know that the matrix $I_{n}-C$ is invertible. In other words, the
matrix $A$ is invertible (since $I_{n}-C=A$). It remains to check that its
inverse $A^{-1}$ is again lower-unitriangular.

We assume WLOG that $n$ is positive (since otherwise, there is nothing to
check). We have%
\begin{align*}
\left(  I_{n}-C\right)  ^{-1}  &  =C^{0}+C^{1}+\cdots+C^{n-1}%
=\underbrace{C^{0}}_{=I_{n}}+\left(  C^{1}+C^{2}+\cdots+C^{n-1}\right) \\
&  =I_{n}+\left(  C^{1}+C^{2}+\cdots+C^{n-1}\right)  .
\end{align*}
Since $I_{n}-C=A$, this rewrites as%
\[
A^{-1}=I_{n}+\left(  C^{1}+C^{2}+\cdots+C^{n-1}\right)  .
\]
Subtracting this equality from $I_{n}=I_{n}$, we obtain%
\begin{align}
I_{n}-A^{-1}  &  =I_{n}-\left(  I_{n}+\left(  C^{1}+C^{2}+\cdots
+C^{n-1}\right)  \right) \nonumber\\
&  =-\left(  C^{1}+C^{2}+\cdots+C^{n-1}\right)  .
\label{pf.thm.triangular.inverse.2nd.1}%
\end{align}


But recall that the matrix $C$ is strictly lower-triangular. Hence, for every
positive integer $k$, the $n\times n$-matrix $C^{k}$ is strictly
lower-triangular (by Corollary \ref{cor.triangular.strict.kpow-weak}, applied
to $C$ instead of $A$). Thus, the matrices $C^{1},C^{2},\ldots,C^{n-1}$ are
$n-1$ strictly lower-triangular $n\times n$-matrices. Proposition
\ref{prop.triangular.strict.sum-of-stricts} (applied to $k=n-1$ and
$A_{k}=C^{k}$) thus shows that $C^{1}+C^{2}+\cdots+C^{n-1}$ is a strictly
lower-triangular $n\times n$-matrix. Hence, Proposition
\ref{prop.triangular.strict.-strict} (applied to $C^{1}+C^{2}+\cdots+C^{n-1}$
instead of $A$) yields that $-\left(  C^{1}+C^{2}+\cdots+C^{n-1}\right)  $ is
a strictly lower-triangular $n\times n$-matrix. In light of
(\ref{pf.thm.triangular.inverse.2nd.1}), this rewrites as follows:
$I_{n}-A^{-1}$ is a strictly lower-triangular $n\times n$-matrix.

But Proposition \ref{prop.triangular.uni-inv.lower} \textbf{(b)} (applied to
$A^{-1}$ instead of $A$) shows that $A^{-1}$ is lower-unitriangular if and
only if $I_{n}-A^{-1}$ is strictly lower-triangular. Since we have just seen
that $I_{n}-A^{-1}$ is strictly lower-triangular, we can therefore conclude
that $A^{-1}$ is lower-unitriangular. Thus, the second proof of Theorem
\ref{thm.triangular.inverse} is complete.
\end{proof}

Of course, an analogous argument (with some inequality signs turned around,
and some \textquotedblleft lower\textquotedblright s replaced by
\textquotedblleft upper\textquotedblright s) can be used to prove Theorem
\ref{thm.triangular.inverse-up}.

\subsection{The $\lambda$-scaling matrices $S_{u}^{\lambda}$}

Now we shall explore another kind of square matrices not unlike the matrices
$A_{u,v}^{\lambda}$ from Definition \ref{def.Alamuv}\footnote{The present
section imitates the structure of Section \ref{sect.gauss.Alamuv}; this is, of
course, fully intentional.}:

\begin{definition}
\label{def.Slamu}Let $n\in\mathbb{N}$. Let $u\in\left\{  1,2,\ldots,n\right\}
$. Let $\lambda$ be a number. Then, $S_{u}^{\lambda}$ shall denote the
$n\times n$-matrix $I_{n}+\left(  \lambda-1\right)  E_{u,u}$ (where $E_{u,u}$
means the $n\times n$-matrix $E_{u,u}$, that is, $E_{u,u,n,n}$).

A few remarks about the notation:

\textbf{(a)} The superscript $\lambda$ in the notation \textquotedblleft%
$S_{u}^{\lambda}$\textquotedblright\ is not an exponent; i.e., the matrix
$S_{u}^{\lambda}$ is not the $\lambda$-th power of some matrix $S_{u}$.
Instead, it is just an argument that we have chosen to write as a superscript
instead of a subscript. So the role of the $\lambda$ in \textquotedblleft%
$S_{u}^{\lambda}$\textquotedblright\ is completely different from the role of
the $-1$ in \textquotedblleft$A_{1}^{-1}$\textquotedblright\ in Proposition
\ref{prop.inverses.A1Ak}.

\textbf{(b)} To be really precise, we ought to denote $S_{u}^{\lambda}$ by
$S_{u,n}^{\lambda}$, because it depends on $n$. (This is similar to how we
ought to denote $E_{u,v}$ by $E_{u,v,n,n}$.) But the $n$ will be really clear
from the context almost every time we deal with these matrices, so we shall
keep it out of our notation.

\textbf{(c)} The notation $S_{u}^{\lambda}$ is not standard in the literature,
but I will use this notation in the following.
\end{definition}

\begin{example}
\label{exe.Slamu}Let $n=4$. Then,%
\begin{align*}
S_{3}^{\lambda}  &  =I_{n}+\left(  \lambda-1\right)  E_{3,3}=\left(
\begin{array}
[c]{cccc}%
1 & 0 & 0 & 0\\
0 & 1 & 0 & 0\\
0 & 0 & \lambda & 0\\
0 & 0 & 0 & 1
\end{array}
\right)  \ \ \ \ \ \ \ \ \ \ \text{and}\\
S_{4}^{\lambda}  &  =I_{n}+\left(  \lambda-1\right)  E_{4,4}=\left(
\begin{array}
[c]{cccc}%
1 & 0 & 0 & 0\\
0 & 1 & 0 & 0\\
0 & 0 & 1 & 0\\
0 & 0 & 0 & \lambda
\end{array}
\right)  .
\end{align*}

\end{example}

The pattern that you see on these examples is true in general:

\begin{proposition}
\label{prop.Slamu.entries}Let $n\in\mathbb{N}$. Let $u\in\left\{
1,2,\ldots,n\right\}  $. Let $\lambda$ be a number. Then, the matrix
$S_{u}^{\lambda}$ has the following entries:

\begin{itemize}
\item Its $\left(  u,u\right)  $-th entry is $\lambda$.

\item All its other diagonal entries are $1$.

\item All its remaining entries are $0$.
\end{itemize}
\end{proposition}

Proposition \ref{prop.Slamu.entries} can be rewritten as follows: The matrix
$S_{u}^{\lambda}$ (where $n\in\mathbb{N}$, where $u\in\left\{  1,2,\ldots
,n\right\}  $, and where $\lambda$ is a number) is the $n\times n$-identity
matrix $I_{n}$ with the $\left(  u,u\right)  $-th entry replaced by $\lambda$.

\begin{proof}
[Proof of Proposition \ref{prop.Slamu.entries}.]Recall that $E_{u,u}$ is the
$n\times n$-matrix whose $\left(  u,u\right)  $-th entry is $1$ and whose all
other entries are $0$ (indeed, this is how $E_{u,u}$ was defined). Since
matrices are scaled entry by entry, we can therefore conclude how $\left(
\lambda-1\right)  E_{u,u}$ looks like: Namely, $\left(  \lambda-1\right)
E_{u,u}$ is the $n\times n$-matrix whose $\left(  u,u\right)  $-th entry is
$\left(  \lambda-1\right)  \cdot1=\lambda-1$ and whose all other entries are
$\left(  \lambda-1\right)  \cdot0=0$. Thus, we know the following:

\begin{itemize}
\item The matrix $I_{n}$ is the $n\times n$-matrix whose diagonal
entries\footnote{This includes the $\left(  u,u\right)  $-th entry.} are $1$,
and whose all other entries are $0$.

\item The matrix $\left(  \lambda-1\right)  E_{u,u}$ is the $n\times n$-matrix
whose $\left(  u,u\right)  $-th entry is $\lambda-1$, and whose all other
entries are $0$.
\end{itemize}

Since matrices are added entry by entry, we can thus infer how $I_{n}+\left(
\lambda-1\right)  E_{u,u}$ looks like: Namely, the matrix $I_{n}+\left(
\lambda-1\right)  E_{u,u}$ is the $n\times n$-matrix whose $\left(
u,u\right)  $-th entry is $1+\left(  \lambda-1\right)  =\lambda$, whose all
other diagonal entries are $1+0=1$, and whose all other entries are $0+0=0$.
Since $I_{n}+\left(  \lambda-1\right)  E_{u,u}=S_{u}^{\lambda}$, this rewrites
as follows: The matrix $S_{u}^{\lambda}$ is the $n\times n$-matrix whose
$\left(  u,u\right)  $-th entry is $\lambda$, whose all other diagonal entries
are $1$, and whose all other entries are $0$. This proves Proposition
\ref{prop.Slamu.entries}.
\end{proof}

We can next see what happens to a matrix when it is multiplied by
$S_{u}^{\lambda}$:

\begin{proposition}
\label{prop.Slamu.laction}Let $n\in\mathbb{N}$ and $m\in\mathbb{N}$. Let
$u\in\left\{  1,2,\ldots,n\right\}  $. Let $\lambda$ be a number. Let $C$ be
an $n\times m$-matrix. Then, $S_{u}^{\lambda}C$ is the $n\times m$-matrix
obtained from $C$ by scaling the $u$-th row by $\lambda$.

(Recall that the rows of $C$ are row vectors, and thus are scaled entry by
entry. Hence, scaling the $u$-th row by $\lambda$ means multiplying each entry
of the $u$-th row of $C$ by $\lambda$.)
\end{proposition}

\begin{example}
\label{exam.prop.Slamu.laction}Let $n=3$ and $m=2$. Let $C$ be the $3\times
2$-matrix $\left(
\begin{array}
[c]{cc}%
a & b\\
a^{\prime} & b^{\prime}\\
a^{\prime\prime} & b^{\prime\prime}%
\end{array}
\right)  $. Let $\lambda$ be a number. Then, Proposition
\ref{prop.Slamu.laction} (applied to $u=2$) claims that $S_{2}^{\lambda}C$ is
the $3\times2$-matrix obtained from $C$ by scaling the $2$-nd row by $\lambda
$. A computation confirms this claim:%
\[
S_{2}^{\lambda}C=\left(
\begin{array}
[c]{ccc}%
1 & 0 & 0\\
0 & \lambda & 0\\
0 & 0 & 1
\end{array}
\right)  \left(
\begin{array}
[c]{cc}%
a & b\\
a^{\prime} & b^{\prime}\\
a^{\prime\prime} & b^{\prime\prime}%
\end{array}
\right)  =\left(
\begin{array}
[c]{cc}%
a & b\\
\lambda a^{\prime} & \lambda b^{\prime}\\
a^{\prime\prime} & b^{\prime\prime}%
\end{array}
\right)  .
\]

\end{example}

\begin{proposition}
\label{prop.Slamu.raction}Let $n\in\mathbb{N}$ and $m\in\mathbb{N}$. Let
$u\in\left\{  1,2,\ldots,n\right\}  $. Let $\lambda$ be a number. Let $C$ be
an $m\times n$-matrix. Then, $CS_{u}^{\lambda}$ is the $m\times n$-matrix
obtained from $C$ by scaling the $u$-th column by $\lambda$.
\end{proposition}

I will prove Proposition \ref{prop.Slamu.laction} in Section
\ref{sect.gauss.Slamu.proofs} below (although by now, this proof could be a
simple exercise).

We shall refer to the matrix $S_{u}^{\lambda}$ defined in Definition
\ref{def.Slamu} as a \textquotedblleft$\lambda$-scaling
matrix\textquotedblright; it is the second of the previously announced three
kinds of elementary matrices.

Here are a few more properties of $\lambda$-scaling matrices:

\begin{proposition}
\label{prop.Slamu.transpose}Let $n\in\mathbb{N}$. Let $u\in\left\{
1,2,\ldots,n\right\}  $. Let $\lambda$ be a number. Then, $\left(
S_{u}^{\lambda}\right)  ^{T}=S_{u}^{\lambda}$.
\end{proposition}

\begin{proposition}
\label{prop.Slamu.lambda+mu}Let $n\in\mathbb{N}$. Let $u\in\left\{
1,2,\ldots,n\right\}  $.

\textbf{(a)} We have $S_{u}^{1}=I_{n}$.

\textbf{(b)} If $\lambda$ and $\mu$ are two numbers, then $S_{u}^{\lambda
}S_{u}^{\mu}=S_{u}^{\lambda\mu}$.

\textbf{(c)} Let $\lambda$ be a nonzero number. Then, the matrix
$S_{u}^{\lambda}$ is invertible, and its inverse is $\left(  S_{u}^{\lambda
}\right)  ^{-1}=S_{u}^{1/\lambda}$.
\end{proposition}

A proof of this proposition will also be given in Section
\ref{sect.gauss.Slamu.proofs}.

\subsection{\label{sect.gauss.Slamu.proofs}(*) Some proofs about the $\lambda
$-scaling matrices}

\begin{proof}
[Proof of Proposition \ref{prop.Slamu.laction}.]Clearly, $S_{u}^{\lambda}C$ is
an $n\times m$-matrix.

Proposition \ref{prop.Euv.laction} (applied to $n$, $m$ and $u$ instead of
$m$, $p$ and $v$) shows that $E_{u,u}C$ is the $n\times m$-matrix whose $u$-th
row is the $u$-th row of $C$, and whose all other rows are filled with zeroes.
Thus,%
\begin{equation}
\operatorname*{row}\nolimits_{u}\left(  E_{u,u}C\right)  =\operatorname*{row}%
\nolimits_{u}C \label{pf.prop.Slamu.laction.1}%
\end{equation}
(since the $u$-the row of $E_{u,u}C$ is the $u$-th row of $C$) and%
\begin{equation}
\operatorname*{row}\nolimits_{i}\left(  E_{u,u}C\right)  =0_{1\times
m}\ \ \ \ \ \ \ \ \ \ \text{for every }i\in\left\{  1,2,\ldots,n\right\}
\text{ satisfying }i\neq u \label{pf.prop.Slamu.laction.2}%
\end{equation}
(since all other rows of $E_{u,u}C$ are filled with zeroes).

But recall that matrices are added entry by entry. Thus, matrices are also
added by row by row -- i.e., if $U$ and $V$ are two $n\times m$-matrices, then
any row of $U+V$ is the sum of the corresponding rows of $U$ and of $V$. In
other words, if $U$ and $V$ are two $n\times m$-matrices, then%
\begin{equation}
\operatorname*{row}\nolimits_{i}\left(  U+V\right)  =\operatorname*{row}%
\nolimits_{i}U+\operatorname*{row}\nolimits_{i}V\ \ \ \ \ \ \ \ \ \ \text{for
every }i\in\left\{  1,2,\ldots,n\right\}  . \label{pf.prop.Slamu.laction.5}%
\end{equation}
Also, if $U$ is an $n\times m$-matrix, then every number $\mu$ satisfies%
\begin{equation}
\operatorname*{row}\nolimits_{i}\left(  \mu U\right)  =\mu\operatorname*{row}%
\nolimits_{i}U\ \ \ \ \ \ \ \ \ \ \text{for every }i\in\left\{  1,2,\ldots
,n\right\}  \label{pf.prop.Slamu.laction.6}%
\end{equation}
(since matrices are scaled entry by entry).

We have%
\begin{align*}
\underbrace{S_{u}^{\lambda}}_{=I_{n}+\left(  \lambda-1\right)  E_{u,u}}C  &
=\left(  I_{n}+\left(  \lambda-1\right)  E_{u,u}\right)  C=\underbrace{I_{n}%
C}_{=C}+\left(  \lambda-1\right)  E_{u,u}C\\
&  =C+\left(  \lambda-1\right)  E_{u,u}C.
\end{align*}
Hence, for each $i\in\left\{  1,2,\ldots,n\right\}  $, we have%
\begin{align}
\operatorname*{row}\nolimits_{i}\left(  \underbrace{S_{u}^{\lambda}%
C}_{=C+\left(  \lambda-1\right)  E_{u,u}C}\right)   &  =\operatorname*{row}%
\nolimits_{i}\left(  C+\left(  \lambda-1\right)  E_{u,u}C\right)
=\operatorname*{row}\nolimits_{i}C+\underbrace{\operatorname*{row}%
\nolimits_{i}\left(  \left(  \lambda-1\right)  E_{u,u}C\right)  }%
_{\substack{=\left(  \lambda-1\right)  \operatorname*{row}\nolimits_{i}\left(
E_{u,u}C\right)  \\\text{(by (\ref{pf.prop.Slamu.laction.6}), applied}%
\\\text{to }\mu=\lambda-1\text{)}}}\nonumber\\
&  \ \ \ \ \ \ \ \ \ \ \left(  \text{by (\ref{pf.prop.Slamu.laction.5}),
applied to }U=C\text{ and }V=\left(  \lambda-1\right)  E_{u,u}C\right)
\nonumber\\
&  =\operatorname*{row}\nolimits_{i}C+\left(  \lambda-1\right)
\operatorname*{row}\nolimits_{i}\left(  E_{u,u}C\right)  .
\label{pf.prop.Slamu.laction.9}%
\end{align}


Now, we must prove that $S_{u}^{\lambda}C$ is the $n\times m$-matrix obtained
from $C$ by scaling the $u$-th row by $\lambda$. In other words, we must prove
the following two claims:

\begin{statement}
\textit{Claim 1:} The $u$-th row of the $n\times m$-matrix $S_{u}^{\lambda}C$
equals $\lambda$ times the $u$-th row of $C$.
\end{statement}

\begin{statement}
\textit{Claim 2:} For each $i\in\left\{  1,2,\ldots,n\right\}  $ satisfying
$i\neq u$, the $i$-th row of the $n\times m$-matrix $S_{u}^{\lambda}C$ equals
the $i$-th row of $C$.
\end{statement}

\textit{Proof of Claim 1:} The $u$-th row of the $n\times m$-matrix
$S_{u}^{\lambda}C$ is%
\begin{align*}
\operatorname*{row}\nolimits_{u}\left(  S_{u}^{\lambda}C\right)   &
=\operatorname*{row}\nolimits_{u}C+\left(  \lambda-1\right)
\underbrace{\operatorname*{row}\nolimits_{u}\left(  E_{u,u}C\right)
}_{\substack{=\operatorname*{row}\nolimits_{u}C\\\text{(by
(\ref{pf.prop.Slamu.laction.1}))}}}\ \ \ \ \ \ \ \ \ \ \left(  \text{by
(\ref{pf.prop.Slamu.laction.9}), applied to }i=u\right) \\
&  =\operatorname*{row}\nolimits_{u}C+\left(  \lambda-1\right)
\operatorname*{row}\nolimits_{u}C=\underbrace{\left(  1+\left(  \lambda
-1\right)  \right)  }_{=\lambda}\operatorname*{row}\nolimits_{u}%
C=\lambda\operatorname*{row}\nolimits_{u}C.
\end{align*}
In other words, the $u$-th row of the $n\times m$-matrix $S_{u}^{\lambda}C$
equals $\lambda$ times the $u$-th row of $C$. This proves Claim 1.

\textit{Proof of Claim 2:} Let $i\in\left\{  1,2,\ldots,n\right\}  $ be such
that $i\neq u$. Then, the $i$-th row of the $n\times m$-matrix $S_{u}%
^{\lambda}C$ is%
\begin{align*}
\operatorname*{row}\nolimits_{i}\left(  S_{u}^{\lambda}C\right)   &
=\operatorname*{row}\nolimits_{i}C+\left(  \lambda-1\right)
\underbrace{\operatorname*{row}\nolimits_{i}\left(  E_{u,u}C\right)
}_{\substack{=0_{1\times m}\\\text{(by (\ref{pf.prop.Slamu.laction.2}))}%
}}\ \ \ \ \ \ \ \ \ \ \left(  \text{by (\ref{pf.prop.Slamu.laction.9})}\right)
\\
&  =\operatorname*{row}\nolimits_{i}C+\underbrace{\left(  \lambda-1\right)
0_{1\times m}}_{=0_{1\times m}}=\operatorname*{row}\nolimits_{i}C+0_{1\times
m}=\operatorname*{row}\nolimits_{i}C.
\end{align*}
In other words, the $i$-th row of the $n\times m$-matrix $S_{u}^{\lambda}C$
equals the $i$-th row of $C$. This proves Claim 2.

Now, we have proven both Claim 1 and Claim 2; this completes the proof of
Proposition \ref{prop.Slamu.laction}.
\end{proof}

The proof of Proposition \ref{prop.Slamu.raction} is analogous.

\begin{proof}
[Proof of Proposition \ref{prop.Slamu.transpose}.]Very easy and left to the
reader. (See the proof of Proposition \ref{prop.Alamuv.transpose} for inspiration.)
\end{proof}

\begin{proof}
[Proof of Proposition \ref{prop.Slamu.lambda+mu}.]\textbf{(a)} The definition
of $S_{u}^{1}$ yields $S_{u}^{1}=I_{n}+\underbrace{\left(  1-1\right)
E_{u,u}}_{=0E_{u,u}=0_{n\times n}}=I_{n}+0_{n\times n}=I_{n}$. This proves
Proposition \ref{prop.Slamu.lambda+mu} \textbf{(a)}.

\textbf{(b)} \textit{First proof:} Let $\lambda$ and $\mu$ be two numbers.
Proposition \ref{prop.Slamu.entries} (applied to $\mu$ instead of $\lambda$)
tells us how the matrix $S_{u}^{\mu}$ looks like: Its $\left(  u,u\right)
$-th entry is $\mu$; all its other diagonal entries are $1$; all its remaining
entries are $0$. In particular, its $u$-th row is%
\begin{equation}
\operatorname*{row}\nolimits_{u}\left(  S_{u}^{\mu}\right)  =\left(
0,0,\ldots,0,\mu,0,0,\ldots,0\right)  \label{pf.prop.Slamu.lambda+mu.b.1}%
\end{equation}
(where the lonely $\mu$ stands in the $u$-th position).

Proposition \ref{prop.Slamu.laction} (applied to $m=n$ and $C=S_{u}^{\mu}$)
shows that $S_{u}^{\lambda}S_{u}^{\mu}$ is the $n\times n$-matrix obtained
from $S_{u}^{\mu}$ by scaling the $u$-th row by $\lambda$. Thus, its $u$-th
row is%
\begin{align*}
\operatorname*{row}\nolimits_{u}\left(  S_{u}^{\lambda}S_{u}^{\mu}\right)   &
=\lambda\underbrace{\operatorname*{row}\nolimits_{u}\left(  S_{u}^{\mu
}\right)  }_{\substack{=\left(  0,0,\ldots,0,\mu,0,0,\ldots,0\right)
\\\text{(by (\ref{pf.prop.Slamu.lambda+mu.b.1}))}}}=\lambda\left(
0,0,\ldots,0,\mu,0,0,\ldots,0\right) \\
&  =\left(  \lambda0,\lambda0,\ldots,\lambda0,\lambda\mu,\lambda
0,\lambda0,\ldots,\lambda0\right) \\
&  =\left(  0,0,\ldots,0,\lambda\mu,0,0,\ldots,0\right)
\end{align*}
(where the lonely $\mu$ or $\lambda\mu$ stands in the $u$-th position, as
before). This is the same as the $u$-th row of $S_{u}^{\mu}$, except that the
$u$-th entry has become $\lambda\mu$ (whereas in the $u$-th row of $S_{u}%
^{\mu}$ it used to be $\mu$). All other rows of $S_{u}^{\lambda}S_{u}^{\mu}$
are equal to the corresponding rows of $S_{u}^{\mu}$ (since $S_{u}^{\lambda
}S_{u}^{\mu}$ was obtained from $S_{u}^{\mu}$ by scaling the $u$-th row by
$\lambda$). Summarizing, we thus conclude that the matrix $S_{u}^{\lambda
}S_{u}^{\mu}$ differs from the matrix $S_{u}^{\mu}$ in only one entry, namely
the $\left(  u,u\right)  $-th entry\footnote{If it differs from it at all! If
$\lambda=1$ or $\mu=0$, then the matrices $S_{u}^{\lambda}S_{u}^{\mu}$ and
$S_{u}^{\mu}$ are completely equal (including in their $\left(  u,u\right)
$-th entries).}; and this $\left(  u,u\right)  $-th entry is $\lambda\mu$ (for
the matrix $S_{u}^{\lambda}S_{u}^{\mu}$). Since we already know how the matrix
$S_{u}^{\mu}$ looks like (namely, its $\left(  u,u\right)  $-th entry is $\mu
$; all its other diagonal entries are $1$; all its remaining entries are $0$),
we thus can conclude how the matrix $S_{u}^{\lambda}S_{u}^{\mu}$ looks like:
Its $\left(  u,u\right)  $-th entry is $\lambda\mu$; all its other diagonal
entries are $1$; all its remaining entries are $0$. But this is precisely how
the matrix $S_{u}^{\lambda\mu}$ looks like (because of Proposition
\ref{prop.Slamu.entries}, applied to $\lambda\mu$ instead of $\lambda$).
Hence, $S_{u}^{\lambda}S_{u}^{\mu}=S_{u}^{\lambda\mu}$. This proves
Proposition \ref{prop.Slamu.lambda+mu} \textbf{(b)}.

\textit{Second proof:} We can also prove Proposition
\ref{prop.Slamu.lambda+mu} \textbf{(b)} easily using Proposition
\ref{prop.Euv.prod}: Indeed, we have $u=u$ and thus $\delta_{u,u}=1$. But
Proposition \ref{prop.Euv.prod} (applied to $m=n$, $p=n$, $x=u$ and $y=u$)
yields $E_{u,u,n,n}E_{u,u,n,n}=\delta_{u,u}E_{u,u,n,n}$. Since $E_{u,u,n,n}%
=E_{u,u}$, this rewrites as $E_{u,u}E_{u,u}=\underbrace{\delta_{u,u}}%
_{=1}E_{u,u}=1E_{u,u}=E_{u,u}$. Now, the definitions of $S_{u}^{\lambda}$ and
$S_{u}^{\mu}$ yield $S_{u}^{\lambda}=I_{n}+\left(  \lambda-1\right)  E_{u,u}$
and $S_{u}^{\mu}=I_{n}+\left(  \mu-1\right)  E_{u,u}$. Multiplying these two
equalities, we find%
\begin{align*}
S_{u}^{\lambda}S_{u}^{\mu}  &  =\left(  I_{n}+\left(  \lambda-1\right)
E_{u,u}\right)  \left(  I_{n}+\left(  \mu-1\right)  E_{u,u}\right) \\
&  =\underbrace{I_{n}\left(  I_{n}+\left(  \mu-1\right)  E_{u,u}\right)
}_{=I_{n}+\left(  \mu-1\right)  E_{u,u}}+\underbrace{\left(  \lambda-1\right)
E_{u,u}\left(  I_{n}+\left(  \mu-1\right)  E_{u,u}\right)  }_{=\left(
\lambda-1\right)  E_{u,u}I_{n}+\left(  \lambda-1\right)  E_{u,u}\left(
\mu-1\right)  E_{u,u}}\\
&  =I_{n}+\left(  \mu-1\right)  E_{u,u}+\left(  \lambda-1\right)
\underbrace{E_{u,u}I_{n}}_{=E_{u,u}}+\underbrace{\left(  \lambda-1\right)
E_{u,u}\left(  \mu-1\right)  E_{u,u}}_{=\left(  \lambda-1\right)  \left(
\mu-1\right)  E_{u,u}E_{u,u}}\\
&  =I_{n}+\underbrace{\left(  \mu-1\right)  E_{u,u}+\left(  \lambda-1\right)
E_{u,u}}_{=\left(  \left(  \mu-1\right)  +\left(  \lambda-1\right)  \right)
E_{u,u}}+\left(  \lambda-1\right)  \left(  \mu-1\right)  \underbrace{E_{u,u}%
E_{u,u}}_{=E_{u,u}}\\
&  =I_{n}+\underbrace{\left(  \left(  \mu-1\right)  +\left(  \lambda-1\right)
\right)  E_{u,u}+\left(  \lambda-1\right)  \left(  \mu-1\right)  E_{u,u}%
}_{=\left(  \left(  \mu-1\right)  +\left(  \lambda-1\right)  +\left(
\lambda-1\right)  \left(  \mu-1\right)  \right)  E_{u,u}}\\
&  =I_{n}+\underbrace{\left(  \left(  \mu-1\right)  +\left(  \lambda-1\right)
+\left(  \lambda-1\right)  \left(  \mu-1\right)  \right)  }_{=\lambda\mu
-1}E_{u,u}=I_{n}+\left(  \lambda\mu-1\right)  E_{u,u}.
\end{align*}
Comparing this with
\[
S_{u}^{\lambda\mu}=I_{n}+\left(  \lambda\mu-1\right)  E_{u,u}%
\ \ \ \ \ \ \ \ \ \ \left(  \text{by the definition of }S_{u}^{\lambda\mu
}\right)  ,
\]
we obtain $S_{u}^{\lambda}S_{u}^{\mu}=S_{u}^{\lambda\mu}$. This proves
Proposition \ref{prop.Slamu.lambda+mu} \textbf{(b)} again.

\textbf{(c)} Notice that $1/\lambda$ is a well-defined number (since $\lambda$
is nonzero). Proposition \ref{prop.Slamu.lambda+mu} \textbf{(b)} (applied to
$\mu=1/\lambda$) yields $S_{u}^{\lambda}S_{u}^{1/\lambda}=S_{u}^{\lambda
\left(  1/\lambda\right)  }=S_{u}^{1}=I_{n}$ (by Proposition
\ref{prop.Slamu.lambda+mu} \textbf{(a)}).

But we can also apply Proposition \ref{prop.Slamu.lambda+mu} \textbf{(b)} to
$1/\lambda$ and $\lambda$ instead of $\lambda$ and $\mu$. We thus obtain
$S_{u}^{1/\lambda}S_{u}^{\lambda}=S_{u}^{\left(  1/\lambda\right)  \lambda
}=S_{u}^{1}=I_{n}$ (by Proposition \ref{prop.Slamu.lambda+mu} \textbf{(a)}).

The two equalities $S_{u}^{\lambda}S_{u}^{1/\lambda}=I_{n}$ and $S_{u}%
^{1/\lambda}S_{u}^{\lambda}=I_{n}$ show that $S_{u}^{1/\lambda}$ is an inverse
of $S_{u}^{\lambda}$. This proves Proposition \ref{prop.Slamu.lambda+mu}
\textbf{(c)}.
\end{proof}

\subsection{Invertibly triangular matrices are products of $S_{u}^{\lambda}$'s
and $A_{u,v}^{\lambda}$'s}

In the same way as the matrices $A_{u,v}^{\lambda}$ served as
\textquotedblleft building blocks\textquotedblright\ for unitriangular
matrices (in Theorem \ref{thm.triangular.Alamuv}), we can obtain
\textquotedblleft building blocks\textquotedblright\ for invertibly triangular
matrices if we accompany these matrices $A_{u,v}^{\lambda}$ by the matrices
$S_{u}^{\lambda}$ with nonzero $\lambda$. We shall state this soon in some
precision; let us first introduce terminology:

\begin{definition}
\label{def.Slamu.scaling}Let $n\in\mathbb{N}$. A \textit{scaling }$n\times
n$\textit{-matrix} means a matrix of the form $S_{u}^{\lambda}$, where
$\lambda$ is a \textbf{nonzero} number, and where $u$ is an element of
$\left\{  1,2,\ldots,n\right\}  $. When $n$ is clear from the context, we
shall omit the \textquotedblleft$n\times n$-\textquotedblright\ and simply say
\textquotedblleft scaling matrix\textquotedblright.
\end{definition}

The name \textquotedblleft scaling matrix\textquotedblright\ is, again, not
standard, but it will be useful for me in this chapter.

\begin{example}
If $n=3$, then the scaling $3\times3$-matrices are the matrices of the form%
\begin{align*}
S_{1}^{\lambda}  &  =\left(
\begin{array}
[c]{ccc}%
\lambda & 0 & 0\\
0 & 1 & 0\\
0 & 0 & 1
\end{array}
\right)  ,\ \ \ \ \ \ \ \ \ \ S_{2}^{\lambda}=\left(
\begin{array}
[c]{ccc}%
1 & 0 & 0\\
0 & \lambda & 0\\
0 & 0 & 1
\end{array}
\right)  ,\\
S_{3}^{\lambda}  &  =\left(
\begin{array}
[c]{ccc}%
1 & 0 & 0\\
0 & 1 & 0\\
0 & 0 & \lambda
\end{array}
\right)
\end{align*}
for all nonzero numbers $\lambda$.
\end{example}

It is clear that each scaling matrix is diagonal (and therefore
lower-triangular and upper-triangular). Thus, every product of scaling
matrices is a product of diagonal matrices, and thus itself must be
diagonal\footnote{Here we are using the fact that any product of diagonal
matrices is diagonal. This is not hard to check.}. Moreover, both scaling
matrices and lower addition matrices\footnote{Those were defined in Definition
\ref{def.Alamuv.laddition}.} are invertibly lower-triangular\footnote{Here, we
are using the requirement that $\lambda$ is nonzero in Definition
\ref{def.Slamu.scaling}. Without this requirement, $S_{u}^{\lambda}$ would not
be invertibly lower-triangular!}. Hence, every product of scaling matrices and
lower addition matrices is a product of invertibly lower-triangular matrices,
and thus itself must be invertibly lower-triangular\footnote{because Corollary
\ref{cor.triangular.inv*inv*inv.lower} shows that any product of invertibly
lower-triangular matrices is invertibly lower-triangular}. It turns out that
the converse is also true: Every invertibly lower-triangular matrix is a
product of scaling matrices and lower addition matrices! This is again a
simple particular case of Gaussian elimination, similar to Theorem
\ref{thm.triangular.Alamuv}; let me state it as a theorem:

\begin{theorem}
\label{thm.triangular.Slamu}Let $n\in\mathbb{N}$. An $n\times n$-matrix $C$ is
invertibly lower-triangular if and only if $C$ is a product of scaling
matrices and lower addition matrices.
\end{theorem}

\begin{example}
\label{exam.thm.triangular.Slamu}\textbf{(a)} The invertibly lower-triangular
$2\times2$-matrix $\left(
\begin{array}
[c]{cc}%
2 & 0\\
5 & 3
\end{array}
\right)  $ is a product of scaling matrices and lower addition matrices:
Namely, it equals $S_{1}^{2}S_{2}^{3}A_{2,1}^{5/3}$.

\textbf{(b)} The invertibly lower-triangular $1\times1$-matrix $\left(
\begin{array}
[c]{c}%
5
\end{array}
\right)  $ is a product of scaling matrices and lower addition matrices:
Namely, it is $S_{1}^{5}$.

\textbf{(c)} Let $C$ be the invertibly lower-triangular $3\times3$-matrix
$\left(
\begin{array}
[c]{ccc}%
u & 0 & 0\\
a & v & 0\\
b & c & w
\end{array}
\right)  $. Then, $C$ is a product of scaling matrices and lower addition
matrices: Namely, it equals $S_{1}^{u}S_{2}^{v}S_{3}^{w}A_{2,1}^{a/v}%
A_{3,1}^{b/w}A_{3,2}^{c/w}$.

Let us actually see how this representation of $C$ can be found. We shall
proceed by writing $C$ as a product of one scaling matrix with a second matrix
$C^{\prime}$, which is still invertibly lower-triangular but has one diagonal
entry equal to $1$. We then will do the same with $C^{\prime}$, obtaining a
third matrix $C^{\prime\prime}$; then, do the same with $C^{\prime\prime}$,
and so on. At the end, we will be left with an invertibly lower-triangular
matrix whose \textbf{all} diagonal entries are $1$. This means that we will be
left with a lower-unitriangular matrix. But from Theorem
\ref{thm.triangular.Alamuv}, we already know that this latter matrix must be a
product of lower addition matrices. In more detail: We first observe that the
diagonal entries $u,v,w$ of $C$ are nonzero (since $C$ is invertibly
lower-triangular). Now, we proceed in several steps:

\begin{description}
\item[Step 1:] Let us turn the $\left(  1,1\right)  $-st entry of $C$ into $1$
by scaling row $1$ by $1/u$. Denote the resulting matrix by $C^{\prime}$.
Thus, $C^{\prime}=\left(
\begin{array}
[c]{ccc}%
1 & 0 & 0\\
a & v & 0\\
b & c & w
\end{array}
\right)  $. Notice that the new matrix $C^{\prime}$ is still invertibly
lower-triangular (since only row $1$ has been changed, and since the scaling
has left all the zero entries in place), but now has its first diagonal entry
equal to $1$. Since $C^{\prime}$ was obtained from $C$ by scaling row $1$ by
$1/u$, we can conversely obtain $C$ from $C^{\prime}$ by scaling row $1$ by
$u$. According to Proposition \ref{prop.Slamu.laction} (applied to $n$, $1$,
$u$ and $C^{\prime}$ instead of $m$, $u$, $\lambda$ and $C$), this means that
$C=S_{1}^{u}C^{\prime}$.

\item[Step 2:] Let us turn the $\left(  2,2\right)  $-st entry of $C^{\prime}$
into $1$ by scaling row $2$ by $1/v$. Denote the resulting matrix by
$C^{\prime\prime}$. Thus, $C^{\prime\prime}=\left(
\begin{array}
[c]{ccc}%
1 & 0 & 0\\
a/v & 1 & 0\\
b & c & w
\end{array}
\right)  $. Again, the new matrix $C^{\prime\prime}$ is still invertibly
lower-triangular (since only row $2$ has been changed, and all zeroes have
survived the scaling), and moreover the first diagonal entry is still $1$
(because only row $2$ has been changed); but now the second diagonal entry is
also $1$. Similarly to how we found that $C=S_{1}^{u}C^{\prime}$ in Step 1, we
now obtain $C^{\prime}=S_{2}^{v}C^{\prime\prime}$.

\item[Step 3:] Let us turn the $\left(  3,3\right)  $-st entry of
$C^{\prime\prime}$ into $1$ by scaling row $3$ by $1/w$. Denote the resulting
matrix by $C^{\prime\prime\prime}$. Thus, $C^{\prime\prime\prime}=\left(
\begin{array}
[c]{ccc}%
1 & 0 & 0\\
a/v & 1 & 0\\
b/w & c/w & 1
\end{array}
\right)  $. Again, the new matrix $C^{\prime\prime\prime}$ is still invertibly
lower-triangular, and the first two diagonal entries are still equal to $1$;
and now the third diagonal entry has become $1$ as well. Similarly to how we
found that $C=S_{1}^{u}C^{\prime}$ in Step 1, we now obtain $C^{\prime\prime
}=S_{3}^{w}C^{\prime\prime\prime}$.
\end{description}

We have thus turned all diagonal entries into $1$. Our final matrix
$C^{\prime\prime\prime}$ is thus lower-unitriangular. Thus, Theorem
\ref{thm.triangular.Alamuv} shows that $C^{\prime\prime\prime}$ is a product
of lower addition matrices. Explicitly, it can be written as follows:%
\[
C^{\prime\prime\prime}=A_{2,1}^{a/v}A_{3,1}^{b/w}A_{3,2}^{c/w}.
\]
(Indeed, this follows from (\ref{eq.exam.thm.triangular.Alamuv.c.3}), applied
to $a/v$, $b/w$ and $c/w$ instead of $a$, $b$ and $c$.) Combining the three
equalities we have found, we obtain%
\begin{align*}
C  &  =S_{1}^{u}\underbrace{C^{\prime}}_{=S_{2}^{v}C^{\prime\prime}}=S_{1}%
^{u}S_{2}^{v}\underbrace{C^{\prime\prime}}_{=S_{3}^{w}C^{\prime\prime\prime}%
}=S_{1}^{u}S_{2}^{v}S_{3}^{w}\underbrace{C^{\prime\prime\prime}}%
_{=A_{2,1}^{a/v}A_{3,1}^{b/w}A_{3,2}^{c/w}}\\
&  =S_{1}^{u}S_{2}^{v}S_{3}^{w}A_{2,1}^{a/v}A_{3,1}^{b/w}A_{3,2}^{c/w}.
\end{align*}
Thus we have represented $C$ as a product of scaling matrices and lower
addition matrices.
\end{example}

The general proof of Theorem \ref{thm.triangular.Slamu} follows the idea
outlined in Example \ref{exam.thm.triangular.Slamu} \textbf{(c)}. The proof is
so similar to the proof of Theorem \ref{thm.triangular.Alamuv} that I shall be
copying the structure of the latter proof:

\begin{proof}
[Proof of Theorem \ref{thm.triangular.Slamu}.]$\Longleftarrow$: We have
already proven that every product of scaling matrices and lower addition
matrices is invertibly lower-triangular. Hence, if $C$ is a product of scaling
matrices and lower addition matrices, then $C$ is invertibly lower-triangular.
This proves the $\Longleftarrow$ direction of Theorem
\ref{thm.triangular.Slamu}.

$\Longrightarrow$: We need to prove that if $C$ is invertibly
lower-triangular, then $C$ is a product of scaling matrices and lower addition matrices.

So let us assume that $C$ is invertibly lower-triangular. Our goal is to prove
that $C$ is a product of scaling matrices and lower addition matrices.

Let me introduce a notation first: A \textit{row scaling} shall mean a
transformation that changes an $n\times m$-matrix (for some $m\in\mathbb{N}$)
by scaling one of its rows by some nonzero number. In more formal terms: A
\textit{row scaling} means a transformation of the form \textquotedblleft
scale the $u$-th row by $\lambda$\textquotedblright, for some fixed nonzero
number $\lambda$ and some fixed $u\in\left\{  1,2,\ldots,n\right\}  $. As we
know from Proposition \ref{prop.Slamu.laction}, this transformation amounts to
multiplying a matrix by $S_{u}^{\lambda}$ from the left (i.e., this
transformation sends any $n\times m$-matrix $B$ to $S_{u}^{\lambda}B$); we
shall therefore denote this transformation itself by $S_{u}^{\lambda}$ as well
(hoping that the reader will not confuse the transformation with the matrix).

Here is an example (for $n=4$): The row scaling $S_{3}^{\lambda}$ is the
transformation that changes a $4\times m$-matrix by scaling the $3$-rd row by
$\lambda$. For example, it transforms the $4\times2$-matrix $\left(
\begin{array}
[c]{cc}%
a & b\\
a^{\prime} & b^{\prime}\\
a^{\prime\prime} & b^{\prime\prime}\\
a^{\prime\prime\prime} & b^{\prime\prime\prime}%
\end{array}
\right)  $ into $\left(
\begin{array}
[c]{cc}%
a & b\\
a^{\prime} & b^{\prime}\\
\lambda a^{\prime\prime} & \lambda b^{\prime\prime}\\
a^{\prime\prime\prime} & b^{\prime\prime\prime}%
\end{array}
\right)  $.

Notice that any row scaling $S_{u}^{\lambda}$ is invertible: Namely, it can be
undone by the row scaling $S_{u}^{1/\lambda}$.\ \ \ \ \footnote{Here are two
ways to prove this:
\par
\textit{First proof:} The row scaling $S_{u}^{\lambda}$ transforms a matrix by
scaling the $u$-th row by $\lambda$, i.e., by multiplying each entry of the
$u$-th row by $\lambda$. The row scaling $S_{u}^{1/\lambda}$ transforms a
matrix by scaling the $u$-th row by $1/\lambda$, i.e., by dividing each entry
of the $u$-th row by $\lambda$. Hence, these two row scalings undo each other
(i.e., if we perform one and then the other, then we arrive back at the matrix
we have started with), because each entry of the $u$-th row is multiplied by
$\lambda$ by the former row scaling and divided by $\lambda$ by the latter
(whereas entries in other rows are left unchanged by both scalings). So we
have shown that the row scaling $S_{u}^{\lambda}$ can be undone by the row
scaling $S_{u}^{1/\lambda}$. Qed.
\par
\textit{Second proof:} Proposition \ref{prop.Slamu.lambda+mu} \textbf{(c)}
shows that the matrix $S_{u}^{1/\lambda}$ is the inverse of the matrix
$S_{u}^{\lambda}$. Hence, multiplying a matrix by $S_{u}^{1/\lambda}$ undoes
multiplying a matrix by $S_{u}^{\lambda}$. In other words, the row scaling
$S_{u}^{1/\lambda}$ undoes the row scaling $S_{u}^{\lambda}$. Qed.}

Notice that, for any row scaling $S_{u}^{\lambda}$, the matrix $S_{u}%
^{\lambda}$ is a row scaling matrix.

I claim that we can transform the invertibly lower-triangular $n\times
n$-matrix $C$ into a lower-unitriangular matrix by performing a sequence of
row scalings. Namely, we should proceed by the following method:\footnote{See
the three-step procedure in Example \ref{exam.thm.triangular.Slamu}
\textbf{(c)} for an illustration of this method.}

\begin{itemize}
\item At first, our matrix is
\[
C=\left(
\begin{array}
[c]{ccccc}%
C_{1,1} & 0 & 0 & \cdots & 0\\
C_{2,1} & C_{2,2} & 0 & \cdots & 0\\
C_{3,1} & C_{3,2} & C_{3,3} & \cdots & 0\\
\vdots & \vdots & \vdots & \ddots & \vdots\\
C_{n,1} & C_{n,2} & C_{n,3} & \cdots & C_{n,n}%
\end{array}
\right)  .
\]
Its diagonal entries $C_{1,1},C_{2,2},\ldots,C_{n,n}$ are nonzero (since $C$
is invertibly lower-triangular).

\item Now, we perform the row scaling $S_{1}^{\lambda}$ (for an appropriate
choice of $\lambda$, namely for $\lambda=1/C_{1,1}$) to turn the $\left(
1,1\right)  $-th entry of the matrix into $1$. This operation preserves the
invertibly lower-triangular nature of the matrix (i.e., the matrix remains
invertibly lower-triangular\footnote{In fact, this holds for any row scaling:
If $B$ is any invertibly lower-triangular matrix and $S_{u}^{\mu}$ is a row
scaling, then the result of applying $S_{u}^{\mu}$ to $B$ will still be
invertibly lower-triangular. To prove this, just observe that all zero entries
of $B$ remain zero when the row scaling $S_{u}^{\mu}$ is applied (since
$S_{u}^{\mu}$ merely scales a row), whereas all nonzero diagonal entries of
$B$ remain nonzero (since $S_{u}^{\mu}$ scales a row by the \textbf{nonzero}
number $\mu$).}). As the result, we have turned the $\left(  1,1\right)  $-th
entry of the matrix into $1$. In other words, our matrix now looks as
follows:
\[
\left(
\begin{array}
[c]{ccccc}%
1 & 0 & 0 & \cdots & 0\\
C_{2,1} & C_{2,2} & 0 & \cdots & 0\\
C_{3,1} & C_{3,2} & C_{3,3} & \cdots & 0\\
\vdots & \vdots & \vdots & \ddots & \vdots\\
C_{n,1} & C_{n,2} & C_{n,3} & \cdots & C_{n,n}%
\end{array}
\right)  .
\]


\item Next, we similarly perform the row scaling $S_{2}^{\lambda}$ (for an
appropriate choice of $\lambda$, namely for $\lambda=1/C_{2,2}$) to turn the
$\left(  2,2\right)  $-th entry of the matrix into $1$. Again, this operation
preserves the invertibly lower-triangular nature of the matrix (i.e., the
matrix remains invertibly lower-triangular)\footnote{\textbf{Warning:} Unlike
in our proof of Theorem \ref{thm.triangular.Alamuv}, this operation will
(usually) change not only the $\left(  2,2\right)  $-th entry, but also the
$\left(  2,1\right)  $-nd entry of our matrix. In our proof of Theorem
\ref{thm.triangular.Alamuv}, each of the operations that we performed changed
only one entry of our matrix; but here in the proof of Theorem
\ref{thm.triangular.Slamu}, this is not the case. Nevertheless, the proof
works, because the exact values of the entries below the diagonal are not
important.}. Furthermore, the $\left(  1,1\right)  $-th entry of the matrix
has not been changed by $S_{2}^{\lambda}$ (since $S_{2}^{\lambda}$ only
changes the $2$-nd row), and thus is still $1$. As the result, we have turned
the $\left(  2,2\right)  $-th entry of the matrix into $1$. In other words,
our matrix now looks as follows:
\[
\left(
\begin{array}
[c]{ccccc}%
1 & 0 & 0 & \cdots & 0\\
C_{2,1}/C_{2,2} & 1 & 0 & \cdots & 0\\
C_{3,1} & C_{3,2} & C_{3,3} & \cdots & 0\\
\vdots & \vdots & \vdots & \ddots & \vdots\\
C_{n,1} & C_{n,2} & C_{n,3} & \cdots & C_{n,n}%
\end{array}
\right)  .
\]


\item Next, we similarly perform the row scaling $S_{3}^{\lambda}$ (for an
appropriate choice of $\lambda$, namely for $\lambda=1/C_{3,3}$) to turn the
$\left(  3,3\right)  $-th entry of the matrix into $1$. As the result, we have
turned the $\left(  3,3\right)  $-th entry of the matrix into $1$. In other
words, our matrix now looks as follows:
\[
\left(
\begin{array}
[c]{ccccc}%
1 & 0 & 0 & \cdots & 0\\
C_{2,1}/C_{2,2} & 1 & 0 & \cdots & 0\\
C_{3,1}/C_{3,3} & C_{3,2}/C_{3,3} & 1 & \cdots & 0\\
\vdots & \vdots & \vdots & \ddots & \vdots\\
C_{n,1} & C_{n,2} & C_{n,3} & \cdots & C_{n,n}%
\end{array}
\right)  .
\]


\item We continue this process, changing each diagonal entry of the matrix
into $1$ (one at a time). At the end, our matrix looks as follows:%
\[
\left(
\begin{array}
[c]{ccccc}%
1 & 0 & 0 & \cdots & 0\\
C_{2,1}/C_{2,2} & 1 & 0 & \cdots & 0\\
C_{3,1}/C_{3,3} & C_{3,2}/C_{3,3} & 1 & \cdots & 0\\
\vdots & \vdots & \vdots & \ddots & \vdots\\
C_{n,1}/C_{n,n} & C_{n,2}/C_{n,n} & C_{n,3}/C_{n,n} & \cdots & 1
\end{array}
\right)  .
\]
This is a lower-unitriangular matrix. Denote it by $D$. Theorem
\ref{thm.triangular.Alamuv} (applied to $D$ instead of $C$) shows that $D$ is
lower-unitriangular if and only if $D$ is a product of lower addition
matrices. Hence, $D$ is product of lower addition matrices (since $D$ is lower-unitriangular).
\end{itemize}

Thus, we have found an algorithm to transform our matrix $C$ into a
lower-unitriangular matrix $D$ by a sequence of row scalings. Therefore, we
can conversely transform the matrix $D$ into $C$ by a sequence of row
scalings\footnote{because (as we have shown) any row scaling $S_{u}^{\lambda}$
is invertible, and can be undone by another row scaling}. Let us denote these
row scalings (used to transform $D$ into $C$) by $S_{u_{1}}^{\lambda_{1}%
},S_{u_{2}}^{\lambda_{2}},\ldots,S_{u_{k}}^{\lambda_{k}}$, \textbf{numbered
backwards} (i.e., starting from the one used last). Since each row scaling
$S_{u}^{\lambda}$ amounts to multiplying a matrix by the matrix $S_{u}%
^{\lambda}$ (that is, it sends any $n\times m$-matrix $B$ to $S_{u}^{\lambda
}B$), we thus conclude that
\begin{equation}
C=S_{u_{1}}^{\lambda_{1}}S_{u_{2}}^{\lambda_{2}}\cdots S_{u_{k}}^{\lambda_{k}%
}D. \label{pf.thm.triangular.Slamu.res1}%
\end{equation}


But $S_{u_{1}}^{\lambda_{1}}S_{u_{2}}^{\lambda_{2}}\cdots S_{u_{k}}%
^{\lambda_{k}}$ is a product of scaling matrices (because each of the matrices
$S_{u_{1}}^{\lambda_{1}},S_{u_{2}}^{\lambda_{2}},\ldots,S_{u_{k}}^{\lambda
_{k}}$ is a scaling matrix\footnote{Here we are using the fact that, for any
row scaling $S_{u}^{\lambda}$, the matrix $S_{u}^{\lambda}$ is a scaling
matrix.}). Hence, (\ref{pf.thm.triangular.Slamu.res1}) rewrites as follows:%
\[
C=\underbrace{S_{u_{1}}^{\lambda_{1}}S_{u_{2}}^{\lambda_{2}}\cdots S_{u_{k}%
}^{\lambda_{k}}}_{\substack{\text{this is a product of}\\\text{scaling
matrices}}}\underbrace{D}_{\substack{\text{this is a product of}\\\text{lower
addition matrices}}}.
\]
Hence, $C$ is a product of scaling matrices and lower addition matrices. This
is precisely what we had to prove. This proves the $\Longrightarrow$ direction
of Theorem \ref{thm.triangular.Slamu}. Hence, the proof of Theorem
\ref{thm.triangular.Slamu} is complete.
\end{proof}

\begin{remark}
Our proof of Theorem \ref{thm.triangular.Slamu} (specifically, of its
$\Longrightarrow$ direction) actually gives an explicit representation of an
invertibly lower-triangular $n\times n$-matrix $C$ as a product of scaling
matrices and lower addition matrices. We leave the details to the reader.
\end{remark}

We can use Theorem \ref{thm.triangular.Slamu} to prove the following analogue
of Theorem \ref{thm.triangular.inverse}, in the same way as we used Theorem
\ref{thm.triangular.Alamuv} to prove Theorem \ref{thm.triangular.inverse} itself:

\begin{theorem}
\label{thm.triangular.inverse.inv}Let $n\in\mathbb{N}$. Let $A$ be an
invertibly lower-triangular $n\times n$-matrix. Then, $A$ is invertible, and
its inverse $A^{-1}$ is again invertibly lower-triangular.
\end{theorem}

\begin{proof}
[Proof of Theorem \ref{thm.triangular.inverse.inv}.]Theorem
\ref{thm.triangular.Slamu} (applied to $C=A$) shows that $A$ is invertibly
lower-triangular if and only if $A$ is a product of scaling matrices and lower
addition matrices. Hence, $A$ is product of scaling matrices and lower
addition matrices (since $A$ is invertibly lower-triangular). In other words,
$A$ has the form $A=A_{1}A_{2}\cdots A_{k}$ for some $k\in\mathbb{N}$ and some
$k$ matrices $A_{1},A_{2},\ldots,A_{k}$, where each of $A_{1},A_{2}%
,\ldots,A_{k}$ is either a scaling matrix or a lower addition matrix. Consider
these $k$ and $A_{1},A_{2},\ldots,A_{k}$.

Observe the following fact:

\begin{statement}
\textit{Fact 1:} Let $i\in\left\{  1,2,\ldots,k\right\}  $. Then, the matrix
$A_{i}$ is invertible, and its inverse $A_{i}^{-1}$ is either a scaling matrix
or a lower addition matrix.
\end{statement}

[\textit{Proof of Fact 1:} We know that $A_{i}$ is either a scaling matrix or
a lower addition matrix (since each of $A_{1},A_{2},\ldots,A_{k}$ is either a
scaling matrix or a lower addition matrix). In other words, we are in one of
the following two cases:

\begin{statement}
\textit{Case 1:} The matrix $A_{i}$ is a scaling matrix.
\end{statement}

\begin{statement}
\textit{Case 2:} The matrix $A_{i}$ is a lower addition matrix.
\end{statement}

Let us consider Case 1 first. In this case, $A_{i}$ is a scaling matrix. In
other words, $A_{i}$ has the form $A_{i}=S_{u}^{\lambda}$ for some nonzero
number $\lambda$ and some $u\in\left\{  1,2,\ldots,n\right\}  $. Consider
these $\lambda$ and $u$. Proposition \ref{prop.Slamu.lambda+mu} \textbf{(c)}
shows that the matrix $S_{u}^{\lambda}$ is invertible, and its inverse is
$\left(  S_{u}^{\lambda}\right)  ^{-1}=S_{u}^{1/\lambda}$. Since
$S_{u}^{\lambda}=A_{i}$, this rewrites as follows: The matrix $A_{i}$ is
invertible, and its inverse is $A_{i}^{-1}=S_{u}^{1/\lambda}$. But
$S_{u}^{1/\lambda}$ is a scaling matrix. In other words, $A_{i}^{-1}$ is a
scaling matrix (since $A_{i}^{-1}=S_{u}^{1/\lambda}$). Hence, $A_{i}^{-1}$ is
either a scaling matrix or a lower addition matrix. Thus, Fact 1 is proven in
Case 1 (since we have already shown that $A_{i}$ is invertible).

Let us now consider Case 2. In this case, $A_{i}$ is a lower addition matrix.
In other words, $A_{i}$ has the form $A_{i}=A_{u,v}^{\lambda}$, where
$\lambda$ is a number, and where $u$ and $v$ are two elements of $\left\{
1,2,\ldots,n\right\}  $ satisfying $u>v$ (by the definition of a
\textquotedblleft lower addition matrix\textquotedblright). Consider these
$\lambda$, $u$ and $v$. Proposition \ref{prop.Alamuv.lambda+mu} \textbf{(c)}
shows that the matrix $A_{u,v}^{\lambda}$ is invertible, and its inverse is
$\left(  A_{u,v}^{\lambda}\right)  ^{-1}=A_{u,v}^{-\lambda}$. Since
$A_{u,v}^{\lambda}=A_{i}$, this rewrites as follows: The matrix $A_{i}$ is
invertible, and its inverse is $A_{i}^{-1}=A_{u,v}^{-\lambda}$. But
$A_{u,v}^{-\lambda}$ is a lower addition matrix (since $u>v$). In other words,
$A_{i}^{-1}$ is a lower addition matrix (since $A_{i}^{-1}=A_{u,v}^{-\lambda}%
$). Hence, $A_{i}^{-1}$ is either a scaling matrix or a lower addition matrix.
Thus, Fact 1 is proven in Case 2 (since we have already shown that $A_{i}$ is invertible).

We have now proven Fact 1 in each of the two Cases 1 and 2. Thus, Fact 1 is proven.]

We are in one of the following two cases:

\begin{statement}
\textit{Case 1:} We have $k\neq0$.
\end{statement}

\begin{statement}
\textit{Case 2:} We have $k=0$.
\end{statement}

Let us deal with Case 1. In this case, we have $k\neq0$; thus, $k$ is a
positive integer.

Fact 1 shows that, for each $i\in\left\{  1,2,\ldots,k\right\}  $, the matrix
$A_{i}$ is invertible, and its inverse $A_{i}^{-1}$ is either a scaling matrix
or a lower addition matrix. In other words, the matrices $A_{1},A_{2}%
,\ldots,A_{k}$ are invertible, and each of their inverses $A_{1}^{-1}%
,A_{2}^{-1},\ldots,A_{k}^{-1}$ is either a scaling matrix or a lower addition
matrix. In other words, each of $A_{k}^{-1},A_{k-1}^{-1},\ldots,A_{1}^{-1}$ is
either a scaling matrix or a lower addition matrix.

Now, Proposition \ref{prop.inverses.A1Ak} shows that the matrix $A_{1}%
A_{2}\cdots A_{k}$ is invertible, and its inverse is $\left(  A_{1}A_{2}\cdots
A_{k}\right)  ^{-1}=A_{k}^{-1}A_{k-1}^{-1}\cdots A_{1}^{-1}$. Since
$A=A_{1}A_{2}\cdots A_{k}$, this rewrites as follows: The matrix $A$ is
invertible, and its inverse is $A^{-1}=A_{k}^{-1}A_{k-1}^{-1}\cdots A_{1}%
^{-1}$.

The equality $A^{-1}=A_{k}^{-1}A_{k-1}^{-1}\cdots A_{1}^{-1}$ shows that
$A^{-1}$ is a product of scaling matrices and lower addition matrices (since
each of $A_{k}^{-1},A_{k-1}^{-1},\ldots,A_{1}^{-1}$ is either a scaling matrix
or a lower addition matrix). But Theorem \ref{thm.triangular.Slamu} (applied
to $C=A^{-1}$) shows that $A^{-1}$ is invertibly lower-triangular if and only
if $A^{-1}$ is a product of scaling matrices and lower addition matrices.
Hence, $A^{-1}$ is invertibly lower-triangular (since $A^{-1}$ is a product of
scaling matrices and lower addition matrices). This completes the proof of
Theorem \ref{thm.triangular.inverse.inv} in Case 1.

Case 2 is trivial (indeed, $A=I_{n}$ in this case) and is left to the
reader.\footnote{Alternatively, our proof for Case 1 can be made to work in
Case 2 as well, because Proposition \ref{prop.inverses.A1Ak} holds for $k=0$
(as long as we define the empty product to be $I_{n}$). See Remark
\ref{rmk.prop.inverses.A1Ak.k=0} for the details.} Thus, Theorem
\ref{thm.triangular.inverse.inv} is proven in both Cases 1 and 2; this shows
that Theorem \ref{thm.triangular.inverse.inv} is always valid.
\end{proof}

Again, a similar result holds for upper-triangular matrices (and, again, has a
similar proof):

\begin{theorem}
\label{thm.triangular.inverse-up.inv}Let $n\in\mathbb{N}$. Let $A$ be an
invertibly upper-triangular $n\times n$-matrix. Then, $A$ is invertible, and
its inverse $A^{-1}$ is again invertibly upper-triangular.
\end{theorem}

\subsection{(*) Yet another proof of triangular invertibility}

Let us now give a second proof of Theorem \ref{thm.triangular.inverse.inv}. We
first shall show a lemma:

\begin{lemma}
\label{lem.triangular.inverse.inv.3rd.1}Let $n\in\mathbb{N}$. Let $A$ be an
invertibly lower-triangular $n\times n$-matrix. Let $b=\left(  b_{1}%
,b_{2},\ldots,b_{n}\right)  ^{T}$ be a column vector of size $n$ (that is, an
$n\times1$-matrix). Let $r\in\left\{  1,2,\ldots,n\right\}  $ be such that
$b_{1}=b_{2}=\cdots=b_{r-1}=0$. (Notice that if $r=1$, then the equality
$b_{1}=b_{2}=\cdots=b_{r-1}=0$ claims nothing, and thus is automatically true
-- i.e., the numbers $b_{1},b_{2},\ldots,b_{n}$ can be arbitrary in this case.)

Then, there exists a column vector $v=\left(  v_{1},v_{2},\ldots,v_{n}\right)
^{T}$ of size $n$ satisfying $Av=b$ and $v_{1}=v_{2}=\cdots=v_{r-1}=0$ and
$v_{r}=\dfrac{1}{A_{r,r}}b_{r}$.
\end{lemma}

\begin{proof}
[Proof of Lemma \ref{lem.triangular.inverse.inv.3rd.1}.]The matrix $A$ is
invertibly lower-triangular. In other words, $A$ is lower-triangular and all
its diagonal entries are nonzero (because this is what \textquotedblleft
invertibly lower-triangular\textquotedblright\ means). Since $A$ is
lower-triangular, we have
\begin{equation}
A_{i,j}=0\ \ \ \ \ \ \ \ \ \ \text{whenever }i<j
\label{pf.lem.triangular.inverse.inv.3rd.1.tria}%
\end{equation}
(by the definition of \textquotedblleft lower-triangular\textquotedblright).
Since all diagonal entries of $A$ are nonzero, we have
\begin{equation}
A_{i,i}\neq0\ \ \ \ \ \ \ \ \ \ \text{for each }i\in\left\{  1,2,\ldots
,n\right\}  . \label{pf.lem.triangular.inverse.inv.3rd.1.inv}%
\end{equation}


We shall now define $n$ numbers $v_{1},v_{2},\ldots,v_{n}$. Our definition is
recursive: For each $i\in\left\{  1,2,\ldots,n\right\}  $, we assume that the
first $i-1$ numbers $v_{1},v_{2},\ldots,v_{i-1}$ are already defined, and we
define the next number $v_{i}$ by%
\begin{equation}
v_{i}=\dfrac{1}{A_{i,i}}\left(  b_{i}-\left(  A_{i,1}v_{1}+A_{i,2}v_{2}%
+\cdots+A_{i,i-1}v_{i-1}\right)  \right)
\label{pf.lem.triangular.inverse.inv.3rd.1.rec}%
\end{equation}
\footnote{The division by $A_{i,i}$ is allowed because of
(\ref{pf.lem.triangular.inverse.inv.3rd.1.inv}).}. This is a valid recursive
definition, because it gives us a way to compute the numbers $v_{1}%
,v_{2},\ldots,v_{n}$ one by one (beginning with $v_{1}$, then proceeding to
$v_{2}$, then to $v_{3}$, and so on). Here is how this computation will look like:

\begin{itemize}
\item The number $v_{1}$ is defined (and can be computed) by $v_{1}=\dfrac
{1}{A_{1,1}}b_{1}$. (This is the particular case of
(\ref{pf.lem.triangular.inverse.inv.3rd.1.rec}) for $i=1$%
.\ \ \ \ \footnote{Note that the sum $A_{i,1}v_{1}+A_{i,2}v_{2}+\cdots
+A_{i,i-1}v_{i-1}$ becomes an empty sum when $i=1$, and thus equals $0$;
therefore, we have omitted it.})

\item The number $v_{2}$ is defined (and can be computed) by $v_{2}=\dfrac
{1}{A_{2,2}}\left(  b_{2}-A_{2,1}v_{1}\right)  $.

\item The number $v_{3}$ is defined (and can be computed) by $v_{3}=\dfrac
{1}{A_{3,3}}\left(  b_{3}-\left(  A_{3,1}v_{1}+A_{3,2}v_{2}\right)  \right)  $.

\item And so on, up to $v_{n}$.
\end{itemize}

We furthermore define a column vector $v$ by $v=\left(  v_{1},v_{2}%
,\ldots,v_{n}\right)  ^{T}$. Now, we claim the following:

\begin{statement}
\textit{Claim 1:} We have $Av=b$.
\end{statement}

\begin{statement}
\textit{Claim 2:} We have $v_{1}=v_{2}=\cdots=v_{r-1}=0$.
\end{statement}

\begin{statement}
\textit{Claim 3:} We have $v_{r}=\dfrac{1}{A_{r,r}}b_{r}$.
\end{statement}

[\textit{Proof of Claim 1:} Multiplying the equalities $A=\left(
\begin{array}
[c]{cccc}%
A_{1,1} & A_{1,2} & \cdots & A_{1,n}\\
A_{2,1} & A_{2,2} & \cdots & A_{2,n}\\
\vdots & \vdots & \ddots & \vdots\\
A_{n,1} & A_{n,2} & \cdots & A_{n,n}%
\end{array}
\right)  $ and $v=\left(  v_{1},v_{2},\ldots,v_{n}\right)  ^{T}=\left(
\begin{array}
[c]{c}%
v_{1}\\
v_{2}\\
\vdots\\
v_{n}%
\end{array}
\right)  $, we obtain%
\begin{align}
Av  &  =\left(
\begin{array}
[c]{cccc}%
A_{1,1} & A_{1,2} & \cdots & A_{1,n}\\
A_{2,1} & A_{2,2} & \cdots & A_{2,n}\\
\vdots & \vdots & \ddots & \vdots\\
A_{n,1} & A_{n,2} & \cdots & A_{n,n}%
\end{array}
\right)  \left(
\begin{array}
[c]{c}%
v_{1}\\
v_{2}\\
\vdots\\
v_{n}%
\end{array}
\right) \nonumber\\
&  =\left(
\begin{array}
[c]{c}%
A_{1,1}v_{1}+A_{1,2}v_{2}+\cdots+A_{1,n}v_{n}\\
A_{2,1}v_{1}+A_{2,2}v_{2}+\cdots+A_{2,n}v_{n}\\
\vdots\\
A_{n,1}v_{1}+A_{n,2}v_{2}+\cdots+A_{n,n}v_{n}%
\end{array}
\right)  . \label{pf.lem.triangular.inverse.inv.3rd.1.Av=}%
\end{align}
But for each $i\in\left\{  1,2,\ldots,n\right\}  $, we have%
\begin{align*}
&  A_{i,1}v_{1}+A_{i,2}v_{2}+\cdots+A_{i,n}v_{n}\\
&  =\left(  A_{i,1}v_{1}+A_{i,2}v_{2}+\cdots+A_{i,i}v_{i}\right) \\
&  \ \ \ \ \ \ \ \ \ \ +\left(  \underbrace{A_{i,i+1}}%
_{\substack{=0\\\text{(by (\ref{pf.lem.triangular.inverse.inv.3rd.1.tria}%
),}\\\text{applied to }j=i+1\text{)}}}v_{i+1}+\underbrace{A_{i,i+2}%
}_{\substack{=0\\\text{(by (\ref{pf.lem.triangular.inverse.inv.3rd.1.tria}%
),}\\\text{applied to }j=i+2\text{)}}}v_{i+2}+\cdots+\underbrace{A_{i,n}%
}_{\substack{=0\\\text{(by (\ref{pf.lem.triangular.inverse.inv.3rd.1.tria}%
),}\\\text{applied to }j=n\text{)}}}v_{n}\right) \\
&  =\left(  A_{i,1}v_{1}+A_{i,2}v_{2}+\cdots+A_{i,i}v_{i}\right)
+\underbrace{\left(  0v_{i+1}+0v_{i+2}+\cdots+0v_{n}\right)  }_{=0}\\
&  =A_{i,1}v_{1}+A_{i,2}v_{2}+\cdots+A_{i,i}v_{i}\\
&  =\left(  A_{i,1}v_{1}+A_{i,2}v_{2}+\cdots+A_{i,i-1}v_{i-1}\right)
+\underbrace{A_{i,i}v_{i}}_{\substack{=b_{i}-\left(  A_{i,1}v_{1}+A_{i,2}%
v_{2}+\cdots+A_{i,i-1}v_{i-1}\right)  \\\text{(by
(\ref{pf.lem.triangular.inverse.inv.3rd.1.rec}))}}}\\
&  =\left(  A_{i,1}v_{1}+A_{i,2}v_{2}+\cdots+A_{i,i-1}v_{i-1}\right)
+b_{i}-\left(  A_{i,1}v_{1}+A_{i,2}v_{2}+\cdots+A_{i,i-1}v_{i-1}\right) \\
&  =b_{i}.
\end{align*}
In other words, the $n$ equalities%
\[%
\begin{array}
[c]{c}%
A_{1,1}v_{1}+A_{1,2}v_{2}+\cdots+A_{1,n}v_{n}=b_{1};\\
A_{2,1}v_{1}+A_{2,2}v_{2}+\cdots+A_{2,n}v_{n}=b_{2};\\
\vdots\\
A_{n,1}v_{1}+A_{n,2}v_{2}+\cdots+A_{n,n}v_{n}=b_{n}%
\end{array}
\]
hold. In other words,%
\[
\left(
\begin{array}
[c]{c}%
A_{1,1}v_{1}+A_{1,2}v_{2}+\cdots+A_{1,n}v_{n}\\
A_{2,1}v_{1}+A_{2,2}v_{2}+\cdots+A_{2,n}v_{n}\\
\vdots\\
A_{n,1}v_{1}+A_{n,2}v_{2}+\cdots+A_{n,n}v_{n}%
\end{array}
\right)  =\left(
\begin{array}
[c]{c}%
b_{1}\\
b_{2}\\
\vdots\\
b_{n}%
\end{array}
\right)  .
\]
Thus, (\ref{pf.lem.triangular.inverse.inv.3rd.1.Av=}) becomes
\[
Av=\left(
\begin{array}
[c]{c}%
A_{1,1}v_{1}+A_{1,2}v_{2}+\cdots+A_{1,n}v_{n}\\
A_{2,1}v_{1}+A_{2,2}v_{2}+\cdots+A_{2,n}v_{n}\\
\vdots\\
A_{n,1}v_{1}+A_{n,2}v_{2}+\cdots+A_{n,n}v_{n}%
\end{array}
\right)  =\left(
\begin{array}
[c]{c}%
b_{1}\\
b_{2}\\
\vdots\\
b_{n}%
\end{array}
\right)  =b.
\]
This proves Claim 1.]

[\textit{Proof of Claim 2:} We shall show that%
\begin{equation}
v_{1}=v_{2}=\cdots=v_{k-1}=0\ \ \ \ \ \ \ \ \ \ \text{for each }k\in\left\{
1,2,\ldots,r\right\}  \text{.} \label{pf.lem.triangular.inverse.inv.3rd.1.C2a}%
\end{equation}
Our proof of (\ref{pf.lem.triangular.inverse.inv.3rd.1.C2a}) will proceed by
induction over $k$:

\textit{Induction base:} For $k=1$, the equality
(\ref{pf.lem.triangular.inverse.inv.3rd.1.C2a}) claims nothing (because there
are no numbers $v_{1},v_{2},\ldots,v_{k-1}$ when $k=1$), and thus is true (for
the stupid reason that an empty claim is always true). This completes the
induction base.

\textit{Induction step:} Let $i\in\left\{  1,2,\ldots,r-1\right\}  $. Assume
that (\ref{pf.lem.triangular.inverse.inv.3rd.1.C2a}) holds for $k=i$. We must
now prove that (\ref{pf.lem.triangular.inverse.inv.3rd.1.C2a}) holds for
$k=i+1$.

We have $i\in\left\{  1,2,\ldots,r-1\right\}  $ and thus $b_{i}=0$ (since
$b_{1}=b_{2}=\cdots=b_{r-1}=0$).

We have assumed that (\ref{pf.lem.triangular.inverse.inv.3rd.1.C2a}) holds for
$k=i$. In other words, we have $v_{1}=v_{2}=\cdots=v_{i-1}=0$. In other words,%
\begin{equation}
v_{\ell}=0\ \ \ \ \ \ \ \ \ \ \text{for each }\ell\in\left\{  1,2,\ldots
,i-1\right\}  . \label{pf.lem.triangular.inverse.inv.3rd.1.C2a.pf.1}%
\end{equation}


But now, (\ref{pf.lem.triangular.inverse.inv.3rd.1.rec}) yields
\begin{align*}
v_{i}  &  =\dfrac{1}{A_{i,i}}\left(  b_{i}-\left(  A_{i,1}\underbrace{v_{1}%
}_{\substack{=0\\\text{(by (\ref{pf.lem.triangular.inverse.inv.3rd.1.C2a.pf.1}%
), applied}\\\text{to }\ell=1\text{)}}}+A_{i,2}\underbrace{v_{2}%
}_{\substack{=0\\\text{(by (\ref{pf.lem.triangular.inverse.inv.3rd.1.C2a.pf.1}%
), applied}\\\text{to }\ell=2\text{)}}}+\cdots+A_{i,i-1}\underbrace{v_{i-1}%
}_{\substack{=0\\\text{(by (\ref{pf.lem.triangular.inverse.inv.3rd.1.C2a.pf.1}%
), applied}\\\text{to }\ell=i-1\text{)}}}\right)  \right) \\
&  =\dfrac{1}{A_{i,i}}\left(  b_{i}-\underbrace{\left(  A_{i,1}0+A_{i,2}%
0+\cdots+A_{i,i-1}0\right)  }_{=0}\right)  =\dfrac{1}{A_{i,i}}%
\underbrace{b_{i}}_{=0}=0.
\end{align*}
Combining this with $v_{1}=v_{2}=\cdots=v_{i-1}=0$, we obtain $v_{1}%
=v_{2}=\cdots=v_{i}=0$. In other words,
(\ref{pf.lem.triangular.inverse.inv.3rd.1.C2a}) holds for $k=i+1$. This
completes the induction step. Hence,
(\ref{pf.lem.triangular.inverse.inv.3rd.1.C2a}) is proven.

Now that we have proven (\ref{pf.lem.triangular.inverse.inv.3rd.1.C2a}), we
can apply (\ref{pf.lem.triangular.inverse.inv.3rd.1.C2a}) to $k=r$. We thus
obtain $v_{1}=v_{2}=\cdots=v_{r-1}=0$. This proves Claim 2.]

[\textit{Proof of Claim 3:} Claim 2 shows that $v_{1}=v_{2}=\cdots=v_{r-1}=0$.
In other words,%
\begin{equation}
v_{\ell}=0\ \ \ \ \ \ \ \ \ \ \text{for each }\ell\in\left\{  1,2,\ldots
,r-1\right\}  . \label{pf.lem.triangular.inverse.inv.3rd.1.C3.pf.1}%
\end{equation}
But (\ref{pf.lem.triangular.inverse.inv.3rd.1.rec}) (applied to $i=r$) yields
\begin{align*}
v_{r}  &  =\dfrac{1}{A_{r,r}}\left(  b_{r}-\left(  A_{r,1}\underbrace{v_{1}%
}_{\substack{=0\\\text{(by (\ref{pf.lem.triangular.inverse.inv.3rd.1.C3.pf.1}%
), applied}\\\text{to }\ell=1\text{)}}}+A_{r,2}\underbrace{v_{2}%
}_{\substack{=0\\\text{(by (\ref{pf.lem.triangular.inverse.inv.3rd.1.C3.pf.1}%
), applied}\\\text{to }\ell=2\text{)}}}+\cdots+A_{r,r-1}\underbrace{v_{r-1}%
}_{\substack{=0\\\text{(by (\ref{pf.lem.triangular.inverse.inv.3rd.1.C3.pf.1}%
), applied}\\\text{to }\ell=r-1\text{)}}}\right)  \right) \\
&  =\dfrac{1}{A_{r,r}}\left(  b_{r}-\underbrace{\left(  A_{r,1}0+A_{r,2}%
0+\cdots+A_{r,r-1}0\right)  }_{=0}\right)  =\dfrac{1}{A_{r,r}}b_{r}.
\end{align*}
This proves Claim 3.]

We have now proven all three claims. Thus, our column vector $v=\left(
v_{1},v_{2},\ldots,v_{n}\right)  ^{T}$ satisfies $Av=b$ and $v_{1}%
=v_{2}=\cdots=v_{r-1}=0$ and $v_{r}=\dfrac{1}{A_{r,r}}b_{r}$. This shows that
such a vector exists; in other words, Lemma
\ref{lem.triangular.inverse.inv.3rd.1} is proven.
\end{proof}

Next, we prove a lemma which is \textquotedblleft almost\textquotedblright%
\ Theorem \ref{thm.triangular.inverse.inv}:

\begin{lemma}
\label{lem.triangular.inverse.inv.3rd.2}Let $n\in\mathbb{N}$. Let $A$ be an
invertibly lower-triangular $n\times n$-matrix. Then, there exists an
invertibly lower-triangular $n\times n$-matrix $B$ such that $AB=I_{n}$.
\end{lemma}

The matrix $B$ in Lemma \ref{lem.triangular.inverse.inv.3rd.2} will turn out
to be the inverse of $A$; but Lemma \ref{lem.triangular.inverse.inv.3rd.2}
does not yet claim this (instead, Lemma \ref{lem.triangular.inverse.inv.3rd.2}
only guarantees that $B$ is a right inverse of $A$).

\begin{proof}
[Proof of Lemma \ref{lem.triangular.inverse.inv.3rd.2}.]For each $j\in\left\{
1,2,\ldots,n\right\}  $, we construct a column vector $v_{\left[  j\right]  }$
of size $n$ as follows:

Consider the vector $\operatorname*{col}\nolimits_{j}\left(  I_{n}\right)  $;
this is the $j$-th column of the identity matrix $I_{n}$. Since $I_{n}=\left(
\delta_{i,j}\right)  _{1\leq i\leq n,\ 1\leq j\leq n}$, we have%
\[
\operatorname*{col}\nolimits_{j}\left(  I_{n}\right)  =\left(
\begin{array}
[c]{c}%
\delta_{1,j}\\
\delta_{2,j}\\
\vdots\\
\delta_{n,j}%
\end{array}
\right)  =\left(  \delta_{1,j},\delta_{2,j},\ldots,\delta_{n,j}\right)  ^{T}.
\]
This vector $\operatorname*{col}\nolimits_{j}\left(  I_{n}\right)  $ is a
column vector of size $n$, and satisfies $\delta_{1,j}=\delta_{2,j}%
=\cdots=\delta_{j-1,j}=0$ (since none of the numbers $1,2,\ldots,j-1$ equals
$j$). Hence, Lemma \ref{lem.triangular.inverse.inv.3rd.1} (applied to
$b=\operatorname*{col}\nolimits_{j}\left(  I_{n}\right)  $ and $b_{i}%
=\delta_{i,j}$ and $r=j$) says that there exists a column vector $v=\left(
v_{1},v_{2},\ldots,v_{n}\right)  ^{T}$ of size $n$ satisfying
$Av=\operatorname*{col}\nolimits_{j}\left(  I_{n}\right)  $ and $v_{1}%
=v_{2}=\cdots=v_{j-1}=0$ and $v_{j}=\dfrac{1}{A_{j,j}}\delta_{j,j}$. Denote
this vector $v$ by $v_{\left[  j\right]  }$, and denote its entries
$v_{1},v_{2},\ldots,v_{n}$ by $v_{1,j},v_{2,j},\ldots,v_{n,j}$ (in order to
stress that they depend on $j$).

Now, forget that we fixed $j$. Thus, for each $j\in\left\{  1,2,\ldots
,n\right\}  $, we have found a column vector
\begin{equation}
v_{\left[  j\right]  }=\left(  v_{1,j},v_{2,j},\ldots,v_{n,j}\right)  ^{T}
\label{pf.lem.triangular.inverse.inv.3rd.2.vj.1}%
\end{equation}
of size $n$ satisfying
\begin{equation}
Av_{\left[  j\right]  }=\operatorname*{col}\nolimits_{j}\left(  I_{n}\right)
\label{pf.lem.triangular.inverse.inv.3rd.2.vj.2}%
\end{equation}
and
\begin{equation}
v_{1,j}=v_{2,j}=\cdots=v_{j-1,j}=0
\label{pf.lem.triangular.inverse.inv.3rd.2.vj.3}%
\end{equation}
and
\begin{equation}
v_{j,j}=\dfrac{1}{A_{j,j}}\delta_{j,j}.
\label{pf.lem.triangular.inverse.inv.3rd.2.vj.4}%
\end{equation}
Altogether, these are $n$ column vectors $v_{\left[  1\right]  },v_{\left[
2\right]  },\ldots,v_{\left[  n\right]  }$ of size $n$. We can assemble them
into a matrix $B$: Namely, set
\[
B=\left(  v_{i,j}\right)  _{1\leq i\leq n,\ 1\leq j\leq n}.
\]
This matrix $B$ therefore satisfies%
\begin{equation}
B_{i,j}=v_{i,j}\ \ \ \ \ \ \ \ \ \ \text{for all }i\in\left\{  1,2,\ldots
,n\right\}  \text{ and }j\in\left\{  1,2,\ldots,n\right\}  .
\label{pf.lem.triangular.inverse.inv.3rd.2.B=v}%
\end{equation}
Hence, the matrix $B$ is lower-triangular\footnote{\textit{Proof.} Let
$j\in\left\{  1,2,\ldots,n\right\}  $. Then, $v_{1,j}=v_{2,j}=\cdots
=v_{j-1,j}=0$ (by (\ref{pf.lem.triangular.inverse.inv.3rd.2.vj.3})). In other
words, $v_{i,j}=0$ for every $i\in\left\{  1,2,\ldots,n\right\}  $ satisfying
$i<j$. In light of (\ref{pf.lem.triangular.inverse.inv.3rd.2.B=v}), this
rewrites as $B_{i,j}=0$ for every $i\in\left\{  1,2,\ldots,n\right\}  $
satisfying $i<j$.
\par
Now, forget that we fixed $j$. We thus have shown that $B_{i,j}=0$ whenever
$i<j$. In other words, the matrix $B$ is lower-triangular.} and all its
diagonal entries are nonzero\footnote{\textit{Proof.} Let $i\in\left\{
1,2,\ldots,n\right\}  $. Applying
(\ref{pf.lem.triangular.inverse.inv.3rd.2.vj.4}) to $j=i$, we find
$v_{i,i}=\dfrac{1}{A_{i,i}}\underbrace{\delta_{i,i}}%
_{\substack{=1\\\text{(since }i=i\text{)}}}=\dfrac{1}{A_{i,i}}\neq0$. But
(\ref{pf.lem.triangular.inverse.inv.3rd.2.B=v}) (applied to $j=i$) yields
$B_{i,i}=v_{i,i}\neq0$.
\par
Now, forget that we fixed $i$. We thus have learnt that $B_{i,i}\neq0$ for
each $i\in\left\{  1,2,\ldots,n\right\}  $. In other words, the diagonal
entries of the matrix $B$ are nonzero.}. In other words, the matrix $B$ is
invertibly lower-triangular.

Also, recall that $B=\left(  v_{i,j}\right)  _{1\leq i\leq n,\ 1\leq j\leq n}%
$. Hence, every $j\in\left\{  1,2,\ldots,n\right\}  $ satisfies%
\begin{equation}
\operatorname*{col}\nolimits_{j}B=\left(
\begin{array}
[c]{c}%
v_{1,j}\\
v_{2,j}\\
\vdots\\
v_{n,j}%
\end{array}
\right)  =\left(  v_{1,j},v_{2,j},\ldots,v_{n,j}\right)  ^{T}=v_{\left[
j\right]  }\ \ \ \ \ \ \ \ \ \ \left(  \text{by
(\ref{pf.lem.triangular.inverse.inv.3rd.2.vj.1})}\right)  .
\label{pf.lem.triangular.inverse.inv.3rd.2.coljB}%
\end{equation}
Now, Proposition \ref{prop.matrix-prod.rc} \textbf{(d)} (applied to $m=n$ and
$p=n$) shows that, for each $j\in\left\{  1,2,\ldots,n\right\}  $, we have%
\[
\operatorname*{col}\nolimits_{j}\left(  AB\right)  =A\cdot
\underbrace{\operatorname*{col}\nolimits_{j}B}_{\substack{=v_{\left[
j\right]  }\\\text{(by (\ref{pf.lem.triangular.inverse.inv.3rd.2.coljB}))}%
}}=Av_{\left[  j\right]  }=\operatorname*{col}\nolimits_{j}\left(
I_{n}\right)  \ \ \ \ \ \ \ \ \ \ \left(  \text{by
(\ref{pf.lem.triangular.inverse.inv.3rd.2.vj.2})}\right)  .
\]
In other words, each column of the $n\times n$-matrix $AB$ equals the
corresponding column of $I_{n}$. Thus, $AB=I_{n}$.

Hence, we have found an invertibly lower-triangular $n\times n$-matrix $B$
such that $AB=I_{n}$. This proves Lemma \ref{lem.triangular.inverse.inv.3rd.2}.
\end{proof}

We are now ready to prove Theorem \ref{thm.triangular.inverse.inv} by a rather
cunning trick:

\begin{proof}
[Proof of Theorem \ref{thm.triangular.inverse.inv}.]Lemma
\ref{lem.triangular.inverse.inv.3rd.2} shows that there exists an invertibly
lower-triangular $n\times n$-matrix $B$ such that $AB=I_{n}$. Fix such a $B$,
and denote it by $C$. Thus, $C$ is an invertibly lower-triangular $n\times
n$-matrix such that $AC=I_{n}$.

But we can also apply Lemma \ref{lem.triangular.inverse.inv.3rd.2} to $C$
instead of $A$. As the result, we conclude that there exists an invertibly
lower-triangular $n\times n$-matrix $B$ such that $CB=I_{n}$. Fix such a $B$,
and denote it by $D$. Thus, $D$ is an invertibly lower-triangular $n\times
n$-matrix such that $CD=I_{n}$.

Now, the matrix $A$ is a left inverse of $C$ (since $AC=I_{n}$), whereas the
matrix $D$ is a right inverse of $C$ (since $CD=I_{n}$). Hence, Proposition
\ref{prop.inverses.L=R} \textbf{(a)} (applied to $n$, $C$, $A$ and $D$ instead
of $m$, $A$, $L$ and $R$) reveals that $A=D$. Hence, $C\underbrace{A}%
_{=D}=CD=I_{n}$. Combining $CA=I_{n}$ with $AC=I_{n}$, we conclude that $C$ is
an inverse of $A$. Thus, the matrix $A$ is invertible, and its inverse is
$A^{-1}=C$. Thus, $A^{-1}$ is invertibly lower-triangular (since $C$ is
invertibly lower-triangular, but $A^{-1}=C$). This proves Theorem
\ref{thm.triangular.inverse.inv}.
\end{proof}

We have thus proven Theorem \ref{thm.triangular.inverse.inv} again.

We could similarly prove Theorem \ref{thm.triangular.inverse-up.inv}, though
this would require an analogue of Lemma \ref{lem.triangular.inverse.inv.3rd.1}
in which (for example) the condition $b_{1}=b_{2}=\cdots=b_{r-1}=0$ would be
replaced by $b_{r+1}=b_{r+2}=\cdots=b_{n}=0$ (and in the proof, we would have
to define the $v_{1},v_{2},\ldots,v_{n}$ by \textquotedblleft reverse
recursion\textquotedblright, beginning with $v_{n}$ and then proceeding with
$v_{n-1}$ and so on). It is probably quicker to derive Theorem
\ref{thm.triangular.inverse-up.inv} from Theorem
\ref{thm.triangular.inverse.inv} in the same way as we derived Theorem
\ref{thm.triangular.inverse-up} from Theorem \ref{thm.triangular.inverse}.
Either way, the proof is straightforward (given what has already been shown),
and is left to the reader.

Theorem \ref{thm.triangular.inverse} and Theorem
\ref{thm.triangular.inverse-up} can also be proven in the same way; the
changes usually boil down to replacing \textquotedblleft$\neq0$%
\textquotedblright\ by \textquotedblleft$=1$\textquotedblright. Again, this
proof can safely be left to the reader (who would thus obtain a third proof of
Theorem \ref{thm.triangular.inverse}!).

\subsection{The swapping matrices $T_{u,v}$}

The $\lambda$-addition matrices $A_{u,v}^{\lambda}$ from Definition
\ref{def.Alamuv} and the $\lambda$-scaling matrices $S_{u}^{\lambda}$ from
Definition \ref{def.Slamu} are two of the three kinds of matrices commonly
called \textquotedblleft elementary matrices\textquotedblright. The third kind
are the \textit{swapping matrices}:

\begin{definition}
\label{def.Tuv}Let $n\in\mathbb{N}$. Let $u$ and $v$ be two elements of
$\left\{  1,2,\ldots,n\right\}  $. Then, $T_{u,v}$ shall denote the $n\times
n$-matrix
\[
I_{n}-E_{u,u}-E_{v,v}+E_{u,v}+E_{v,u}.
\]
(Here, all the matrices of the form $E_{p,q}$ are meant to be $n\times
n$-matrices: that is, $E_{p,q}=E_{p,q,n,n}$ for all $p\in\left\{
1,2,\ldots,n\right\}  $ and $q\in\left\{  1,2,\ldots,n\right\}  $.)

Again, the notation $T_{u,v}$ is hiding the dependency on $n$, and we ought to
write $T_{u,v,n}$ instead; but we will not, because there will not be any real
occasion for confusion.
\end{definition}

\begin{example}
Let $n=4$. Then,%
\begin{align*}
T_{1,3}  &  =I_{n}-E_{1,1}-E_{3,3}+E_{1,3}+E_{3,1}=\left(
\begin{array}
[c]{cccc}%
0 & 0 & 1 & 0\\
0 & 1 & 0 & 0\\
1 & 0 & 0 & 0\\
0 & 0 & 0 & 1
\end{array}
\right)  \ \ \ \ \ \ \ \ \ \ \text{and}\\
T_{2,3}  &  =I_{n}-E_{2,2}-E_{3,3}+E_{2,3}+E_{3,2}=\left(
\begin{array}
[c]{cccc}%
1 & 0 & 0 & 0\\
0 & 0 & 1 & 0\\
0 & 1 & 0 & 0\\
0 & 0 & 0 & 1
\end{array}
\right)  \ \ \ \ \ \ \ \ \ \ \text{and}\\
T_{2,2}  &  =I_{n}-E_{2,2}-E_{2,2}+E_{2,2}+E_{2,2}=I_{n}=\left(
\begin{array}
[c]{cccc}%
1 & 0 & 0 & 0\\
0 & 1 & 0 & 0\\
0 & 0 & 1 & 0\\
0 & 0 & 0 & 1
\end{array}
\right)  .
\end{align*}

\end{example}

These examples exhibit a pattern: If $u=v$, then the matrix $T_{u,v}$ is
simply the identity matrix $I_{n}$; otherwise, it differs from $I_{n}$ only in
the $u$-th and $v$-th rows, which have their $1$'s in the \textquotedblleft
wrong\textquotedblright\ positions. This pattern holds in general:

\begin{proposition}
\label{prop.Tuv.entries}Let $n\in\mathbb{N}$. Let $u$ and $v$ be two distinct
elements of $\left\{  1,2,\ldots,n\right\}  $. Then, the matrix $T_{u,v}$ has
the following entries:

\begin{itemize}
\item Its $\left(  u,u\right)  $-th and $\left(  v,v\right)  $-th entries are
$0$.

\item All its other diagonal entries are $1$.

\item Its $\left(  u,v\right)  $-th and $\left(  v,u\right)  $-th entries are
$1$.

\item All its remaining entries are $0$.
\end{itemize}
\end{proposition}

\begin{proof}
[Proof of Proposition \ref{prop.Tuv.entries}.]For each $p\in\left\{
1,2,\ldots,n\right\}  $ and $q\in\left\{  1,2,\ldots,n\right\}  $, the matrix
$E_{p,q}$ is the $n\times n$-matrix whose $\left(  p,q\right)  $-th entry is
$1$ and whose all other entries are $0$ (indeed, this is how $E_{p,q}$ was
defined). Hence, we obtain the following two facts:

\begin{statement}
\textit{Fact 1:} Adding $E_{p,q}$ to an $n\times n$-matrix $C$ has the effect
that the $\left(  p,q\right)  $-th entry of $C$ is increased by $1$ (while all
other entries remain unchanged).
\end{statement}

\begin{statement}
\textit{Fact 2:} Subtracting $E_{p,q}$ from an $n\times n$-matrix $C$ has the
effect that the $\left(  p,q\right)  $-th entry of $C$ is decreased by $1$
(while all other entries remain unchanged).
\end{statement}

But recall that $T_{u,v}=I_{n}-E_{u,u}-E_{v,v}+E_{u,v}+E_{v,u}$. Thus, the
matrix $T_{u,v}$ is obtained from the matrix $I_{n}$ by first subtracting
$E_{u,u}$, then subtracting $E_{v,v}$, then adding $E_{u,v}$, and then adding
$E_{v,u}$. Using Fact 1 and Fact 2, we can see how these subtractions and
additions affect the entries of a matrix; thus, we can find all entries of the
matrix $T_{u,v}=I_{n}-E_{u,u}-E_{v,v}+E_{u,v}+E_{v,u}$:

\begin{itemize}
\item The matrix $I_{n}$ is the $n\times n$-matrix whose diagonal
entries\footnote{This includes the $\left(  u,u\right)  $-th entry.} are $1$,
and whose all other entries are $0$.

\item Subtracting $E_{u,u}$ from $I_{n}$ has the effect that the $\left(
u,u\right)  $-th entry is decreased by $1$; thus, it becomes $1-1=0$ (because
it was $1$ in $I_{n}$). Hence, the $\left(  u,u\right)  $-th entry of the
matrix $I_{n}-E_{u,u}$ is $0$, all its other diagonal entries are $1$, and all
its remaining entries are $0$.

\item Subtracting $E_{v,v}$ from $I_{n}-E_{u,u}$ has the effect that the
$\left(  v,v\right)  $-th entry is decreased by $1$; thus, it becomes $1-1=0$
(because it was $1$ in $I_{n}-E_{u,u}$). Hence, the $\left(  u,u\right)  $-th
and $\left(  v,v\right)  $-th entries of the matrix $I_{n}-E_{u,u}-E_{v,v}$
are $0$, all its other diagonal entries are $1$, and all its remaining entries
are $0$.

\item Adding $E_{u,v}$ to $I_{n}-E_{u,u}-E_{v,v}$ has the effect that the
$\left(  u,v\right)  $-th entry is increased by $1$; thus, it becomes $0+1=1$
(because it was $0$ in $I_{n}-E_{u,u}-E_{v,v}$). Hence, the $\left(
u,u\right)  $-th and $\left(  v,v\right)  $-th entries of the matrix
$I_{n}-E_{u,u}-E_{v,v}+E_{u,v}$ are $0$, all its other diagonal entries are
$1$, its $\left(  u,v\right)  $-th entry is $1$, and all its remaining entries
are $0$.

\item Adding $E_{v,u}$ to $I_{n}-E_{u,u}-E_{v,v}+E_{u,v}$ has the effect that
the $\left(  v,u\right)  $-th entry is increased by $1$; thus, it becomes
$0+1=1$ (because it was $0$ in $I_{n}-E_{u,u}-E_{v,v}+E_{u,v}$). Hence, the
$\left(  u,u\right)  $-th and $\left(  v,v\right)  $-th entries of the matrix
$I_{n}-E_{u,u}-E_{v,v}+E_{u,v}+E_{v,u}$ are $0$, all its other diagonal
entries are $1$, its $\left(  u,v\right)  $-th and $\left(  v,u\right)  $-th
entries are $1$, and all its remaining entries are $0$. Since $I_{n}%
-E_{u,u}-E_{v,v}+E_{u,v}+E_{v,u}=T_{u,v}$, this rewrites as follows: The
$\left(  u,u\right)  $-th and $\left(  v,v\right)  $-th entries of the matrix
$T_{u,v}$ are $0$, all its other diagonal entries are $1$, its $\left(
u,v\right)  $-th and $\left(  v,u\right)  $-th entries are $1$, and all its
remaining entries are $0$. This proves Proposition \ref{prop.Tuv.entries}.
\end{itemize}
\end{proof}

We can next see what happens to a matrix when it is multiplied by $T_{u,v}$:

\begin{proposition}
\label{prop.Tuv.laction}Let $n\in\mathbb{N}$ and $m\in\mathbb{N}$. Let $u$ and
$v$ be two distinct elements of $\left\{  1,2,\ldots,n\right\}  $. Let $C$ be
an $n\times m$-matrix. Then, $T_{u,v}C$ is the $n\times m$-matrix obtained
from $C$ by swapping the $u$-th row with the $v$-th row.
\end{proposition}

\begin{example}
\label{exam.prop.Tuv.laction}Let $n=3$ and $m=2$. Let $C$ be the $3\times
2$-matrix $\left(
\begin{array}
[c]{cc}%
a & b\\
a^{\prime} & b^{\prime}\\
a^{\prime\prime} & b^{\prime\prime}%
\end{array}
\right)  $. Then, Proposition \ref{prop.Tuv.laction} (applied to $u=1$ and
$v=3$) claims that $T_{1,3}C$ is the $3\times2$-matrix obtained from $C$ by
swapping the $1$-st row with the $3$-rd row. A computation confirms this
claim:%
\[
T_{1,3}C=\left(
\begin{array}
[c]{ccc}%
0 & 0 & 1\\
0 & 1 & 0\\
1 & 0 & 0
\end{array}
\right)  \left(
\begin{array}
[c]{cc}%
a & b\\
a^{\prime} & b^{\prime}\\
a^{\prime\prime} & b^{\prime\prime}%
\end{array}
\right)  =\left(
\begin{array}
[c]{cc}%
a^{\prime\prime} & b^{\prime\prime}\\
a^{\prime} & b^{\prime}\\
a & b
\end{array}
\right)  .
\]

\end{example}

\begin{proposition}
\label{prop.Tuv.raction}Let $n\in\mathbb{N}$ and $m\in\mathbb{N}$. Let $u$ and
$v$ be two distinct elements of $\left\{  1,2,\ldots,n\right\}  $. Let $C$ be
an $m\times n$-matrix. Then, $CT_{u,v}$ is the $m\times n$-matrix obtained
from $C$ by swapping the $u$-th column with the $v$-th column.
\end{proposition}

\begin{corollary}
\label{cor.Tuv.as-switch}Let $n\in\mathbb{N}$. Let $u$ and $v$ be two distinct
elements of $\left\{  1,2,\ldots,n\right\}  $. Then:

\textbf{(a)} The matrix $T_{u,v}$ can be obtained from $I_{n}$ by swapping the
$u$-th row with the $v$-th row.

\textbf{(b)} The matrix $T_{u,v}$ can be obtained from $I_{n}$ by swapping the
$u$-th column with the $v$-th column.
\end{corollary}

I will prove Proposition \ref{prop.Tuv.laction} and Corollary
\ref{cor.Tuv.as-switch} in Section \ref{sect.gauss.Tuv.proofs} below.

We shall refer to the matrix $T_{u,v}$ defined in Definition \ref{def.Tuv} as
a \textquotedblleft swapping matrix\textquotedblright.\footnote{The letter $T$
in \textquotedblleft$T_{u,v}$\textquotedblright\ stands for \textquotedblleft
transposition\textquotedblright, but this is not really related to the
transpose of a matrix. It is instead due to the fact that the word
\textquotedblleft transposition\textquotedblright\ means \textquotedblleft
changing the positions (of something)\textquotedblright, and in mathematics is
often used for swapping two things.} It is the third type of elementary matrices.

Here are a few more properties of swapping matrices:

\begin{proposition}
\label{prop.Tuv.transpose}Let $n\in\mathbb{N}$. Let $u$ and $v$ be two
elements of $\left\{  1,2,\ldots,n\right\}  $. Then, $\left(  T_{u,v}\right)
^{T}=T_{u,v}$.
\end{proposition}

\begin{proposition}
\label{prop.Tuv.lambda+mu}Let $n\in\mathbb{N}$.

\textbf{(a)} Let $u$ be an element of $\left\{  1,2,\ldots,n\right\}  $. Then,
$T_{u,u}=I_{n}$.

\textbf{(b)} Let $u$ and $v$ be two elements of $\left\{  1,2,\ldots
,n\right\}  $. Then, the matrix $T_{u,v}$ is invertible, and its inverse is
$\left(  T_{u,v}\right)  ^{-1}=T_{u,v}$.

\textbf{(c)} Let $u$ and $v$ be two elements of $\left\{  1,2,\ldots
,n\right\}  $. Then, $T_{v,u}=T_{u,v}$.
\end{proposition}

These facts will be proven in Section \ref{sect.gauss.Tuv.proofs}.

\subsection{\label{sect.gauss.Tuv.proofs}(*) Some proofs about the swapping
matrices}

\begin{proof}
[Proof of Proposition \ref{prop.Tuv.laction}.]Clearly, $T_{u,v}C$ is an
$n\times m$-matrix.

Let $x$ and $y$ be two elements of $\left\{  1,2,\ldots,n\right\}  $.
Proposition \ref{prop.Euv.laction} (applied to $n$, $m$, $x$ and $y$ instead
of $m$, $p$, $u$ and $v$) shows that $E_{x,y}C$ is the $n\times m$-matrix
whose $x$-th row is the $y$-th row of $C$, and whose all other rows are filled
with zeroes. Thus,%
\begin{equation}
\operatorname*{row}\nolimits_{x}\left(  E_{x,y}C\right)  =\operatorname*{row}%
\nolimits_{y}C \label{pf.prop.Tuv.laction.1}%
\end{equation}
(since the $x$-the row of $E_{x,y}C$ is the $y$-th row of $C$) and%
\begin{equation}
\operatorname*{row}\nolimits_{i}\left(  E_{x,y}C\right)  =0_{1\times
m}\ \ \ \ \ \ \ \ \ \ \text{for every }i\in\left\{  1,2,\ldots,n\right\}
\text{ satisfying }i\neq x \label{pf.prop.Tuv.laction.2}%
\end{equation}
(since all other rows of $E_{x,y}C$ are filled with zeroes).

Now, forget that we fixed $x$ and $y$. We thus have proven
(\ref{pf.prop.Tuv.laction.1}) and (\ref{pf.prop.Tuv.laction.2}) for every two
elements $x$ and $y$ of $\left\{  1,2,\ldots,n\right\}  $. In particular,
every $y\in\left\{  1,2,\ldots,n\right\}  $ satisfies%
\begin{equation}
\operatorname*{row}\nolimits_{u}\left(  E_{v,y}C\right)  =0_{1\times m}
\label{pf.prop.Tuv.laction.2a}%
\end{equation}
(by (\ref{pf.prop.Tuv.laction.2}), applied to $i=u$ and $x=v$ (since $u\neq
v$)) and%
\begin{equation}
\operatorname*{row}\nolimits_{v}\left(  E_{u,y}C\right)  =0_{1\times m}
\label{pf.prop.Tuv.laction.2b}%
\end{equation}
(by (\ref{pf.prop.Tuv.laction.2}), applied to $i=v$ and $x=u$ (since $v\neq u$)).

But recall that matrices are added entry by entry. Thus, matrices are also
added by row by row -- i.e., if $U$ and $V$ are two $n\times m$-matrices, then
any row of $U+V$ is the sum of the corresponding rows of $U$ and of $V$. In
other words, if $U$ and $V$ are two $n\times m$-matrices, then%
\begin{equation}
\operatorname*{row}\nolimits_{i}\left(  U+V\right)  =\operatorname*{row}%
\nolimits_{i}U+\operatorname*{row}\nolimits_{i}V\ \ \ \ \ \ \ \ \ \ \text{for
every }i\in\left\{  1,2,\ldots,n\right\}  . \label{pf.prop.Tuv.laction.5}%
\end{equation}
Similarly,%
\begin{equation}
\operatorname*{row}\nolimits_{i}\left(  U-V\right)  =\operatorname*{row}%
\nolimits_{i}U-\operatorname*{row}\nolimits_{i}V\ \ \ \ \ \ \ \ \ \ \text{for
every }i\in\left\{  1,2,\ldots,n\right\}  . \label{pf.prop.Tuv.laction.6}%
\end{equation}
Using (\ref{pf.prop.Tuv.laction.5}) and (\ref{pf.prop.Tuv.laction.6})
repeatedly, we can show that%
\begin{equation}
\operatorname*{row}\nolimits_{i}\left(  U-V-W+X+Y\right)  =\operatorname*{row}%
\nolimits_{i}U-\operatorname*{row}\nolimits_{i}V-\operatorname*{row}%
\nolimits_{i}W+\operatorname*{row}\nolimits_{i}X+\operatorname*{row}%
\nolimits_{i}Y \label{pf.prop.Tuv.laction.7}%
\end{equation}
for every $i\in\left\{  1,2,\ldots,n\right\}  $.

We have%
\begin{align*}
\underbrace{T_{u,v}}_{=I_{n}-E_{u,u}-E_{v,v}+E_{u,v}+E_{v,u}}C  &  =\left(
I_{n}-E_{u,u}-E_{v,v}+E_{u,v}+E_{v,u}\right)  C\\
&  =\underbrace{I_{n}C}_{=C}-\,E_{u,u}C-E_{v,v}C+E_{u,v}C+E_{v,u}C\\
&  =C-E_{u,u}C-E_{v,v}C+E_{u,v}C+E_{v,u}C.
\end{align*}
Hence, for each $i\in\left\{  1,2,\ldots,n\right\}  $, we have%
\begin{align}
&  \operatorname*{row}\nolimits_{i}\left(  \underbrace{T_{u,v}C}%
_{=C-E_{u,u}C-E_{v,v}C+E_{u,v}C+E_{v,u}C}\right) \nonumber\\
&  =\operatorname*{row}\nolimits_{i}\left(  C-E_{u,u}C-E_{v,v}C+E_{u,v}%
C+E_{v,u}C\right) \nonumber\\
&  =\operatorname*{row}\nolimits_{i}C-\operatorname*{row}\nolimits_{i}\left(
E_{u,u}C\right)  -\operatorname*{row}\nolimits_{i}\left(  E_{v,v}C\right)
+\operatorname*{row}\nolimits_{i}\left(  E_{u,v}C\right)  +\operatorname*{row}%
\nolimits_{i}\left(  E_{v,u}C\right)  \label{pf.prop.Tuv.laction.9}%
\end{align}
(by (\ref{pf.prop.Tuv.laction.7}), applied to $U=C$, $V=E_{u,u}C$,
$W=E_{v,v}C$, $X=E_{u,v}C$ and $Y=E_{v,u}C$).

Now, we must prove that $T_{u,v}C$ is the $n\times m$-matrix obtained from $C$
by swapping the $u$-th row with the $v$-th row. In other words, we must prove
the following three claims:

\begin{statement}
\textit{Claim 1:} The $u$-th row of the $n\times m$-matrix $T_{u,v}C$ equals
the $v$-th row of $C$.
\end{statement}

\begin{statement}
\textit{Claim 2:} The $v$-th row of the $n\times m$-matrix $T_{u,v}C$ equals
the $u$-th row of $C$.
\end{statement}

\begin{statement}
\textit{Claim 3:} For each $i\in\left\{  1,2,\ldots,n\right\}  $ satisfying
$i\neq u$ and $i\neq v$, the $i$-th row of the $n\times m$-matrix $T_{u,v}C$
equals the $i$-th row of $C$.
\end{statement}

\textit{Proof of Claim 1:} The $u$-th row of the $n\times m$-matrix $T_{u,v}C$
is%
\begin{align*}
&  \operatorname*{row}\nolimits_{u}\left(  T_{u,v}C\right) \\
&  =\operatorname*{row}\nolimits_{u}C-\underbrace{\operatorname*{row}%
\nolimits_{u}\left(  E_{u,u}C\right)  }_{\substack{=\operatorname*{row}%
\nolimits_{u}C\\\text{(by (\ref{pf.prop.Tuv.laction.1}), applied
to}\\x=u\text{ and }y=u\text{)}}}-\underbrace{\operatorname*{row}%
\nolimits_{u}\left(  E_{v,v}C\right)  }_{\substack{=0_{1\times m}\\\text{(by
(\ref{pf.prop.Tuv.laction.2a}),}\\\text{applied to }y=v\text{)}}%
}+\underbrace{\operatorname*{row}\nolimits_{u}\left(  E_{u,v}C\right)
}_{\substack{=\operatorname*{row}\nolimits_{v}C\\\text{(by
(\ref{pf.prop.Tuv.laction.1}), applied to}\\x=u\text{ and }y=v\text{)}%
}}+\underbrace{\operatorname*{row}\nolimits_{u}\left(  E_{v,u}C\right)
}_{\substack{=0_{1\times m}\\\text{(by (\ref{pf.prop.Tuv.laction.2a}%
),}\\\text{applied to }y=u\text{)}}}\\
&  \ \ \ \ \ \ \ \ \ \ \left(  \text{by (\ref{pf.prop.Tuv.laction.9}), applied
to }i=u\right) \\
&  =\operatorname*{row}\nolimits_{u}C-\operatorname*{row}\nolimits_{u}%
C-0_{1\times m}+\operatorname*{row}\nolimits_{v}C+0_{1\times m}%
=\operatorname*{row}\nolimits_{v}C.
\end{align*}
In other words, the $u$-th row of the $n\times m$-matrix $T_{u,v}C$ equals the
$v$-th row of $C$. This proves Claim 1.

\textit{Proof of Claim 2:} The $v$-th row of the $n\times m$-matrix $T_{u,v}C$
is%
\begin{align*}
&  \operatorname*{row}\nolimits_{v}\left(  T_{u,v}C\right) \\
&  =\operatorname*{row}\nolimits_{v}C-\underbrace{\operatorname*{row}%
\nolimits_{v}\left(  E_{u,u}C\right)  }_{\substack{=0_{1\times m}\\\text{(by
(\ref{pf.prop.Tuv.laction.2b}),}\\\text{applied to }y=u\text{)}}%
}-\underbrace{\operatorname*{row}\nolimits_{v}\left(  E_{v,v}C\right)
}+_{\substack{=\operatorname*{row}\nolimits_{v}C\\\text{(by
(\ref{pf.prop.Tuv.laction.1}), applied to}\\x=v\text{ and }y=v\text{)}%
}}+\underbrace{\operatorname*{row}\nolimits_{v}\left(  E_{u,v}C\right)
}_{\substack{=0_{1\times m}\\\text{(by (\ref{pf.prop.Tuv.laction.2b}%
),}\\\text{applied to }y=v\text{)}}}+\underbrace{\operatorname*{row}%
\nolimits_{v}\left(  E_{v,u}C\right)  }_{\substack{=\operatorname*{row}%
\nolimits_{u}C\\\text{(by (\ref{pf.prop.Tuv.laction.1}), applied
to}\\x=v\text{ and }y=u\text{)}}}\\
&  \ \ \ \ \ \ \ \ \ \ \left(  \text{by (\ref{pf.prop.Tuv.laction.9}), applied
to }i=v\right) \\
&  =\operatorname*{row}\nolimits_{v}C-0_{1\times m}-\operatorname*{row}%
\nolimits_{v}C+0_{1\times m}+\operatorname*{row}\nolimits_{u}%
C=\operatorname*{row}\nolimits_{u}C.
\end{align*}
In other words, the $v$-th row of the $n\times m$-matrix $T_{u,v}C$ equals the
$u$-th row of $C$. This proves Claim 2.

\textit{Proof of Claim 3:} Let $i\in\left\{  1,2,\ldots,n\right\}  $ be such
that $i\neq u$ and $i\neq v$. Then, the $i$-th row of the $n\times m$-matrix
$T_{u,v}C$ is%
\begin{align*}
&  \operatorname*{row}\nolimits_{i}\left(  T_{u,v}C\right) \\
&  =\operatorname*{row}\nolimits_{i}C-\underbrace{\operatorname*{row}%
\nolimits_{i}\left(  E_{u,u}C\right)  }_{\substack{=0_{1\times m}\\\text{(by
(\ref{pf.prop.Tuv.laction.2}), applied to}\\x=u\text{ and }y=u\text{)}%
}}-\underbrace{\operatorname*{row}\nolimits_{i}\left(  E_{v,v}C\right)
}_{\substack{=0_{1\times m}\\\text{(by (\ref{pf.prop.Tuv.laction.2}), applied
to}\\x=v\text{ and }y=v\text{)}}}+\underbrace{\operatorname*{row}%
\nolimits_{i}\left(  E_{u,v}C\right)  }_{\substack{=0_{1\times m}\\\text{(by
(\ref{pf.prop.Tuv.laction.2}), applied to}\\x=u\text{ and }y=v\text{)}%
}}+\underbrace{\operatorname*{row}\nolimits_{v}\left(  E_{v,u}C\right)
}_{\substack{=0_{1\times m}\\\text{(by (\ref{pf.prop.Tuv.laction.2}), applied
to}\\x=v\text{ and }y=u\text{)}}}\\
&  \ \ \ \ \ \ \ \ \ \ \left(  \text{by (\ref{pf.prop.Tuv.laction.9})}\right)
\\
&  =\operatorname*{row}\nolimits_{i}C-0_{1\times m}-0_{1\times m}+0_{1\times
m}+0_{1\times m}=\operatorname*{row}\nolimits_{i}C.
\end{align*}
In other words, the $i$-th row of the $n\times m$-matrix $T_{u,v}C$ equals the
$i$-th row of $C$. This proves Claim 3.

Now, we have proven Claim 1, Claim 2 and Claim 3; this completes the proof of
Proposition \ref{prop.Tuv.laction}.
\end{proof}

The proof of Proposition \ref{prop.Tuv.raction} is analogous.

\begin{proof}
[Proof of Corollary \ref{cor.Tuv.as-switch}.]\textbf{(a)} Proposition
\ref{prop.Tuv.laction} (applied to $m=n$ and $C=I_{n}$) shows that
$T_{u,v}I_{n}$ is the $n\times n$-matrix obtained from $I_{n}$ by swapping the
$u$-th row with the $v$-th row. In other words, $T_{u,v}$ is the $n\times
n$-matrix obtained from $I_{n}$ by swapping the $u$-th row with the $v$-th row
(since $T_{u,v}I_{n}=T_{u,v}$). This proves Corollary \ref{cor.Tuv.as-switch}
\textbf{(a)}.

\textbf{(b)} The proof of Corollary \ref{cor.Tuv.as-switch} \textbf{(b)} is
similar, except that now we have to use Proposition \ref{prop.Tuv.raction}.
\end{proof}

\begin{proof}
[Proof of Proposition \ref{prop.Tuv.transpose}.]Proposition
\ref{prop.transpose.opers} \textbf{(d)} shows that any two $n\times
m$-matrices $A$ and $B$ (where $m\in\mathbb{N}$) satisfy%
\begin{equation}
\left(  A+B\right)  ^{T}=A^{T}+B^{T}. \label{pf.prop.Tuv.transpose.1}%
\end{equation}
Similarly, any two $n\times m$-matrices $A$ and $B$ (where $m\in\mathbb{N}$)
satisfy%
\begin{equation}
\left(  A-B\right)  ^{T}=A^{T}-B^{T}. \label{pf.prop.Tuv.transpose.2}%
\end{equation}
By repeated application of (\ref{pf.prop.Tuv.transpose.1}) and
(\ref{pf.prop.Tuv.transpose.2}), we can show the following fact: If $A$, $B$,
$C$, $D$ and $E$ are five $n\times m$-matrices (for some $m\in\mathbb{N}$),
then%
\begin{equation}
\left(  A-B-C+D+E\right)  ^{T}=A^{T}-B^{T}-C^{T}+D^{T}+E^{T}.
\label{pf.prop.Tuv.transpose.3}%
\end{equation}


The definition of $T_{u,v}$ yields $T_{u,v}=I_{n}-E_{u,u}-E_{v,v}%
+E_{u,v}+E_{v,u}$. Hence,%
\begin{align*}
\left(  T_{u,v}\right)  ^{T}  &  =\left(  I_{n}-E_{u,u}-E_{v,v}+E_{u,v}%
+E_{v,u}\right)  ^{T}\\
&  =\underbrace{\left(  I_{n}\right)  ^{T}}_{\substack{=I_{n}\\\text{(by
Proposition \ref{prop.transpose.opers} \textbf{(a)})}}}-\underbrace{\left(
E_{u,u}\right)  ^{T}}_{\substack{=E_{u,u}\\\text{(by Proposition
\ref{prop.Euv.transpose},}\\\text{applied to }n\text{, }u\text{ and
}u\\\text{instead of }m\text{, }u\text{ and }v\text{)}}}-\underbrace{\left(
E_{v,v}\right)  ^{T}}_{\substack{=E_{v,v}\\\text{(by Proposition
\ref{prop.Euv.transpose},}\\\text{applied to }n\text{, }v\text{ and
}v\\\text{instead of }m\text{, }u\text{ and }v\text{)}}}\\
&  \ \ \ \ \ \ \ \ \ \ +\underbrace{\left(  E_{u,v}\right)  ^{T}%
}_{\substack{=E_{v,u}\\\text{(by Proposition \ref{prop.Euv.transpose}%
,}\\\text{applied to }m=n\text{)}}}+\underbrace{\left(  E_{v,u}\right)  ^{T}%
}_{\substack{=E_{u,v}\\\text{(by Proposition \ref{prop.Euv.transpose}%
,}\\\text{applied to }n\text{, }v\text{ and }u\\\text{instead of }m\text{,
}u\text{ and }v\text{)}}}\\
&  \ \ \ \ \ \ \ \ \ \ \left(
\begin{array}
[c]{c}%
\text{by (\ref{pf.prop.Tuv.transpose.3}), applied to }m=n\text{, }%
A=I_{n}\text{, }B=E_{u,u}\text{,}\\
C=E_{v,v}\text{, }D=E_{u,v}\text{ and }E=E_{v,u}%
\end{array}
\right) \\
&  =I_{n}-E_{u,u}-E_{v,v}+E_{v,u}+E_{u,v}\\
&  =I_{n}-E_{u,u}-E_{v,v}+E_{u,v}+E_{v,u}=T_{u,v}.
\end{align*}
This proves Proposition \ref{prop.Tuv.transpose}.
\end{proof}

\begin{proof}
[Proof of Proposition \ref{prop.Tuv.lambda+mu}.]\textbf{(a)} The definition of
$T_{u,u}$ yields $T_{u,u}=I_{n}-E_{u,u}-E_{u,u}+E_{u,u}+E_{u,u}=I_{n}$. This
proves Proposition \ref{prop.Tuv.lambda+mu} \textbf{(a)}.

\textbf{(b)} \textit{First proof:} If $v=u$, then Proposition
\ref{prop.Tuv.lambda+mu} \textbf{(a)} yields $T_{u,v}=T_{u,u}=I_{n}$ and
$T_{v,u}=T_{u,u}=I_{n}$, which renders the claim of Proposition
\ref{prop.Tuv.lambda+mu} \textbf{(b)} obvious in this case (since the matrix
$I_{n}$ is invertible with inverse $I_{n}^{-1}=I_{n}$). Thus, it only remains
to consider the case when $v\neq u$. So let us assume that $v\neq u$. In other
words, $u$ and $v$ are distinct.

Hence, Corollary \ref{cor.Tuv.as-switch} \textbf{(a)} shows that $T_{u,v}$ is
the $n\times n$-matrix obtained from $I_{n}$ by swapping the $u$-th row with
the $v$-th row. Proposition \ref{prop.Tuv.laction} (applied to $m=n$ and
$C=T_{u,v}$) shows that $T_{u,v}T_{u,v}$ is the $n\times n$-matrix obtained
from $T_{u,v}$ by swapping the $u$-th row with the $v$-th row. Thus, in order
to obtain the matrix $T_{u,v}T_{u,v}$ from $I_{n}$, we have to do the
following procedure:

\begin{itemize}
\item swap the $u$-th row with the $v$-th row;

\item then, \textbf{again} swap the $u$-th row with the $v$-th row.
\end{itemize}

But this procedure clearly returns the matrix $I_{n}$ with which we started
(since the second swap undoes the first swap). Thus, $T_{u,v}T_{u,v}=I_{n}$.

From $T_{u,v}T_{u,v}=I_{n}$ and $T_{u,v}T_{u,v}=I_{n}$ (yes, this is one
equality repeated twice), it follows that the matrix $T_{u,v}$ is an inverse
of $T_{u,v}$. Hence, the matrix $T_{u,v}$ is invertible, and its inverse is
$\left(  T_{u,v}\right)  ^{-1}=T_{u,v}$. This proves Proposition
\ref{prop.Tuv.lambda+mu} \textbf{(b)}.

\textit{Second proof:} Just as we did in the first proof, we restrict
ourselves to the case when $v\neq u$, since the case $v=u$ is easy. Thus,
$u\neq v$.

We can also prove Proposition \ref{prop.Tuv.lambda+mu} \textbf{(b)} using
Proposition \ref{prop.Euv.prod}, provided that we can tolerate some rather
lengthy computations.

If $i$, $j$, $x$ and $y$ are four elements of $\left\{  1,2,\ldots,n\right\}
$, then
\begin{equation}
E_{i,j}E_{x,y}=\delta_{j,x}E_{i,y} \label{pf.prop.Tuv.lambda+mu.2nd.1}%
\end{equation}
(where all three matrices $E_{i,j}$, $E_{x,y}$ and $E_{i,y}$ have to be
understood as $n\times n$-matrices)\footnote{\textit{Proof of
(\ref{pf.prop.Tuv.lambda+mu.2nd.1}):} Let $i$, $j$, $x$ and $y$ be four
elements of $\left\{  1,2,\ldots,n\right\}  $. Then, Proposition
\ref{prop.Euv.prod} (applied to $n$, $n$, $i$ and $j$ instead of $m$, $p$, $u$
and $v$) shows that $E_{i,j,n,n}E_{x,y,n,n}=\delta_{j,x}E_{i,y,n,n}$. Since we
abbreviate the matrices $E_{i,j,n,n}$, $E_{x,y,n,n}$ and $E_{i,y,n,n}$ as
$E_{i,j}$, $E_{x,y}$ and $E_{i,y}$ (respectively), this can be rewritten as
$E_{i,j}E_{x,y}=\delta_{j,x}E_{i,y}$. Thus, (\ref{pf.prop.Tuv.lambda+mu.2nd.1}%
) is proven.}. In particular, every $i\in\left\{  1,2,\ldots,n\right\}  $ and
$y\in\left\{  1,2,\ldots,n\right\}  $ satisfy%
\begin{equation}
E_{i,u}E_{v,y}=0_{n\times n} \label{pf.prop.Tuv.lambda+mu.2nd.2}%
\end{equation}
\footnote{\textit{Proof of (\ref{pf.prop.Tuv.lambda+mu.2nd.2}):} Let
$i\in\left\{  1,2,\ldots,n\right\}  $ and $y\in\left\{  1,2,\ldots,n\right\}
$. Then, (\ref{pf.prop.Tuv.lambda+mu.2nd.1}) (applied to $j=u$ and $x=v$)
yields $E_{i,u}E_{v,y}=\underbrace{\delta_{u,v}}_{\substack{=0\\\text{(since
}u\neq v\text{)}}}E_{i,y}=0E_{i,y}=0_{n\times n}$, qed.} and%
\begin{equation}
E_{i,v}E_{u,y}=0_{n\times n} \label{pf.prop.Tuv.lambda+mu.2nd.3}%
\end{equation}
\footnote{\textit{Proof of (\ref{pf.prop.Tuv.lambda+mu.2nd.3}):} Let
$i\in\left\{  1,2,\ldots,n\right\}  $ and $y\in\left\{  1,2,\ldots,n\right\}
$. Then, (\ref{pf.prop.Tuv.lambda+mu.2nd.1}) (applied to $j=v$ and $x=u$)
yields $E_{i,v}E_{u,y}=\underbrace{\delta_{v,u}}_{\substack{=0\\\text{(since
}v\neq u\text{)}}}E_{i,y}=0E_{i,y}=0_{n\times n}$, qed.}. Furthermore, if $i$,
$x$ and $y$ are three elements of $\left\{  1,2,\ldots,n\right\}  $, then%
\begin{equation}
E_{i,x}E_{x,y}=E_{i,y} \label{pf.prop.Tuv.lambda+mu.2nd.4}%
\end{equation}
\footnote{\textit{Proof of (\ref{pf.prop.Tuv.lambda+mu.2nd.4}):} Let $i$, $j$
and $x$ be three elements of $\left\{  1,2,\ldots,n\right\}  $. Then,
(\ref{pf.prop.Tuv.lambda+mu.2nd.1}) (applied to $j=x$) yields $E_{i,x}%
E_{x,y}=\underbrace{\delta_{x,x}}_{\substack{=1\\\text{(since }x=x\text{)}%
}}E_{i,y}=1E_{i,y}=E_{i,y}$, qed.}.

Now, the definition of $T_{u,v}$ yields $T_{u,v}=I_{n}-E_{u,u}-E_{v,v}%
+E_{u,v}+E_{v,u}$. Hence,%
\begin{align*}
&  \underbrace{T_{u,v}}_{=I_{n}-E_{u,u}-E_{v,v}+E_{u,v}+E_{v,u}}%
\underbrace{T_{u,v}}_{=I_{n}-E_{u,u}-E_{v,v}+E_{u,v}+E_{v,u}}\\
&  =\left(  I_{n}-E_{u,u}-E_{v,v}+E_{u,v}+E_{v,u}\right)  \left(
I_{n}-E_{u,u}-E_{v,v}+E_{u,v}+E_{v,u}\right) \\
&  =\underbrace{I_{n}\left(  I_{n}-E_{u,u}-E_{v,v}+E_{u,v}+E_{v,u}\right)
}_{=I_{n}I_{n}-I_{n}E_{u,u}-I_{n}E_{v,v}+I_{n}E_{u,v}+I_{n}E_{v,u}}\\
&  \ \ \ \ \ \ \ \ \ \ -\underbrace{E_{u,u}\left(  I_{n}-E_{u,u}%
-E_{v,v}+E_{u,v}+E_{v,u}\right)  }_{=E_{u,u}I_{n}-E_{u,u}E_{u,u}%
-E_{u,u}E_{v,v}+E_{u,u}E_{u,v}+E_{u,u}E_{v,u}}\\
&  \ \ \ \ \ \ \ \ \ \ -\underbrace{E_{v,v}\left(  I_{n}-E_{u,u}%
-E_{v,v}+E_{u,v}+E_{v,u}\right)  }_{=E_{v,v}I_{n}-E_{v,v}E_{u,u}%
-E_{v,v}E_{v,v}+E_{v,v}E_{u,v}+E_{v,v}E_{v,u}}\\
&  \ \ \ \ \ \ \ \ \ \ +\underbrace{E_{u,v}\left(  I_{n}-E_{u,u}%
-E_{v,v}+E_{u,v}+E_{v,u}\right)  }_{=E_{u,v}I_{n}-E_{u,v}E_{u,u}%
-E_{u,v}E_{v,v}+E_{u,v}E_{u,v}+E_{u,v}E_{v,u}}\\
&  \ \ \ \ \ \ \ \ \ \ +\underbrace{E_{v,u}\left(  I_{n}-E_{u,u}%
-E_{v,v}+E_{u,v}+E_{v,u}\right)  }_{=E_{v,u}I_{n}-E_{v,u}E_{u,u}%
-E_{v,u}E_{v,v}+E_{v,u}E_{u,v}+E_{v,u}E_{v,u}}%
\end{align*}%
\begin{align*}
&  =\left(  \underbrace{I_{n}I_{n}}_{=I_{n}}-\underbrace{I_{n}E_{u,u}%
}_{=E_{u,u}}-\underbrace{I_{n}E_{v,v}}_{=E_{v,v}}+\underbrace{I_{n}E_{u,v}%
}_{=E_{u,v}}+\underbrace{I_{n}E_{v,u}}_{=E_{v,u}}\right) \\
&  \ \ \ \ \ \ \ \ \ \ -\left(  \underbrace{E_{u,u}I_{n}}_{=E_{u,u}%
}-\underbrace{E_{u,u}E_{u,u}}_{\substack{=E_{u,u}\\\text{(by
(\ref{pf.prop.Tuv.lambda+mu.2nd.4}), applied}\\\text{to }i=u\text{, }x=u\text{
and }y=u\text{)}}}-\underbrace{E_{u,u}E_{v,v}}_{\substack{=0_{n\times
n}\\\text{(by (\ref{pf.prop.Tuv.lambda+mu.2nd.2}), applied}\\\text{to
}i=u\text{ and }y=v\text{)}}}+\underbrace{E_{u,u}E_{u,v}}_{\substack{=E_{u,v}%
\\\text{(by (\ref{pf.prop.Tuv.lambda+mu.2nd.4}), applied}\\\text{to
}i=u\text{, }x=u\text{ and }y=v\text{)}}}+\underbrace{E_{u,u}E_{v,u}%
}_{\substack{=0_{n\times n}\\\text{(by (\ref{pf.prop.Tuv.lambda+mu.2nd.2}),
applied}\\\text{to }i=u\text{ and }y=u\text{)}}}\right) \\
&  \ \ \ \ \ \ \ \ \ \ -\left(  \underbrace{E_{v,v}I_{n}}_{=E_{v,v}%
}-\underbrace{E_{v,v}E_{u,u}}_{\substack{=0_{n\times n}\\\text{(by
(\ref{pf.prop.Tuv.lambda+mu.2nd.3}), applied}\\\text{to }i=v\text{ and
}y=u\text{)}}}-\underbrace{E_{v,v}E_{v,v}}_{\substack{=E_{v,v}\\\text{(by
(\ref{pf.prop.Tuv.lambda+mu.2nd.4}), applied}\\\text{to }i=v\text{, }x=v\text{
and }y=v\text{)}}}+\underbrace{E_{v,v}E_{u,v}}_{\substack{=0_{n\times
n}\\\text{(by (\ref{pf.prop.Tuv.lambda+mu.2nd.3}), applied}\\\text{to
}i=v\text{ and }y=v\text{)}}}+\underbrace{E_{v,v}E_{v,u}}_{\substack{=E_{v,u}%
\\\text{(by (\ref{pf.prop.Tuv.lambda+mu.2nd.4}), applied}\\\text{to
}i=v\text{, }x=v\text{ and }y=u\text{)}}}\right) \\
&  \ \ \ \ \ \ \ \ \ \ +\left(  \underbrace{E_{u,v}I_{n}}_{=E_{u,v}%
}-\underbrace{E_{u,v}E_{u,u}}_{\substack{=0_{n\times n}\\\text{(by
(\ref{pf.prop.Tuv.lambda+mu.2nd.3}), applied}\\\text{to }i=u\text{ and
}y=u\text{)}}}-\underbrace{E_{u,v}E_{v,v}}_{\substack{=E_{u,v}\\\text{(by
(\ref{pf.prop.Tuv.lambda+mu.2nd.4}), applied}\\\text{to }i=u\text{, }x=v\text{
and }y=v\text{)}}}+\underbrace{E_{u,v}E_{u,v}}_{\substack{=0_{n\times
n}\\\text{(by (\ref{pf.prop.Tuv.lambda+mu.2nd.3}), applied}\\\text{to
}i=u\text{ and }y=v\text{)}}}+\underbrace{E_{u,v}E_{v,u}}_{\substack{=E_{u,u}%
\\\text{(by (\ref{pf.prop.Tuv.lambda+mu.2nd.4}), applied}\\\text{to
}i=u\text{, }x=v\text{ and }y=u\text{)}}}\right) \\
&  \ \ \ \ \ \ \ \ \ \ +\left(  \underbrace{E_{v,u}I_{n}}_{=E_{v,u}%
}-\underbrace{E_{v,u}E_{u,u}}_{\substack{=E_{v,u}\\\text{(by
(\ref{pf.prop.Tuv.lambda+mu.2nd.4}), applied}\\\text{to }i=v\text{, }x=u\text{
and }y=u\text{)}}}-\underbrace{E_{v,u}E_{v,v}}_{\substack{=0_{n\times
n}\\\text{(by (\ref{pf.prop.Tuv.lambda+mu.2nd.2}), applied}\\\text{to
}i=v\text{ and }y=v\text{)}}}+\underbrace{E_{v,u}E_{u,v}}_{\substack{=E_{v,v}%
\\\text{(by (\ref{pf.prop.Tuv.lambda+mu.2nd.4}), applied}\\\text{to
}i=v\text{, }x=u\text{ and }y=v\text{)}}}+\underbrace{E_{v,u}E_{v,u}%
}_{\substack{=0_{n\times n}\\\text{(by (\ref{pf.prop.Tuv.lambda+mu.2nd.2}),
applied}\\\text{to }i=v\text{ and }y=u\text{)}}}\right) \\
&  =\left(  I_{n}-E_{u,u}-E_{v,v}+E_{u,v}+E_{v,u}\right)  -\left(
E_{u,u}-E_{u,u}-0_{n\times n}+E_{u,v}+0_{n\times n}\right) \\
&  \ \ \ \ \ \ \ \ \ \ -\left(  E_{v,v}-0_{n\times n}-E_{v,v}+0_{n\times
n}+E_{v,u}\right)  +\left(  E_{u,v}-0_{n\times n}-E_{u,v}+0_{n\times
n}+E_{u,u}\right) \\
&  \ \ \ \ \ \ \ \ \ \ +\left(  E_{v,u}-E_{v,u}-0_{n\times n}+E_{v,v}%
+0_{n\times n}\right) \\
&  =I_{n}.
\end{align*}


From $T_{u,v}T_{u,v}=I_{n}$ and $T_{u,v}T_{u,v}=I_{n}$ (yes, this is one
equality repeated twice), it follows that the matrix $T_{u,v}$ is an inverse
of $T_{u,v}$. Hence, the matrix $T_{u,v}$ is invertible, and its inverse is
$\left(  T_{u,v}\right)  ^{-1}=T_{u,v}$. This proves Proposition
\ref{prop.Tuv.lambda+mu} \textbf{(b)} again.

\textbf{(c)} The definition of $T_{v,u}$ yields $T_{v,u}=I_{n}-E_{v,v}%
-E_{u,u}+E_{v,u}+E_{u,v}=I_{n}-E_{u,u}-E_{v,v}+E_{u,v}+E_{v,u}$. Comparing
this with $T_{u,v}=I_{n}-E_{u,u}-E_{v,v}+E_{u,v}+E_{v,u}$, we obtain
$T_{v,u}=T_{u,v}$. This proves Proposition \ref{prop.Tuv.lambda+mu}
\textbf{(c)}.
\end{proof}

\subsection{Permutation matrices}

\begin{definition}
\label{def.Tuv.swapping}Let $n\in\mathbb{N}$. A \textit{swapping }$n\times
n$\textit{-matrix} means a matrix of the form $T_{u,v}$, where $u$ and $v$ are
two distinct elements of $\left\{  1,2,\ldots,n\right\}  $. When $n$ is clear
from the context, we shall omit the \textquotedblleft$n\times n$%
-\textquotedblright\ and simply say \textquotedblleft swapping
matrix\textquotedblright.
\end{definition}

In Theorem \ref{thm.triangular.Alamuv}, we have characterized
lower-unitriangular matrices as products of lower addition matrices.
Similarly, in Theorem \ref{thm.triangular.Slamu}, we have characterized
invertibly lower-triangular matrices as products of scaling matrices and lower
addition matrices. What kind of matrices are characterized as products of
swapping matrices $T_{u,v}$ ? We should not expect anything with
\textquotedblleft triangular\textquotedblright\ in its name (after all, the
matrices $T_{u,v}$ themselves are not triangular). Instead, we obtain the
so-called permutation matrices.

\begin{definition}
\label{def.permat}Let $n\in\mathbb{N}$. An $n\times n$-matrix $A$ is said to
be a \textit{permutation matrix} if it satisfies the following conditions:

\textbf{(a)} Each entry of $A$ is either a $0$ or a $1$.

\textbf{(b)} Each row of $A$ has exactly one entry equal to $1$.

\textbf{(c)} Each column of $A$ has exactly one entry equal to $1$.
\end{definition}

\begin{example}
\label{exam.permat}\textbf{(a)} The $3\times3$-matrix $\left(
\begin{array}
[c]{ccc}%
0 & 1 & 0\\
0 & 0 & 1\\
1 & 0 & 0
\end{array}
\right)  $ is a permutation matrix.

\textbf{(b)} The $3\times3$-matrix $\left(
\begin{array}
[c]{ccc}%
0 & 1 & 0\\
0 & 0 & 3\\
1 & 0 & 0
\end{array}
\right)  $ is not a permutation matrix, since it fails condition \textbf{(a)}
of Definition \ref{def.permat}.

\textbf{(c)} The $3\times3$-matrix $\left(
\begin{array}
[c]{ccc}%
0 & 1 & 0\\
0 & 0 & 1\\
0 & 0 & 1
\end{array}
\right)  $ is not a permutation matrix, since it fails condition \textbf{(c)}
of Definition \ref{def.permat} (namely, the $3$-rd column has two entries
equal to $1$, whereas the $1$-st column has none).

\textbf{(d)} The $3\times3$-matrix $\left(
\begin{array}
[c]{ccc}%
0 & 0 & 0\\
0 & 1 & 1\\
1 & 0 & 0
\end{array}
\right)  $ is not a permutation matrix, since it fails condition \textbf{(b)}
of Definition \ref{def.permat} (namely, the $2$-nd row has two entries equal
to $1$, whereas the $1$-st row has none).

\textbf{(e)} For each $n\in\mathbb{N}$, the $n\times n$ identity matrix
$I_{n}$ is a permutation matrix. (This is Lemma \ref{lem.permat.Tuv.In}
further below.)

\textbf{(f)} Let $n\in\mathbb{N}$. Let $u$ and $v$ be two distinct elements of
$\left\{  1,2,\ldots,n\right\}  $. Then, the swapping matrix $T_{u,v}$
(defined in Definition \ref{def.Tuv}) is a permutation matrix. (This is a
particular case of Lemma \ref{lem.permat.Tuv.prod-swap} below.)
\end{example}

These examples do not exhaust the set of all permutation matrices. However
(unlike, e.g., the lower-triangular matrices), this set is finite for each
$n\in\mathbb{N}$. More precisely:

\begin{proposition}
\label{prop.permat.n!}Let $n\in\mathbb{N}$. Then, there are precisely $n!$
permutation matrices of size $n\times n$.

(Recall that $n!$ denotes the number $1\cdot2\cdot3\cdot\cdots\cdot n$; it is
called the \textquotedblleft factorial of $n$\textquotedblright. For instance,
$5!=1\cdot2\cdot3\cdot4\cdot5=120$.)
\end{proposition}

We will outline a proof of Proposition \ref{prop.permat.n!} in Section
\ref{sect.gauss.permat.perms}.

\begin{example}
Proposition \ref{prop.permat.n!} (applied to $n=3$) says that there are
precisely $3!=6$ permutation matrices of size $3\times3$. Here are they:%
\begin{align*}
I_{3}  &  =\left(
\begin{array}
[c]{ccc}%
1 & 0 & 0\\
0 & 1 & 0\\
0 & 0 & 1
\end{array}
\right)  ,\ \ \ \ \ \ \ \ \ \ T_{1,2}=\left(
\begin{array}
[c]{ccc}%
0 & 1 & 0\\
1 & 0 & 0\\
0 & 0 & 1
\end{array}
\right)  ,\\
T_{1,3}  &  =\left(
\begin{array}
[c]{ccc}%
0 & 0 & 1\\
0 & 1 & 0\\
1 & 0 & 0
\end{array}
\right)  ,\ \ \ \ \ \ \ \ \ \ T_{2,3}=\left(
\begin{array}
[c]{ccc}%
1 & 0 & 0\\
0 & 0 & 1\\
0 & 1 & 0
\end{array}
\right)  ,\\
A  &  =\left(
\begin{array}
[c]{ccc}%
0 & 0 & 1\\
1 & 0 & 0\\
0 & 1 & 0
\end{array}
\right)  ,\ \ \ \ \ \ \ \ \ \ B=\left(
\begin{array}
[c]{ccc}%
0 & 1 & 0\\
0 & 0 & 1\\
1 & 0 & 0
\end{array}
\right)  .
\end{align*}
The two last matrices, which I have here denoted by $A$ and $B$, are neither
the identity matrix $I_{3}$ nor swapping matrices $T_{u,v}$. However, they can
be written as products of swapping matrices:%
\[
A=T_{1,2}T_{2,3},\ \ \ \ \ \ \ \ \ \ B=T_{2,3}T_{1,2}.
\]
(Of course, they can also be written as products of swapping matrices in many
other ways. For instance, $A=T_{1,3}T_{1,2}=T_{2,3}T_{2,3}T_{1,2}%
T_{2,3}=T_{2,3}T_{1,2}T_{2,3}T_{1,2}$.) This is not a coincidence: As we will
see shortly (in Theorem \ref{thm.permat.Tuv}), the permutation matrices are
precisely the products of swapping matrices.
\end{example}

\begin{theorem}
\label{thm.permat.Tuv}Let $n\in\mathbb{N}$. An $n\times n$-matrix $C$ is a
permutation matrix if and only if $C$ is a product of swapping matrices.
\end{theorem}

Some authors (e.g., Olver and Shakiban in \cite[Chapter 1, Definition
1.8]{OlvSha06}) use Theorem \ref{thm.permat.Tuv} as a definition of
permutation matrices. (I.e., they \textbf{define} permutation matrices as
products of swapping matrices, instead of using Definition \ref{def.permat}.)
More precisely, the following equivalent definitions of permutation matrices exist:

\begin{itemize}
\item Our Definition \ref{def.permat} above.

\item An $n\times n$-matrix is called a \textit{permutation matrix} if it is a
product of swapping matrices. (This is the definition used in \cite[Chapter 1,
Definition 1.8]{OlvSha06}; and Theorem \ref{thm.permat.Tuv} reveals that it is
equivalent to our Definition \ref{def.permat}.)

\item An $n\times n$-matrix is called a \textit{permutation matrix} if it has
the same rows as $I_{n}$ but (possibly) in a different order.

\item An $n\times n$-matrix is called a \textit{permutation matrix} if it has
the same columns as $I_{n}$ but (possibly) in a different order.

\item An $n\times n$-matrix is called a \textit{permutation matrix} if it has
the form $\left(  \delta_{w\left(  i\right)  ,j}\right)  _{1\leq i\leq
n,\ 1\leq j\leq n}$ for some bijective map $w:\left\{  1,2,\ldots,n\right\}
\rightarrow\left\{  1,2,\ldots,n\right\}  $. (This is the definition used in
Stanley's \cite[\S 1.5]{Stanley-EC1}. We will discuss it in some more detail
in Section \ref{sect.gauss.permat.perms}.)
\end{itemize}

The equivalence of all these definitions is easy to see (once Theorem
\ref{thm.permat.Tuv} is proven); nevertheless, we shall be using Definition
\ref{def.permat} only.

We shall give a complete proof of Theorem \ref{thm.permat.Tuv} in Section
\ref{sect.gauss.permat.proofs}; but first, let us state some basic facts on
which said proof relies:

\begin{lemma}
\label{lem.permat.Tuv.1}Let $n\in\mathbb{N}$. Let $A$ be an $n\times
n$-matrix. Assume that $A$ is a permutation matrix. Let $u$ and $v$ be two
distinct elements of $\left\{  1,2,\ldots,n\right\}  $. Let $B$ be the matrix
obtained from $A$ by swapping the $u$-th row with the $v$-th row. Then, $B$ is
a permutation matrix.
\end{lemma}

\begin{proof}
[Proof of Lemma \ref{lem.permat.Tuv.1}.]We know that $A$ is a permutation
matrix. According to the definition of a \textquotedblleft permutation
matrix\textquotedblright, this means that $A$ satisfies the following three statements:

\begin{statement}
\textit{Statement 1:} Each entry of $A$ is either a $0$ or a $1$.
\end{statement}

\begin{statement}
\textit{Statement 2:} Each row of $A$ has exactly one entry equal to $1$.
\end{statement}

\begin{statement}
\textit{Statement 3:} Each column of $A$ has exactly one entry equal to $1$.
\end{statement}

Thus, we know that Statements 1, 2 and 3 are satisfied.

On the other hand, we want to prove that $B$ is a permutation matrix.
According to the definition of a \textquotedblleft permutation
matrix\textquotedblright, this means proving that $B$ satisfies the following
three statements:

\begin{statement}
\textit{Statement 4:} Each entry of $B$ is either a $0$ or a $1$.
\end{statement}

\begin{statement}
\textit{Statement 5:} Each row of $B$ has exactly one entry equal to $1$.
\end{statement}

\begin{statement}
\textit{Statement 6:} Each column of $B$ has exactly one entry equal to $1$.
\end{statement}

Hence, it remains to prove that Statements 4, 5 and 6 are satisfied.

Recall that the matrix $B$ is obtained from $A$ by swapping the $u$-th row
with the $v$-th row. Hence, each row of $B$ equals some row of $A$. Thus,
Statement 5 follows from Statement 2. Therefore, Statement 5 is satisfied.

Also, each entry of $B$ equals some entry of $A$ (since each row of $B$ equals
some row of $A$). Thus, Statement 4 follows from Statement 1. Hence, Statement
4 is satisfied.

Recall again that the matrix $B$ is obtained from $A$ by swapping the $u$-th
row with the $v$-th row. Hence, each column of $B$ is obtained from the
corresponding column of $A$ by swapping the $u$-th entry with the $v$-th
entry. Hence, each column of $B$ has exactly as many entries equal to $1$ as
the corresponding column of $A$ (because swapping two entries does not change
the number of entries equal to $1$). Therefore, Statement 6 follows from
Statement 3. Hence, Statement 6 is satisfied.

We thus have shown that Statements 4, 5 and 6 are satisfied. As we have said,
this completes the proof of Lemma \ref{lem.permat.Tuv.1}.
\end{proof}

\begin{lemma}
\label{lem.permat.Tuv.In}Let $n\in\mathbb{N}$. Then, the identity matrix
$I_{n}$ is a permutation matrix.
\end{lemma}

Lemma \ref{lem.permat.Tuv.In} is very easy to prove (and will be proven in
detail in Section \ref{sect.gauss.permat.proofs} below).

\begin{lemma}
\label{lem.permat.Tuv.prod-swap}Let $n\in\mathbb{N}$. Then, any product of
swapping $n\times n$-matrices is a permutation matrix.

(Here, we are again using the convention that the empty product of $n\times
n$-matrices is $I_{n}$.)
\end{lemma}

Lemma \ref{lem.permat.Tuv.prod-swap} is, of course, the $\Longleftarrow$
direction of Theorem \ref{thm.permat.Tuv}; it can be proven by induction using
Lemma \ref{lem.permat.Tuv.1} and Proposition \ref{prop.Tuv.laction}. The
(rather straightforward) proof can be found in Section
\ref{sect.gauss.permat.proofs} below.

Now, let me give some examples for Theorem \ref{thm.permat.Tuv}:

\begin{example}
\label{exam.thm.permat.Tuv}\textbf{(a)} For each $n\in\mathbb{N}$, the
$n\times n$ identity matrix $I_{n}$ is a product of swapping matrices: Namely,
it is the empty product (since the empty product of $n\times n$-matrices is
$I_{n}$ by definition).

\textbf{(b)} Each swapping matrix $T_{u,v}$ itself is a product of swapping
matrices: namely, it is a product of itself.

\textbf{(c)} Let $C$ be the permutation matrix $\left(
\begin{array}
[c]{cccc}%
0 & 0 & 1 & 0\\
1 & 0 & 0 & 0\\
0 & 0 & 0 & 1\\
0 & 1 & 0 & 0
\end{array}
\right)  $. Then, $C$ is a product of swapping matrices: Namely, it equals
$T_{1,2}T_{2,4}T_{3,4}$.

Let us actually see how this representation of $C$ can be found. We shall
proceed by writing $C$ as a product of one swapping matrix with a second
matrix $C^{\prime}$, which is still a permutation matrix but has one diagonal
entry equal to $1$. We then will do the same with $C^{\prime}$, obtaining a
third matrix $C^{\prime\prime}$; then, do the same with $C^{\prime\prime}$,
and so on. At the end, we will be left with a permutation matrix whose
\textbf{all} diagonal entries are $1$. This means that we will be left with
the identity matrix $I_{4}$.

In more detail: We proceed in several steps:

\begin{description}
\item[Step 1:] Let us turn the $\left(  1,1\right)  $-th entry of $C$ into $1$
by swapping the $1$-st row with the $2$-nd row. Denote the resulting matrix by
$C^{\prime}$. Thus, $C^{\prime}=\left(
\begin{array}
[c]{cccc}%
1 & 0 & 0 & 0\\
0 & 0 & 1 & 0\\
0 & 0 & 0 & 1\\
0 & 1 & 0 & 0
\end{array}
\right)  $. Notice that the new matrix $C^{\prime}$ is still a permutation
matrix (by Lemma \ref{lem.permat.Tuv.1}, applied to $A=C$, $B=C^{\prime}$,
$u=1$ and $v=2$), but now has its first diagonal entry equal to $1$. Since
$C^{\prime}$ was obtained from $C$ by swapping the $1$-st row with the $2$-nd
row, we can conversely obtain $C$ from $C^{\prime}$ by swapping the $1$-st row
with the $2$-nd row. According to Proposition \ref{prop.Tuv.laction} (applied
to $n$, $1$, $2$ and $C^{\prime}$ instead of $m$, $u$, $v$ and $C$), this
means that $C=T_{1,2}C^{\prime}$.

\item[Step 2:] Let us turn the $\left(  2,2\right)  $-th entry of $C^{\prime}$
into $1$ by swapping the $2$-nd row with the $4$-th row. Denote the resulting
matrix by $C^{\prime\prime}$. Thus, $C^{\prime\prime}=\left(
\begin{array}
[c]{cccc}%
1 & 0 & 0 & 0\\
0 & 1 & 0 & 0\\
0 & 0 & 0 & 1\\
0 & 0 & 1 & 0
\end{array}
\right)  $. Again, the new matrix $C^{\prime\prime}$ is still a permutation
matrix (by Lemma \ref{lem.permat.Tuv.1}, applied to $A=C^{\prime}$,
$B=C^{\prime\prime}$, $u=2$ and $v=4$), and still has its first diagonal entry
equal to $1$ (since the $1$-st row has not been changed); but now its second
diagonal entry is also $1$. Similarly to how we found that $C=T_{1,2}%
C^{\prime}$ in Step 1, we now obtain $C^{\prime}=T_{2,4}C^{\prime\prime}$.

\item[Step 3:] Let us turn the $\left(  3,3\right)  $-th entry of
$C^{\prime\prime}$ into $1$ by swapping the $3$-rd row with the $4$-th row.
Denote the resulting matrix by $C^{\prime\prime\prime}$. Thus, $C^{\prime
\prime\prime}=\left(
\begin{array}
[c]{cccc}%
1 & 0 & 0 & 0\\
0 & 1 & 0 & 0\\
0 & 0 & 1 & 0\\
0 & 0 & 0 & 1
\end{array}
\right)  $. Again, the new matrix $C^{\prime\prime\prime}$ is still a
permutation matrix, and the first two diagonal entries are still equal to $1$;
and now the third diagonal entry has become $1$ as well. Similarly to how we
found that $C=T_{1,2}C^{\prime}$ in Step 1, we now obtain $C^{\prime\prime
}=T_{3,4}C^{\prime\prime\prime}$.

\item[Step 4:] We should now turn the $\left(  4,4\right)  $-th entry of
$C^{\prime\prime\prime}$ into $1$, but fortunately this is unnecessary: It
already is $1$.
\end{description}

We have thus turned all diagonal entries into $1$. Our final matrix
$C^{\prime\prime\prime}$ thus equals $I_{4}$ (since it is a permutation
matrix). Combining the three equalities we have found, we obtain%
\begin{align*}
C  &  =T_{1,2}\underbrace{C^{\prime}}_{=T_{2,4}C^{\prime\prime}}%
=T_{1,2}T_{2,4}\underbrace{C^{\prime\prime}}_{=T_{3,4}C^{\prime\prime\prime}%
}=T_{1,2}T_{2,4}T_{3,4}\underbrace{C^{\prime\prime\prime}}_{=I_{4}}\\
&  =T_{1,2}T_{2,4}T_{3,4}I_{4}=T_{1,2}T_{2,4}T_{3,4}.
\end{align*}
Thus we have represented $C$ as a product of swapping matrices.
\end{example}

Example \ref{exam.thm.permat.Tuv} \textbf{(c)} essentially demonstrates how
Theorem \ref{thm.permat.Tuv} (or, more precisely, the $\Longrightarrow$
direction of Theorem \ref{thm.permat.Tuv}) can be proven in the general case
(similarly to how Example \ref{exam.thm.triangular.Alamuv} \textbf{(c)}
outlines the proof of Theorem \ref{thm.triangular.Alamuv}, and how Example
\ref{exam.thm.triangular.Slamu} \textbf{(c)} outlines the proof of Theorem
\ref{thm.triangular.Slamu}). Transforming the example into an actual rigorous
proof, however, requires work: Not only would we have to formalize the
algorithm, but we would also need to formally justify that the algorithm
works\footnote{For example, we would need to verify that each step results in
a permutation matrix, and that the $k$-th step (for each $k$) leaves the first
$k-1$ diagonal entries unchanged.}. In Section \ref{sect.gauss.permat.proofs},
we shall give a proof of Theorem \ref{thm.permat.Tuv} which is, \textbf{more
or less}, the one suggested by Example \ref{exam.thm.permat.Tuv} \textbf{(c)};
however, it will be organized rather differently (for the sake of easier
readability)\footnote{Namely, instead of using a procedure with several steps,
it will be based on an induction argument. The ideas will, of course, be the
same; this is just an example of how algorithmic arguments can be rewritten as
induction proofs.}.

One easy corollary of Theorem \ref{thm.permat.Tuv} is the following:

\begin{proposition}
\label{prop.permat.AB}Let $n\in\mathbb{N}$. Let $A$ and $B$ be two $n\times
n$-matrices that are permutation matrices. Then, $AB$ is also a permutation matrix.
\end{proposition}

Furthermore, we can show:

\begin{proposition}
\label{prop.permat.inverse}Let $n\in\mathbb{N}$. Let $A$ be an $n\times
n$-matrix that is a permutation matrix. Then:

\textbf{(a)} The matrix $A$ is invertible.

\textbf{(b)} Its inverse is $A^{-1}=A^{T}$.

\textbf{(c)} This inverse $A^{-1}$ is a permutation matrix.
\end{proposition}

\begin{proposition}
\label{prop.permat.laction}Let $n\in\mathbb{N}$ and $m\in\mathbb{N}$. Let $P$
be an $n\times n$-matrix that is a permutation matrix. Let $C$ be an $n\times
m$-matrix. Then, the $n\times m$-matrix $PC$ can be obtained from $C$ by
rearranging the rows in a certain way that depends on $P$. (In more rigorous
terms, this means that there exists a bijective map $w:\left\{  1,2,\ldots
,n\right\}  \rightarrow\left\{  1,2,\ldots,n\right\}  $ such that every
$i\in\left\{  1,2,\ldots,n\right\}  $ satisfies $\operatorname*{row}%
\nolimits_{i}\left(  PC\right)  =\operatorname*{row}\nolimits_{w\left(
i\right)  }C$. If this sounds confusing to you, think of this map $w$ as a way
to match up the rows of $PC$ with the rows of $C$ such that each row of $PC$
equals the corresponding row of $C$. We will go over this in more detail in
Section \ref{sect.gauss.permat.perms}.)
\end{proposition}

\begin{example}
Let $n=3$ and $m=2$. Let $C$ be the $3\times2$-matrix $\left(
\begin{array}
[c]{cc}%
a & b\\
a^{\prime} & b^{\prime}\\
a^{\prime\prime} & b^{\prime\prime}%
\end{array}
\right)  $. Let $P$ be the $3\times3$-matrix $\left(
\begin{array}
[c]{ccc}%
0 & 1 & 0\\
0 & 0 & 1\\
1 & 0 & 0
\end{array}
\right)  $; as we know, this is a permutation matrix. Then, Proposition
\ref{prop.permat.laction} claims that the $3\times2$-matrix $PC$ can be
obtained from $C$ by rearranging the rows in a certain way. And this can
indeed be confirmed by a computation:%
\[
PC=\left(
\begin{array}
[c]{ccc}%
0 & 1 & 0\\
0 & 0 & 1\\
1 & 0 & 0
\end{array}
\right)  \left(
\begin{array}
[c]{cc}%
a & b\\
a^{\prime} & b^{\prime}\\
a^{\prime\prime} & b^{\prime\prime}%
\end{array}
\right)  =\left(
\begin{array}
[c]{cc}%
a^{\prime} & b^{\prime}\\
a^{\prime\prime} & b^{\prime\prime}\\
a & b
\end{array}
\right)  .
\]
The rearrangement moves the first row to the very bottom, while letting the
other two rows slide up one level. Other permutation matrices $P$ would
produce other rearrangements.
\end{example}

Proposition \ref{prop.permat.laction} is (in a sense) a partial generalization
of Proposition \ref{prop.Tuv.laction} (although, of course, not a complete
generalization, since it fails to specify the precise rearrangement). A
similar partial generalization can be stated for Proposition
\ref{prop.Tuv.raction}; this time, of course, it will be the columns (not the
rows) that get rearranged in $CP$.

We will prove Proposition \ref{prop.permat.laction} in Section
\ref{sect.gauss.permat.perms}.

\subsection{\label{sect.gauss.permat.proofs}(*) Proofs about permutation
matrices}

Let us now catch up on some proofs that we promised in the previous section.

\begin{proof}
[Proof of Lemma \ref{lem.permat.Tuv.In}.]We want to prove that $I_{n}$ is a
permutation matrix. According to the definition of a \textquotedblleft
permutation matrix\textquotedblright, this means proving that $I_{n}$
satisfies the following three statements:

\begin{statement}
\textit{Statement 1:} Each entry of $I_{n}$ is either a $0$ or a $1$.
\end{statement}

\begin{statement}
\textit{Statement 2:} Each row of $I_{n}$ has exactly one entry equal to $1$.
\end{statement}

\begin{statement}
\textit{Statement 3:} Each column of $I_{n}$ has exactly one entry equal to
$1$.
\end{statement}

Hence, it remains to prove that Statements 1, 2 and 3 are satisfied.

But Statement 1 is obvious. Statement 2 is also clear (since each row of
$I_{n}$ has exactly one entry equal to $1$ -- namely, the diagonal entry), and
Statement 3 is clear as well (for similar reasons). Thus, Statements 1, 2 and
3 are satisfied. This proves Lemma \ref{lem.permat.Tuv.In}.
\end{proof}

\begin{proof}
[Proof of Lemma \ref{lem.permat.Tuv.prod-swap}.]Let $M$ be any product of
swapping $n\times n$-matrices. We must show that $M$ is a permutation matrix.

We have assumed that $M$ is a product of swapping $n\times n$-matrices. In
other words, $M=A_{k}A_{k-1}\cdots A_{1}$ for some $k\in\mathbb{N}$ and some
$k$ swapping $n\times n$-matrices $A_{1},A_{2},\ldots,A_{k}$. Consider this
$k$ and these $A_{1},A_{2},\ldots,A_{k}$.

We shall show that%
\begin{equation}
A_{i}A_{i-1}\cdots A_{1}\text{ is a permutation matrix}
\label{pf.lem.permat.Tuv.prod-swap.goal}%
\end{equation}
for every $i\in\left\{  0,1,\ldots,k\right\}  $ (where, as usual, $A_{0}%
A_{-1}\cdots A_{1}$ has to be interpreted as an empty product and thus equals
$I_{n}$).

[\textit{Proof of (\ref{pf.lem.permat.Tuv.prod-swap.goal}):} We will prove
(\ref{pf.lem.permat.Tuv.prod-swap.goal}) by induction over $i$:

\textit{Induction base:} Lemma \ref{lem.permat.Tuv.In} says that $I_{n}$ is a
permutation matrix. In other words, $A_{0}A_{-1}\cdots A_{1}$ is a permutation
matrix\footnote{since $A_{0}A_{-1}\cdots A_{1}=\left(  \text{empty product of
}n\times n\text{-matrices}\right)  =I_{n}$}. In other words,
(\ref{pf.lem.permat.Tuv.prod-swap.goal}) holds for $i=0$. This completes the
induction base.

\textit{Induction step:} Let $j\in\left\{  0,1,\ldots,k\right\}  $ be
positive. Assume (as the \textit{induction hypothesis}) that
(\ref{pf.lem.permat.Tuv.prod-swap.goal}) holds for $i=j-1$. We must show that
(\ref{pf.lem.permat.Tuv.prod-swap.goal}) holds for $i=j$.

The induction hypothesis tells us that (\ref{pf.lem.permat.Tuv.prod-swap.goal}%
) holds for $i=j-1$. In other words, $A_{j-1}A_{j-2}\cdots A_{1}$ is a
permutation matrix. Set $C=A_{j-1}A_{j-2}\cdots A_{1}$. Thus, $C$ is a
permutation matrix (since $A_{j-1}A_{j-2}\cdots A_{1}$ is a permutation matrix).

But $A_{j}$ is a swapping $n\times n$-matrix (since $A_{1},A_{2},\ldots,A_{k}$
are $k$ swapping $n\times n$-matrices). In other words, $A_{j}$ has the form
$A_{j}=T_{u,v}$, where $u$ and $v$ are two distinct elements of $\left\{
1,2,\ldots,n\right\}  $. Consider these $u$ and $v$.

Now,%
\begin{equation}
A_{j}A_{j-1}\cdots A_{1}=\underbrace{A_{j}}_{=T_{u,v}}\underbrace{\left(
A_{j-1}A_{j-2}\cdots A_{1}\right)  }_{=C}=T_{u,v}C.
\label{pf.lem.permat.Tuv.prod-swap.goal.pf.1}%
\end{equation}


But Proposition \ref{prop.Tuv.laction} (applied to $m=n$) shows that
$T_{u,v}C$ is the $n\times n$-matrix obtained from $C$ by swapping the $u$-th
row with the $v$-th row. Hence, Lemma \ref{lem.permat.Tuv.1} (applied to $A=C$
and $B=T_{u,v}C$) shows that $T_{u,v}C$ is a permutation matrix. In light of
(\ref{pf.lem.permat.Tuv.prod-swap.goal.pf.1}), this rewrites as follows:
$A_{j}A_{j-1}\cdots A_{1}$ is a permutation matrix. In other words,
(\ref{pf.lem.permat.Tuv.prod-swap.goal}) holds for $i=j$. This completes the
induction step, and thus the inductive proof of
(\ref{pf.lem.permat.Tuv.prod-swap.goal}).]

Now, (\ref{pf.lem.permat.Tuv.prod-swap.goal}) (applied to $i=k$) yields that
$A_{k}A_{k-1}\cdots A_{1}$ is a permutation matrix. In other words, $M$ is a
permutation matrix (since $M=A_{k}A_{k-1}\cdots A_{1}$). This completes the
proof of Lemma \ref{lem.permat.Tuv.prod-swap}.
\end{proof}

The general proof of Theorem \ref{thm.permat.Tuv} follows the idea outlined in
Example \ref{exam.thm.permat.Tuv} \textbf{(c)}, but we are going to make it
more manageable by introducing a convenient notion:

\begin{definition}
\label{def.thm.permat.Tuv.k-id}Let $n\in\mathbb{N}$ and $k\in\left\{
0,1,\ldots,n\right\}  $. An $n\times n$-matrix $A$ is said to be
$k$\textit{-identical} if it satisfies $A_{1,1}=A_{2,2}=\cdots=A_{k,k}=1$.
(Note that the condition $A_{1,1}=A_{2,2}=\cdots=A_{k,k}=1$ means
\textquotedblleft$A_{i,i}=1$ for each $i\in\left\{  1,2,\ldots,k\right\}
$\textquotedblright. Thus, if $k=0$, then this condition is vacuously true,
since there exists no $i\in\left\{  1,2,\ldots,k\right\}  $ in this case.)
\end{definition}

This notion allows us to speak about our procedure from Example
\ref{exam.thm.permat.Tuv} \textbf{(c)} more crisply: We started with an
arbitrary permutation matrix $C$, which was $0$-identical. Then, in Step 1, we
made it $1$-identical by switching two rows. Then, in Step 2, we made it
$2$-identical by switching two further rows. Then, in Step 3, we made it
$3$-identical by switching two further rows. Then, in Step 4, we made it
$4$-identical by doing nothing (since it already was $4$-identical). At the
end of the procedure, it was an identity matrix.

Here are some properties of $k$-identical permutation matrices:

\begin{lemma}
\label{lem.permat.Tuv.k-id.1}Let $n\in\mathbb{N}$ and $k\in\left\{
0,1,\ldots,n\right\}  $. Let $A$ be an $n\times n$-matrix. Assume that $A$ is
a $k$-identical permutation matrix.

\textbf{(a)} If $k=n$, then $A=I_{n}$.

\textbf{(b)} For each $v\in\left\{  k+1,k+2,\ldots,n\right\}  $, there exists
some $u\in\left\{  k+1,k+2,\ldots,n\right\}  $ such that $A_{u,v}=1$.

\textbf{(c)} If $k<n$ and $A_{k+1,k+1}=1$, then $A$ is a $\left(  k+1\right)
$-identical permutation matrix.

\textbf{(d)} If $u$ and $v$ are two distinct elements of $\left\{
k+1,k+2,\ldots,n\right\}  $ satisfying $v=k+1$ and $A_{u,v}=1$, then
$T_{u,v}A$ is a $\left(  k+1\right)  $-identical permutation matrix.
\end{lemma}

\begin{proof}
[Proof of Lemma \ref{lem.permat.Tuv.k-id.1}.]We have assumed that $A$ is a
permutation matrix. According to the definition of a \textquotedblleft
permutation matrix\textquotedblright, this means that $A$ satisfies the
following three statements:

\begin{statement}
\textit{Statement 1:} Each entry of $A$ is either a $0$ or a $1$.
\end{statement}

\begin{statement}
\textit{Statement 2:} Each row of $A$ has exactly one entry equal to $1$.
\end{statement}

\begin{statement}
\textit{Statement 3:} Each column of $A$ has exactly one entry equal to $1$.
\end{statement}

Thus, we know that Statements 1, 2 and 3 are satisfied.

We have assumed that $A$ is $k$-identical. In other words,%
\begin{equation}
A_{1,1}=A_{2,2}=\cdots=A_{k,k}=1 \label{pf.lem.permat.Tuv.k-id.1.1}%
\end{equation}
(by the definition of \textquotedblleft$k$-identical\textquotedblright). In
other words,%
\begin{equation}
A_{i,i}=1\ \ \ \ \ \ \ \ \ \ \text{for each }i\in\left\{  1,2,\ldots
,k\right\}  . \label{pf.lem.permat.Tuv.k-id.1.1b}%
\end{equation}
Next, we observe that%
\begin{equation}
A_{i,j}=\delta_{i,j}\ \ \ \ \ \ \ \ \ \ \text{for all }i\in\left\{
1,2,\ldots,k\right\}  \text{ and }j\in\left\{  1,2,\ldots,n\right\}
\label{pf.lem.permat.Tuv.k-id.1.2}%
\end{equation}
\footnote{\textit{Proof of (\ref{pf.lem.permat.Tuv.k-id.1.2}):} Let
$i\in\left\{  1,2,\ldots,k\right\}  $ and $j\in\left\{  1,2,\ldots,n\right\}
$. We must prove that $A_{i,j}=\delta_{i,j}$.
\par
We have $i\in\left\{  1,2,\ldots,k\right\}  $. Hence, $A_{i,i}=1$ (by
(\ref{pf.lem.permat.Tuv.k-id.1.1b})).
\par
We are in one of the following two cases:
\par
\textit{Case 1:} We have $i=j$.
\par
\textit{Case 2:} We have $i\neq j$.
\par
Let us first consider Case 1. In this case, we have $i=j$. Hence, $j=i$, so
that $A_{i,j}=A_{i,i}=1$. Comparing this with $\delta_{i,j}=1$ (since $i=j$),
we obtain $A_{i,j}=\delta_{i,j}$. Hence, $A_{i,j}=\delta_{i,j}$ is proven in
Case 1.
\par
Let us now consider Case 2. In this case, we have $i\neq j$. Thus,
$\delta_{i,j}=0$.
\par
Now, assume (for the sake of contradiction) that $A_{i,j}\neq0$. But $A_{i,j}$
is an entry of $A$, and thus is either a $0$ or a $1$ (by Statement 1). In
other words, $A_{i,j}=0$ or $A_{i,j}=1$. Therefore, $A_{i,j}=1$ (since
$A_{i,j}\neq0$). Combining this with $A_{i,i}=1$, we conclude that the $i$-th
row of $A$ has at least two entries equal to $1$: namely, the entries
$A_{i,i}$ and $A_{i,j}$. (And these two entries actually lie in different
cells, since $i\neq j$.)
\par
But each row of $A$ has exactly one entry equal to $1$ (because Statement 2 is
satisfied). In particular, the $i$-th row of $A$ has exactly one entry equal
to $1$. This contradicts the fact that the $i$-th row of $A$ has at least two
entries equal to $1$. This contradiction shows that our assumption (that
$A_{i,j}\neq0$) was false. Hence, we have $A_{i,j}=0$. Compared with
$\delta_{i,j}=0$, this yields $A_{i,j}=\delta_{i,j}$. Thus, $A_{i,j}%
=\delta_{i,j}$ is proven in Case 2.
\par
Now, $A_{i,j}=\delta_{i,j}$ is proven in each of the two Cases 1 and 2. Hence,
$A_{i,j}=\delta_{i,j}$ always holds. In other words,
(\ref{pf.lem.permat.Tuv.k-id.1.2}) is proven.}.

\textbf{(a)} Assume that $k=n$. Then, every $i\in\left\{  1,2,\ldots
,n\right\}  $ and $j\in\left\{  1,2,\ldots,n\right\}  $ satisfy $A_{i,j}%
=\left(  I_{n}\right)  _{i,j}$\ \ \ \ \footnote{\textit{Proof.} Recall that
$I_{n}=\left(  \delta_{i,j}\right)  _{1\leq i\leq n,\ 1\leq j\leq n}$ (by the
definition of $I_{n}$). Hence, every $i\in\left\{  1,2,\ldots,n\right\}  $ and
$j\in\left\{  1,2,\ldots,n\right\}  $ satisfy%
\begin{equation}
\left(  I_{n}\right)  _{i,j}=\delta_{i,j}.
\label{pf.lem.permat.Tuv.k-id.1.fn5.1}%
\end{equation}
But $\left\{  1,2,\ldots,n\right\}  =\left\{  1,2,\ldots,k\right\}  $ (since
$n=k$). Now, every $i\in\left\{  1,2,\ldots,n\right\}  $ and $j\in\left\{
1,2,\ldots,n\right\}  $ satisfy%
\begin{align*}
A_{i,j}  &  =\delta_{i,j}\ \ \ \ \ \ \ \ \ \ \left(  \text{by
(\ref{pf.lem.permat.Tuv.k-id.1.2}), since }i\in\left\{  1,2,\ldots,n\right\}
=\left\{  1,2,\ldots,k\right\}  \right) \\
&  =\left(  I_{n}\right)  _{i,j}\ \ \ \ \ \ \ \ \ \ \left(  \text{by
(\ref{pf.lem.permat.Tuv.k-id.1.fn5.1})}\right)  .
\end{align*}
}. In other words, each entry of the matrix $A$ equals the corresponding entry
of $I_{n}$. In other words, $A=I_{n}$. This proves Lemma
\ref{lem.permat.Tuv.k-id.1} \textbf{(a)}.

\textbf{(b)} Let $v\in\left\{  k+1,k+2,\ldots,n\right\}  $. Statement 3 shows
that each column of $A$ has exactly one entry equal to $1$. In particular, the
$v$-th column has exactly one entry equal to $1$. In other words, there exists
exactly one $i\in\left\{  1,2,\ldots,n\right\}  $ such that $A_{i,v}=1$.
Consider this $i$. We are going to show that $i\in\left\{  k+1,k+2,\ldots
,n\right\}  $.

In fact, assume the contrary. Thus, $i\notin\left\{  k+1,k+2,\ldots,n\right\}
$. Combining $i\in\left\{  1,2,\ldots,n\right\}  $ with $i\notin\left\{
k+1,k+2,\ldots,n\right\}  $, we obtain%
\[
i\in\left\{  1,2,\ldots,n\right\}  \setminus\left\{  k+1,k+2,\ldots,n\right\}
=\left\{  1,2,\ldots,k\right\}  .
\]
Hence, $A_{i,v}=\delta_{i,v}$ (by (\ref{pf.lem.permat.Tuv.k-id.1.2}), applied
to $j=v$). But $i\in\left\{  1,2,\ldots,k\right\}  $, so that $i\leq k$.
However, $v\geq k+1$ (since $v\in\left\{  k+1,k+2,\ldots,n\right\}  $). Hence,
$k+1\leq v$, and thus $i\leq k<k+1\leq v$. Thus, $i\neq v$, so that
$\delta_{i,v}=0$ and thus $A_{i,v}=\delta_{i,v}=0$. This contradicts
$A_{i,v}=1\neq0$.

This contradiction shows that our assumption was false. Hence, $i\in\left\{
k+1,k+2,\ldots,n\right\}  $ is proven. Thus, there exists some $u\in\left\{
k+1,k+2,\ldots,n\right\}  $ such that $A_{u,v}=1$ (namely, $u=i$). This proves
Lemma \ref{lem.permat.Tuv.k-id.1} \textbf{(b)}.

\textbf{(c)} Assume that $k<n$ and $A_{k+1,k+1}=1$. We need to show that $A$
is a $\left(  k+1\right)  $-identical permutation matrix.

Combining (\ref{pf.lem.permat.Tuv.k-id.1.1}) with $A_{k+1,k+1}=1$, we obtain
$A_{1,1}=A_{2,2}=\cdots=A_{k+1,k+1}=1$.

But the matrix $A$ is $\left(  k+1\right)  $-identical if and only if
$A_{1,1}=A_{2,2}=\cdots=A_{k+1,k+1}=1$ (because this is how \textquotedblleft%
$\left(  k+1\right)  $-identical\textquotedblright\ is defined). Thus, the
matrix $A$ is $\left(  k+1\right)  $-identical (since $A_{1,1}=A_{2,2}%
=\cdots=A_{k+1,k+1}=1$). Also, $A$ is a permutation matrix. The proof of Lemma
\ref{lem.permat.Tuv.k-id.1} \textbf{(c)} is thus complete.

\textbf{(d)} Let $u$ and $v$ be two distinct elements of $\left\{
k+1,k+2,\ldots,n\right\}  $ satisfying $v=k+1$ and $A_{u,v}=1$. We must prove
that $T_{u,v}A$ is a $\left(  k+1\right)  $-identical permutation matrix.

Let $B=T_{u,v}A$. Proposition \ref{prop.Tuv.laction} (applied to $m=n$ and
$C=A$) shows that $T_{u,v}A$ is the $n\times n$-matrix obtained from $A$ by
swapping the $u$-th row with the $v$-th row. Since $B=T_{u,v}A$, this rewrites
as follows: $B$ is the $n\times n$-matrix obtained from $A$ by swapping the
$u$-th row with the $v$-th row. Hence, Lemma \ref{lem.permat.Tuv.1} shows that
$B$ is a permutation matrix. We shall next show that $B$ is $\left(
k+1\right)  $-identical.

Recall that $B$ is the $n\times n$-matrix obtained from $A$ by swapping the
$u$-th row with the $v$-th row. Hence, the following facts hold:

\begin{statement}
\textit{Fact 4:} The $u$-th row of the matrix $B$ equals the $v$-th row of $A$.
\end{statement}

\begin{statement}
\textit{Fact 5:} The $v$-th row of the matrix $B$ equals the $u$-th row of $A$.
\end{statement}

\begin{statement}
\textit{Fact 6:} If $i\in\left\{  1,2,\ldots,n\right\}  $ is such that $i\neq
u$ and $i\neq v$, then the $i$-th row of the matrix $B$ equals the $i$-th row
of $A$.
\end{statement}

Now, using Fact 6, we can easily see that $B_{i,i}=A_{i,i}$ for each
$i\in\left\{  1,2,\ldots,k\right\}  $\ \ \ \ \footnote{\textit{Proof.} Let
$i\in\left\{  1,2,\ldots,k\right\}  $. Then, $i\leq k<k+1\leq u$ (since $u\geq
k+1$ (since $u\in\left\{  k+1,k+2,\ldots,n\right\}  $)). Hence, $i\neq u$. The
same argument (but made for $v$ instead of $u$) shows that $i\neq v$. Hence,
Fact 6 shows that the $i$-th row of the matrix $B$ equals the $i$-th row of
$A$. In other words, each entry of the $i$-th row of the matrix $B$ equals the
corresponding entry of the $i$-th row of $A$. In other words, $B_{i,j}%
=A_{i,j}$ for each $j\in\left\{  1,2,\ldots,n\right\}  $. Applying this to
$j=i$, we obtain $B_{i,i}=A_{i,i}$. Qed.}. Hence, for each $i\in\left\{
1,2,\ldots,k\right\}  $, we have $B_{i,i}=A_{i,i}=1$ (by
(\ref{pf.lem.permat.Tuv.k-id.1.1b})). In other words,%
\begin{equation}
B_{1,1}=B_{2,2}=\cdots=B_{k,k}=1. \label{pf.lem.permat.Tuv.k-id.1.d.1}%
\end{equation}


But Fact 5 shows that each entry of the $v$-th row of the matrix $B$ equals
the corresponding entry of the $u$-th row of $A$. In other words,
$B_{v,j}=A_{u,j}$ for each $j\in\left\{  1,2,\ldots,n\right\}  $. Applying
this to $j=v$, we obtain $B_{v,v}=A_{u,v}=1$. Since $v=k+1$, this rewrites as
$B_{k+1,k+1}=1$. Combining this with (\ref{pf.lem.permat.Tuv.k-id.1.d.1}), we
obtain $B_{1,1}=B_{2,2}=\cdots=B_{k+1,k+1}=1$.

But the matrix $B$ is $\left(  k+1\right)  $-identical if and only if
$B_{1,1}=B_{2,2}=\cdots=B_{k+1,k+1}=1$ (because this is how \textquotedblleft%
$\left(  k+1\right)  $-identical\textquotedblright\ is defined). Thus, the
matrix $B$ is $\left(  k+1\right)  $-identical (since $B_{1,1}=B_{2,2}%
=\cdots=B_{k+1,k+1}=1$). Hence, $B$ is a $\left(  k+1\right)  $-identical
permutation matrix (since we already know that $B$ is a permutation matrix).
In other words, $T_{u,v}A$ is a $\left(  k+1\right)  $-identical permutation
matrix (since $B=T_{u,v}A$). This proves Lemma \ref{lem.permat.Tuv.k-id.1}
\textbf{(d)}.
\end{proof}

Next, we show a slightly stronger version of the $\Longrightarrow$ direction
of Theorem \ref{thm.permat.Tuv}:

\begin{lemma}
\label{lem.permat.Tuv.k-id.2}Let $n\in\mathbb{N}$ and $p\in\left\{
0,1,\ldots,n\right\}  $. Let $A$ be an $n\times n$-matrix. If $A$ is an
$\left(  n-p\right)  $-identical permutation matrix, then $A$ is a product of
at most $p$ swapping matrices.
\end{lemma}

\begin{proof}
[Proof of Lemma \ref{lem.permat.Tuv.k-id.2}.]We shall prove Lemma
\ref{lem.permat.Tuv.k-id.2} by induction over $p$:

\textit{Induction base:} If $A$ is an $\left(  n-0\right)  $-identical
permutation matrix, then $A$ is a product of at most $0$ swapping
matrices\footnote{\textit{Proof.} Assume that $A$ is an $\left(  n-0\right)
$-identical permutation matrix. In other words, $A$ is an $n$-identical
permutation matrix. Lemma \ref{lem.permat.Tuv.k-id.1} \textbf{(a)} (applied to
$k=n$) thus yields $A=I_{n}$. Thus $A=I_{n}=\left(  \text{the empty product of
swapping matrices}\right)  $ (since the empty product of swapping matrices is
defined to be $I_{n}$). Hence, $A$ is a product of $0$ swapping matrices.
Thus, $A$ is a product of at most $0$ swapping matrices. Qed.}. In other
words, Lemma \ref{lem.permat.Tuv.k-id.2} holds for $p=0$. This completes the
induction base.

\textit{Induction step:} Let $q \in\left\{  0,1,\ldots,n\right\}  $ be
positive. Assume (as the \textit{induction hypothesis}) that Lemma
\ref{lem.permat.Tuv.k-id.2} holds for $p=q-1$. We must now prove that Lemma
\ref{lem.permat.Tuv.k-id.2} holds for $p=q$.

Assume that $A$ is an $\left(  n-q\right)  $-identical permutation matrix. We
shall show that
\begin{equation}
A\text{ is a product of at most }q\text{ swapping matrices.}
\label{pf.lem.permat.Tuv.k-id.2.indgoal}%
\end{equation}


Set $k = n-q$. Thus, $A$ is a $k$-identical permutation matrix (since $A$ is
an $\left(  n-q\right)  $-identical permutation matrix).

We have $k=n-q\in\left\{  0,1,\ldots,n\right\}  $ (since $q\in\left\{
0,1,\ldots,n\right\}  $). Furthermore, $k=n-q<n$ (since $q$ is positive), so
that $k\leq n-1$ (since $k$ and $n$ are integers).

Set $v=k+1$. Then, $v=k+1\geq k+1$ and $v=k+1\leq n$ (since $k\leq n-1$), so
that $v\in\left\{  k+1,k+2,\ldots,n\right\}  $. Hence, Lemma
\ref{lem.permat.Tuv.k-id.1} \textbf{(b)} shows that there exists some
$u\in\left\{  k+1,k+2,\ldots,n\right\}  $ such that $A_{u,v}=1$. Consider this
$u$. We are in one of the following two cases:

\begin{statement}
\textit{Case 1:} We have $u=k+1$.
\end{statement}

\begin{statement}
\textit{Case 2:} We have $u\neq k+1$.
\end{statement}

Let us first consider Case 1. In this case, we have $u=k+1$. From $u=k+1$ and
$v=k+1$, we obtain $A_{u,v}=A_{k+1,k+1}$, so that $A_{k+1,k+1}=A_{u,v}=1$.
Hence, Lemma \ref{lem.permat.Tuv.k-id.1} \textbf{(c)} shows that $A$ is a
$\left(  k+1\right)  $-identical permutation matrix. Since $\underbrace{k}%
_{=n-q}+1=n-q+1=n-\left(  q-1\right)  $, this rewrites as follows: $A$ is an
$\left(  n-\left(  q-1\right)  \right)  $-identical permutation matrix. But
the induction hypothesis shows that Lemma \ref{lem.permat.Tuv.k-id.2} holds
for $p=q-1$. Hence, Lemma \ref{lem.permat.Tuv.k-id.2} can be applied to
$p=q-1$ (since $A$ is an $\left(  n-\left(  q-1\right)  \right)  $-identical
permutation matrix). As a result, we conclude that $A$ is a product of at most
$q-1$ swapping matrices. In other words, there exists some $s\in\left\{
0,1,\ldots,q-1\right\}  $ such that $A$ is a product of at most $s$ swapping
matrices. Consider this $s$. Thus, $A$ is a product of at most $q$ swapping
matrices (since $A$ is a product of $s$ swapping matrices, but we have
$s\in\left\{  0,1,\ldots,q-1\right\}  \subseteq\left\{  0,1,\ldots,q\right\}
$). In other words, (\ref{pf.lem.permat.Tuv.k-id.2.indgoal}) holds. Hence,
(\ref{pf.lem.permat.Tuv.k-id.2.indgoal}) is proven in Case 1.

Let us now consider Case 2. In this case, we have $u\neq k+1$. Thus, $u\neq
k+1=v$. Therefore, Lemma \ref{lem.permat.Tuv.k-id.1} \textbf{(d)} reveals that
$T_{u,v}A$ is a $\left(  k+1\right)  $-identical permutation matrix. Since
$\underbrace{k}_{=n-q}+1=n-q+1=n-\left(  q-1\right)  $, this rewrites as
follows: $T_{u,v}A$ is an $\left(  n-\left(  q-1\right)  \right)  $-identical
permutation matrix. But the induction hypothesis shows that Lemma
\ref{lem.permat.Tuv.k-id.2} holds for $p=q-1$. Hence, Lemma
\ref{lem.permat.Tuv.k-id.2} can be applied to $q-1$ and $T_{u,v}A$ instead of
$p$ and $A$ (since $T_{u,v}A$ is an $\left(  n-\left(  q-1\right)  \right)
$-identical permutation matrix). As a result, we conclude that $T_{u,v}A$ is a
product of at most $q-1$ swapping matrices. In other words, there exists some
$s\in\left\{  0,1,\ldots,q-1\right\}  $ and some $s$ swapping matrices
$M_{1},M_{2},\ldots,M_{s}$ such that $T_{u,v}A=M_{1}M_{2}\cdots M_{s}$.
Consider this $s$ and these $M_{1},M_{2},\ldots,M_{s}$.

From $s\in\left\{  0,1,\ldots,q-1\right\}  $, we obtain $s+1\in\left\{
1,2,\ldots,q\right\}  \subseteq\left\{  0,1,\ldots,q\right\}  $.

Proposition \ref{prop.Tuv.lambda+mu} \textbf{(b)} yields that the matrix
$T_{u,v}$ is invertible, and its inverse is $\left(  T_{u,v}\right)
^{-1}=T_{u,v}$. Thus, $T_{u,v}T_{u,v}=I_{n}$. Hence, $\underbrace{T_{u,v}%
T_{u,v}}_{=I_{n}}A=I_{n}A=A$, so that%
\[
A=T_{u,v}\underbrace{T_{u,v}A}_{=M_{1}M_{2}\cdots M_{s}}=T_{u,v}M_{1}%
M_{2}\cdots M_{s}.
\]
But $T_{u,v}$ is a swapping matrix (by the definition of a \textquotedblleft
swapping matrix\textquotedblright), and the matrices $M_{1},M_{2},\ldots
,M_{s}$ are swapping matrices as well. Hence, $T_{u,v},M_{1},M_{2}%
,\ldots,M_{s}$ are swapping matrices. Thus, $T_{u,v}M_{1}M_{2}\cdots M_{s}$ is
a product of $s+1$ swapping matrices. In other words, $A$ is a product of
$s+1$ swapping matrices (since $A=T_{u,v}M_{1}M_{2}\cdots M_{s}$). Hence, $A$
is a product of at most $q$ swapping matrices (since $s+1\in\left\{
0,1,\ldots,q\right\}  $). In other words,
(\ref{pf.lem.permat.Tuv.k-id.2.indgoal}) holds. Hence,
(\ref{pf.lem.permat.Tuv.k-id.2.indgoal}) is proven in Case 2.

Now, we have proven (\ref{pf.lem.permat.Tuv.k-id.2.indgoal}) in each of the
two Cases 1 and 2. Hence, (\ref{pf.lem.permat.Tuv.k-id.2.indgoal}) always
holds. In other words, $A$ is a product of at most $q$ swapping matrices.

Now, forget that we assumed that $A$ is an $\left(  n-q\right)  $-identical
permutation matrix. Thus, we have shown that if $A$ is an $\left(  n-q\right)
$-identical permutation matrix, then $A$ is a product of at most $q$ swapping
matrices. In other words, Lemma \ref{lem.permat.Tuv.k-id.2} holds for $p=q$.
This completes the induction step. Thus, Lemma \ref{lem.permat.Tuv.k-id.2} is
proven by induction.
\end{proof}

Theorem \ref{thm.permat.Tuv} is now an easy consequence of the above:

\begin{proof}
[Proof of Theorem \ref{thm.permat.Tuv}.]$\Longleftarrow$: Lemma
\ref{lem.permat.Tuv.prod-swap} says that any product of swapping matrices is a
permutation matrix. Hence, if $C$ is a product of swapping matrices, then $C$
is a permutation matrix. This proves the $\Longleftarrow$ direction of Theorem
\ref{thm.permat.Tuv}.

$\Longrightarrow$: We need to prove that if $C$ is a permutation matrix, then
$C$ is a product of swapping matrices.

So let us assume that $C$ is a permutation matrix. Clearly, the matrix $C$ is
$0$-identical\footnote{\textit{Proof.} The matrix $C$ satisfies the equality
$C_{1,1}=C_{2,2}=\cdots=C_{0,0}=1$ (in fact, this equality is vacuously true).
But according to the definition of \textquotedblleft$0$%
-identical\textquotedblright, this means precisely that $C$ is $0$-identical.
Qed.}. In other words, the matrix $C$ is $\left(  n-n\right)  $-identical
(since $0=n-n$). Thus, Lemma \ref{lem.permat.Tuv.k-id.2} (applied to $p=n$ and
$A=C$) shows that $C$ is a product of at most $n$ swapping matrices. Hence,
$C$ is a product of swapping matrices. This is precisely what we had to prove.
This proves the $\Longrightarrow$ direction of Theorem \ref{thm.permat.Tuv}.
Hence, the proof of Theorem \ref{thm.permat.Tuv} is complete.
\end{proof}

\begin{proof}
[Proof of Proposition \ref{prop.permat.AB}.]Theorem \ref{thm.permat.Tuv}
(applied to $C=A$) shows that $A$ is a permutation matrix if and only if $A$
is a product of swapping matrices. Thus, $A$ is a product of swapping matrices
(since $A$ is a permutation matrix). The same argument (applied to $B$ instead
of $A$) shows that $B$ is a product of swapping matrices.

Now, we know that each of the two matrices $A$ and $B$ is a product of
swapping matrices. Hence, their product $AB$ is the product of two products of
swapping matrices. Therefore, $AB$ is itself a product of swapping matrices
(since a product of two products of swapping matrices must itself be a product
of swapping matrices).

Theorem \ref{thm.permat.Tuv} (applied to $C=AB$) shows that $AB$ is a
permutation matrix if and only if $AB$ is a product of swapping matrices.
Thus, $AB$ is a permutation matrix (since $AB$ is a product of swapping
matrices). This proves Proposition \ref{prop.permat.AB}.
\end{proof}

Let us now come to the proof of Proposition \ref{prop.permat.inverse}. Parts
\textbf{(a)} and \textbf{(c)} of Proposition \ref{prop.permat.inverse} could
be verified similarly to how we have proved Theorem
\ref{thm.triangular.inverse} (but using Theorem \ref{thm.permat.Tuv} now); but
this would not help us proving part \textbf{(b)}. So we take a different path instead.

\begin{proof}
[Proof of Proposition \ref{prop.permat.inverse}.]We have assumed that $A$ is a
permutation matrix. According to the definition of a \textquotedblleft
permutation matrix\textquotedblright, this means that $A$ satisfies the
following three statements:

\begin{statement}
\textit{Statement 1:} Each entry of $A$ is either a $0$ or a $1$.
\end{statement}

\begin{statement}
\textit{Statement 2:} Each row of $A$ has exactly one entry equal to $1$.
\end{statement}

\begin{statement}
\textit{Statement 3:} Each column of $A$ has exactly one entry equal to $1$.
\end{statement}

Thus, we know that Statements 1, 2 and 3 are satisfied.

We have $A^{T}=\left(  A_{j,i}\right)  _{1\leq i\leq n,\ 1\leq j\leq n}$ (by
the definition of $A^{T}$). Hence,%
\begin{equation}
\left(  A^{T}\right)  _{i,j}=A_{j,i}\ \ \ \ \ \ \ \ \ \ \text{for every }%
i\in\left\{  1,2,\ldots,n\right\}  \text{ and }j\in\left\{  1,2,\ldots
,n\right\}  . \label{pf.prop.permat.inverse.c.1}%
\end{equation}


We are next going to show that%
\begin{equation}
A^{T}\text{ is a permutation matrix.} \label{pf.prop.permat.inverse.ATper}%
\end{equation}
[\textit{Proof of (\ref{pf.prop.permat.inverse.ATper}):} We must show that
$A^{T}$ is a permutation matrix. According to the definition of a
\textquotedblleft permutation matrix\textquotedblright, this means proving
that $A^{T}$ satisfies the following three statements:

\begin{statement}
\textit{Statement 4:} Each entry of $A^{T}$ is either a $0$ or a $1$.
\end{statement}

\begin{statement}
\textit{Statement 5:} Each row of $A^{T}$ has exactly one entry equal to $1$.
\end{statement}

\begin{statement}
\textit{Statement 6:} Each column of $A^{T}$ has exactly one entry equal to
$1$.
\end{statement}

Hence, it remains to prove that Statements 4, 5 and 6 are satisfied.

The definition of $A^{T}$ shows that the entries of $A^{T}$ are precisely the
entries of $A$, just moved to different cells. Thus, Statement 4 follows from
Statement 1. Hence, Statement 4 is satisfied.

Let $i\in\left\{  1,2,\ldots,n\right\}  $. The $i$-th column of $A$ has
exactly one entry equal to $1$ (by Statement 3). In other words, there exists
exactly one $j\in\left\{  1,2,\ldots,n\right\}  $ satisfying $A_{j,i}=1$. In
view of (\ref{pf.prop.permat.inverse.c.1}), this rewrites as follows: There
exists exactly one $j\in\left\{  1,2,\ldots,n\right\}  $ satisfying $\left(
A^{T}\right)  _{i,j}=1$. In other words, the $i$-th row of $A^{T}$ has exactly
one entry equal to $1$.

Now, forget that we fixed $i$. We thus have shown that for each $i\in\left\{
1,2,\ldots,n\right\}  $, the $i$-th row of $A^{T}$ has exactly one entry equal
to $1$. In other words, each row of $A^{T}$ has exactly one entry equal to
$1$. This proves Statement 5.

A similar argument (but with the roles of rows and columns switched, and also
the roles of $i$ and $j$ switched, and using Statement 2 instead of Statement
3) can be used to prove Statement 6.

We thus have shown that Statements 4, 5 and 6 are satisfied. As we have said,
this completes the proof of (\ref{pf.prop.permat.inverse.ATper}).]

Every $i\in\left\{  1,2,\ldots,n\right\}  $ and $j\in\left\{  1,2,\ldots
,n\right\}  $ satisfy%
\begin{equation}
\left(  AA^{T}\right)  _{i,j}=\delta_{i,j}. \label{pf.prop.permat.inverse.a.1}%
\end{equation}


[\textit{Proof of (\ref{pf.prop.permat.inverse.a.1}):} Let $i\in\left\{
1,2,\ldots,n\right\}  $ and $j\in\left\{  1,2,\ldots,n\right\}  $.

The $i$-th row of $A$ has exactly one entry equal to $1$ (by Statement 2). In
other words, there exists a unique $u\in\left\{  1,2,\ldots,n\right\}  $
satisfying $A_{i,u}=1$. Consider this $u$. We have%
\begin{equation}
A_{i,k}=\delta_{k,u}\ \ \ \ \ \ \ \ \ \ \text{for every }k\in\left\{
1,2,\ldots,n\right\}  \label{pf.prop.permat.inverse.a.1.pf.1}%
\end{equation}
\footnote{\textit{Proof of (\ref{pf.prop.permat.inverse.a.1.pf.1}):} Let
$k\in\left\{  1,2,\ldots,n\right\}  $. We must prove that $A_{i,k}%
=\delta_{k,u}$.
\par
We will prove this by contradiction. Thus, assume the contrary. Hence,
$A_{i,k}\neq\delta_{k,u}$.
\par
If we had $k=u$, then we would have $A_{i,k}=A_{i,u}=1=\delta_{k,u}$ (since
$\delta_{k,u}=1$ (because $k=u$)), which would contradict $A_{i,k}\neq
\delta_{k,u}$. Thus, we cannot have $k=u$. We therefore must have $k\neq u$.
(Yes, we have just done a proof by contradiction inside a proof by
contradiction. This is perfectly legitimate!)
\par
Since $k\neq u$, we have $\delta_{k,u}=0$. On the other hand, Statement 1
shows that each entry of $A$ is either a $0$ or a $1$. In particular,
$A_{i,k}$ is either a $0$ or a $1$ (since $A_{i,k}$ is an entry of $A$). Since
$A_{i,k}$ is not a $0$ (because $A_{i,k}\neq\delta_{k,u}=0$), we therefore
conclude that $A_{i,k}$ is a $1$. In other words, $A_{i,k}=1$.
\par
So we know that both numbers $A_{i,k}$ and $A_{i,u}$ are equal to $1$. These
numbers $A_{i,k}$ and $A_{i,u}$ are two entries of the $i$-th row of $A$, and
lie in different cells (since $k\neq u$). Thus, the $i$-th row of $A$ has at
least two entries equal to $1$ (namely, the entries $A_{i,k}$ and $A_{i,u}$).
This contradicts the fact that the $i$-th row of $A$ has exactly one entry
equal to $1$. This contradiction shows that our assumption must have been
false. Hence, $A_{i,k}=\delta_{k,u}$ is proven. In other words,
(\ref{pf.prop.permat.inverse.a.1.pf.1}) is proven.}.

On the other hand, the $u$-th column of $A$ has exactly one entry equal to $1$
(by Statement 3). We can use this to show that%
\begin{equation}
A_{j,u}=\delta_{i,j} \label{pf.prop.permat.inverse.a.1.pf.2}%
\end{equation}
\footnote{\textit{Proof of (\ref{pf.prop.permat.inverse.a.1.pf.2}):} We will
prove this by contradiction. Thus, assume the contrary. Hence, $A_{j,u}%
\neq\delta_{i,j}$.
\par
If we had $i=j$, then we would have%
\begin{align*}
A_{j,u}  &  =A_{i,u}\ \ \ \ \ \ \ \ \ \ \left(  \text{since }j=i\right) \\
&  =1=\delta_{i,j}\ \ \ \ \ \ \ \ \ \ \left(  \text{since }\delta
_{i,j}=1\text{ (because }i=j\text{)}\right)  ,
\end{align*}
which would contradict $A_{j,u}\neq\delta_{i,j}$. Thus, we cannot have $i=j$.
We therefore must have $i\neq j$. (Once again, we have just made a proof by
contradiction within a proof by contradiction.)
\par
Since $i\neq j$, we have $\delta_{i,j}=0$. On the other hand, Statement 1
shows that each entry of $A$ is either a $0$ or a $1$. In particular,
$A_{j,u}$ is either a $0$ or a $1$ (since $A_{j,u}$ is an entry of $A$). Since
$A_{j,u}$ is not a $0$ (because $A_{j,u}\neq\delta_{i,j}=0$), we therefore
conclude that $A_{j,u}$ is a $1$. In other words, $A_{j,u}=1$.
\par
So we know that both numbers $A_{i,u}$ and $A_{j,u}$ are equal to $1$. These
numbers $A_{i,u}$ and $A_{j,u}$ are two entries of the $u$-th column of $A$,
and lie in different cells (since $i\neq j$). Thus, the $u$-th column of $A$
has at least two entries equal to $1$ (namely, the entries $A_{i,u}$ and
$A_{j,u}$). This contradicts the fact that the $u$-th column of $A$ has
exactly one entry equal to $1$. This contradiction shows that our assumption
must have been false. Hence, $A_{j,u}=\delta_{i,j}$ is proven. In other words,
(\ref{pf.prop.permat.inverse.a.1.pf.2}) is proven.}.

But Proposition \ref{prop.AB} (applied to $m=n$, $p=n$ and $B=A^{T}$) shows
that%
\[
\left(  AA^{T}\right)  _{i,j}=\sum_{k=1}^{n}\underbrace{A_{i,k}}%
_{\substack{=\delta_{k,u}\\\text{(by (\ref{pf.prop.permat.inverse.a.1.pf.1}%
))}}}\underbrace{\left(  A^{T}\right)  _{k,j}}_{\substack{=A_{j,k}\\\text{(by
(\ref{pf.prop.permat.inverse.c.1}), applied}\\\text{to }k\text{ instead of
}i\text{)}}}=\sum_{k=1}^{n}\delta_{k,u}A_{j,k}=\sum_{k=1}^{n}A_{j,k}%
\delta_{k,u}=A_{j,u}%
\]
(by Proposition \ref{prop.sum.delta}, applied to $p=1$, $q=n$, $r=u$ and
$a_{k}=A_{j,k}$). Hence, $\left(  AA^{T}\right)  _{i,j}=A_{j,u}=\delta_{i,j}$
(by (\ref{pf.prop.permat.inverse.a.1.pf.2})). This proves
(\ref{pf.prop.permat.inverse.a.1}).]

On the other hand, $I_{n}=\left(  \delta_{i,j}\right)  _{1\leq i\leq n,\ 1\leq
j\leq n}$. Hence, every $i\in\left\{  1,2,\ldots,n\right\}  $ and
$j\in\left\{  1,2,\ldots,n\right\}  $ satisfy $\left(  I_{n}\right)
_{i,j}=\delta_{i,j}$. Comparing this with (\ref{pf.prop.permat.inverse.a.1}),
we find that every $i\in\left\{  1,2,\ldots,n\right\}  $ and $j\in\left\{
1,2,\ldots,n\right\}  $ satisfy $\left(  AA^{T}\right)  _{i,j}=\left(
I_{n}\right)  _{i,j}$. In other words, each entry of the matrix $AA^{T}$
equals the corresponding entry of $I_{n}$. Therefore, $AA^{T}=I_{n}$.

Now, let us change our point of view. We have just shown that $AA^{T}=I_{n}$.
But (\ref{pf.prop.permat.inverse.ATper}) shows that $A^{T}$ is a permutation
matrix. Hence, the argument that we used to prove $AA^{T}=I_{n}$ can also be
applied to $A^{T}$ instead of $A$. It therefore yields $A^{T}\left(
A^{T}\right)  ^{T}=I_{n}$. Since $\left(  A^{T}\right)  ^{T}=A$ (indeed, this
follows from Proposition \ref{prop.transpose.invol}), this rewrites as
$A^{T}A=I_{n}$.

Now, the two equalities $AA^{T}=I_{n}$ and $A^{T}A=I_{n}$ (combined) show that
the matrix $A^{T}$ is an inverse of $A$. Hence, the matrix $A$ is invertible,
and its inverse is $A^{-1}=A^{T}$. This proves parts \textbf{(a)} and
\textbf{(b)} of Proposition \ref{prop.permat.inverse}. It remains to prove
part \textbf{(c)}. But this is obvious now: From
(\ref{pf.prop.permat.inverse.ATper}), we know that $A^{T}$ is a permutation
matrix. Since $A^{-1}=A^{T}$, this rewrites as follows: $A^{-1}$ is a
permutation matrix. This proves Proposition \ref{prop.permat.inverse}
\textbf{(c)}.
\end{proof}

Let me add one more result, which somewhat extends the $\Longrightarrow$
direction of Theorem \ref{thm.permat.Tuv}:

\begin{proposition}
\label{prop.permat.Tkk+1}Let $n\in\mathbb{N}$. A \textit{simple swapping
matrix} will mean an $n\times n$-matrix of the form $T_{k,k+1}$ for some
$k\in\left\{  1,2,\ldots,n-1\right\}  $.

Each permutation matrix of size $n\times n$ is a product of simple swapping matrices.
\end{proposition}

\begin{example}
Let $n=3$. Then, there are three swapping matrices: $T_{1,2}$, $T_{1,3}$ and
$T_{2,3}$. (You can also write down $T_{2,1}$, $T_{3,1}$ and $T_{3,2}$, but
these are the same three matrices by different names.) Out of these three
matrices, two (namely, $T_{1,2}$ and $T_{2,3}$) are simple swapping matrices
(in the sense of Proposition \ref{prop.permat.Tkk+1}), whereas the remaining
one ($T_{1,3}$) is not.

Proposition \ref{prop.permat.Tkk+1} (applied to $n=3$) says that each
permutation matrix of size $3\times3$ is a product of simple swapping
matrices. Let us check this for the permutation matrix $T_{1,3}$. Just writing
$T_{1,3}$ as the product of itself alone does not count anymore, because (as
we said) $T_{1,3}$ is not a simple swapping matrix. However, we can write
$T_{1,3}$ as the product $T_{1,2}T_{2,3}T_{1,2}$, and this does the trick.
(Alternatively, you can write $T_{1,3}$ as the product $T_{2,3}T_{1,2}T_{2,3}%
$. Again, there are many possibilities.)
\end{example}

\begin{proof}
[Proof of Proposition \ref{prop.permat.Tkk+1} (sketched).]We will not need
Proposition \ref{prop.permat.Tkk+1}, so let me only give a brief outline of
its proof.

We want to prove that each permutation matrix of size $n\times n$ is a product
of simple swapping matrices. But the $\Longrightarrow$ direction of Theorem
\ref{thm.permat.Tuv} shows that each permutation matrix of size $n\times n$ is
a product of swapping matrices. Hence, it suffices to prove that%
\begin{equation}
\text{each swapping matrix is a product of simple swapping matrices.}
\label{pf.prop.permat.Tkk+1.goal2}%
\end{equation}


So let $A$ be a swapping matrix. Thus, $A=T_{u,v}$ for two distinct elements
$u$ and $v$ of $\left\{  1,2,\ldots,n\right\}  $. Consider these $u$ and $v$.
We can WLOG assume that $u<v$ (since otherwise, we can simply switch $u$ with
$v$, without changing $A$ because $T_{u,v}=T_{v,u}$). So, assume this. Then, I
claim that%
\begin{equation}
T_{u,v}=\left(  T_{u,u+1}T_{u+1,u+2}\cdots T_{v-2,v-1}\right)  T_{v-1,v}%
\left(  T_{v-2,v-1}T_{v-3,v-2}\cdots T_{u,u+1}\right)  .
\label{pf.prop.permat.Tkk+1.goal4}%
\end{equation}
(To ease understanding, let me explain how the right hand side of
(\ref{pf.prop.permat.Tkk+1.goal4}) should be interpreted: The first
parenthesized factor, $T_{u,u+1}T_{u+1,u+2}\cdots T_{v-2,v-1}$, is the product
of all $T_{k,k+1}$ with $k$ ranging over $\left\{  u,u+1,\ldots,v-2\right\}  $
\textbf{in increasing order}. If the set $\left\{  u,u+1,\ldots,v-2\right\}  $
is empty, then this product has to be understood as the empty product (which
equals $I_{n}$ as usual). The second parenthesized factor, $T_{v-2,v-1}%
T_{v-3,v-2}\cdots T_{u,u+1}$, is the product of all $T_{k,k+1}$ with $k$
ranging over $\left\{  u,u+1,\ldots,v-2\right\}  $ \textbf{in decreasing
order}. Again, it is the empty product if $\left\{  u,u+1,\ldots,v-2\right\}
$ is empty (and thus equals $I_{n}$ in this case).)

Clearly, the equality (\ref{pf.prop.permat.Tkk+1.goal4}) represents $T_{u,v}$
as a product of simple swapping matrices; thus, once it is proven, the claim
(\ref{pf.prop.permat.Tkk+1.goal2}) will immediately follow.

One way to prove (\ref{pf.prop.permat.Tkk+1.goal4}) is to rewrite the right
hand side of (\ref{pf.prop.permat.Tkk+1.goal4}) as%
\begin{equation}
\left(  T_{u,u+1}T_{u+1,u+2}\cdots T_{v-2,v-1}\right)  T_{v-1,v}\left(
T_{v-2,v-1}T_{v-3,v-2}\cdots T_{u,u+1}\right)  I_{n}.
\label{pf.prop.permat.Tkk+1.RHS}%
\end{equation}
By using Proposition \ref{prop.Tuv.laction} repeatedly, we thus see that it is
the matrix obtained from $I_{n}$ by:

\begin{itemize}
\item first, swapping the $u$-th row with the $\left(  u+1\right)  $-th row,

\item then, swapping the $\left(  u+1\right)  $-th row with the $\left(
u+2\right)  $-th row,

\item and so on, until finally swapping the $\left(  v-2\right)  $-th row with
the $\left(  v-1\right)  $-th row,

\item then, swapping the $\left(  v-1\right)  $-th row with the $v$-th row,

\item then, swapping the $\left(  v-2\right)  $-th row with the $\left(
v-1\right)  $-th row,

\item then, swapping the $\left(  v-3\right)  $-th row with the $\left(
v-2\right)  $-th row,

\item and so on, until finally swapping the $u$-th row with the $\left(
u+1\right)  $-th row.
\end{itemize}

This sequence of swappings has the following consequence: The $u$-th row has
been moved further and further down (by repeatedly getting swapped with the
next row) until it finally found rest in the position of the $v$-th row; then,
the former $v$-th row has been moved further and further up (by repeatedly
getting swapped with the previous row) until it finally found rest in the
position of the $u$-th row. At the end of this process, the $u$-th and the
$v$-th rows have traded places, but all the other rows have remained at the
positions where they started out (although some of them were temporarily moved
back and forth in the process). So the right hand side of
(\ref{pf.prop.permat.Tkk+1.goal4}) can be obtained from $I_{n}$ by swapping
the $u$-th row with the $v$-th row. But (according to Corollary
\ref{cor.Tuv.as-switch} \textbf{(a)}) the matrix $T_{u,v}$ can be obtained in
exactly the same way. Hence, the right hand side of
(\ref{pf.prop.permat.Tkk+1.goal4}) simply is $T_{u,v}$. Thus,
(\ref{pf.prop.permat.Tkk+1.goal4}) is proven. As explained, this proves
Proposition \ref{prop.permat.Tkk+1}.

(If you are looking for a more rigorous proof of
(\ref{pf.prop.permat.Tkk+1.goal4}) -- without the swapping process so
confusingly described above --, you can also proceed by induction over $v-u$.
In the induction step, (\ref{pf.prop.permat.Tkk+1.goal4}) is derived from the
equality
\[
T_{u+1,v}=\left(  T_{u+1,u+2}T_{u+2,u+3}\cdots T_{v-2,v-1}\right)
T_{v-1,v}\left(  T_{v-2,v-1}T_{v-3,v-2}\cdots T_{u+1,u+2}\right)  ,
\]
which follows from the induction hypothesis. The details are, again, omitted.)
\end{proof}

\subsection{\label{sect.gauss.permat.perms}(*) Permutation matrices and
permutations}

In Definition \ref{def.permat}, we have defined permutation matrices as
matrices satisfying three particular conditions. This may not be their most
natural definition, and certainly has the disadvantage of making them look
like curiosities rather than like something important. In this section, I
shall show a different approach to them which better demonstrates their
significance. The idea is that permutation matrices are matrix representations
of \textit{permutations}, a fundamental notion of combinatorics (= the theory
of finite sets).

First, let me recall some basic terminology:

\begin{itemize}
\item A map\footnote{The words \textquotedblleft map\textquotedblright,
\textquotedblleft mapping\textquotedblright, \textquotedblleft
function\textquotedblright, \textquotedblleft transformation\textquotedblright%
\ and \textquotedblleft operator\textquotedblright\ are synonyms in
mathematics. (That said, mathematicians often show some nuance by using one of
them and not the other. However, we do not need to concern ourselves with this
here.)} $f:X\rightarrow Y$ between two sets $X$ and $Y$ is said to be
\textit{injective} if it has the following property:

\begin{itemize}
\item If $x_{1}$ and $x_{2}$ are two elements of $X$ satisfying $f\left(
x_{1}\right)  =f\left(  x_{2}\right)  $, then $x_{1}=x_{2}$. (In words: If two
elements of $X$ are sent to one and the same element of $Y$ by $f$, then these
two elements of $X$ must have been equal in the first place. In other words:
An element of $X$ is uniquely determined by its image under $f$.)
\end{itemize}

Injective maps are often called \textquotedblleft one-to-one
maps\textquotedblright\ or \textquotedblleft injections\textquotedblright.

For example:

\begin{itemize}
\item The map $\mathbb{Z}\rightarrow\mathbb{Z},\ x\mapsto2x$ (this is the map
that sends each integer $x$ to $2x$) is injective, because if $x_{1}$ and
$x_{2}$ are two integers satisfying $2x_{1}=2x_{2}$, then $x_{1}=x_{2}$.

\item The map $\mathbb{Z}\rightarrow\mathbb{Z},\ x\mapsto x^{2}$ (this is the
map that sends each integer $x$ to $x^{2}$) is \textbf{not} injective, because
if $x_{1}$ and $x_{2}$ are two integers satisfying $x_{1}^{2}=x_{2}^{2}$, then
we do not necessarily have $x_{1}=x_{2}$. (For example, if $x_{1}=-1$ and
$x_{2}=1$, then $x_{1}^{2}=x_{2}^{2}$ but not $x_{1}=x_{2}$.)
\end{itemize}

\item A map $f:X\rightarrow Y$ between two sets $X$ and $Y$ is said to be
\textit{surjective} if it has the following property:

\begin{itemize}
\item For each $y\in Y$, there exists some $x\in X$ satisfying $f\left(
x\right)  =y$. (In words: Each element of $Y$ is an image of some element of
$X$ under $f$.)
\end{itemize}

Surjective maps are often called \textquotedblleft onto maps\textquotedblright%
\ or \textquotedblleft surjections\textquotedblright.

For example:

\begin{itemize}
\item The map $\mathbb{Z}\rightarrow\mathbb{Z},\ x\mapsto x+1$ (this is the
map that sends each integer $x$ to $x+1$) is surjective, because each integer
$y$ has some integer satisfying $x+1=y$ (namely, $x=y-1$).

\item The map $\mathbb{Z}\rightarrow\mathbb{Z},\ x\mapsto2x$ (this is the map
that sends each integer $x$ to $2x$) is \textbf{not} surjective, because not
each integer $y$ has some integer $x$ satisfying $2x=y$. (For instance, $y=1$
has no such $x$, since $y$ is odd.)

\item The map $\left\{  1,2,3,4\right\}  \rightarrow\left\{
1,2,3,4,5\right\}  ,\ x\mapsto x$ (this is the map sending each $x$ to $x$) is
\textbf{not} surjective, because not each $y\in\left\{  1,2,3,4,5\right\}  $
has some $x\in\left\{  1,2,3,4\right\}  $ satisfying $x=y$. (Namely, $y=5$ has
no such $x$.)
\end{itemize}

\item A map $f:X\rightarrow Y$ between two sets $X$ and $Y$ is said to be
\textit{bijective} if it is both injective and surjective. Bijective maps are
often called \textquotedblleft one-to-one correspondences\textquotedblright%
\ or \textquotedblleft bijections\textquotedblright.

For example:

\begin{itemize}
\item The map $\mathbb{Z}\rightarrow\mathbb{Z},\ x\mapsto x+1$ is bijective,
since it is both injective and surjective.

\item The map $\left\{  1,2,3,4\right\}  \rightarrow\left\{
1,2,3,4,5\right\}  ,\ x\mapsto x$ is \textbf{not} bijective, since it is not surjective.

\item The map $\mathbb{Z}\rightarrow\mathbb{Z},\ x\mapsto x^{2}$ is
\textbf{not} bijective, since it is not injective. (It also is not surjective.)
\end{itemize}

\item If $X$ is a set, then $\operatorname*{id}\nolimits_{X}$ denotes the map
from $X$ to $X$ that sends each $x\in X$ to $x$ itself. (In words:
$\operatorname*{id}\nolimits_{X}$ denotes the map which sends each element of
$X$ to itself.) The map $\operatorname*{id}\nolimits_{X}$ is often called the
\textit{identity map on }$X$, and often denoted by $\operatorname*{id}$ (when
$X$ is clear from the context or irrelevant). The identity map
$\operatorname*{id}\nolimits_{X}$ is always bijective.

\item If $f:X\rightarrow Y$ and $g:Y\rightarrow Z$ are two maps, then the
\textit{composition} $g\circ f$ of the maps $g$ and $f$ is defined to be the
map from $X$ to $Z$ that sends each $x\in X$ to $g\left(  f\left(  x\right)
\right)  $. (In words: The composition $g\circ f$ is the map from $X$ to $Z$
that applies the map $f$ \textbf{first} and \textbf{then} applies the map
$g$.) You might find it confusing that this map is denoted by $g\circ f$
(rather than $f\circ g$), given that it proceeds by applying $f$ first and $g$
last; however, this has its reasons: It satisfies $\left(  g\circ f\right)
\left(  x\right)  =g\left(  f\left(  x\right)  \right)  $. Had we denoted it
by $f\circ g$ instead, this equality would instead become $\left(  f\circ
g\right)  \left(  x\right)  =g\left(  f\left(  x\right)  \right)  $, which
would be even more confusing.

\item If $f:X\rightarrow Y$ is a map between two sets $X$ and $Y$, then an
\textit{inverse} of $f$ means a map $g:Y\rightarrow X$ satisfying $f\circ
g=\operatorname*{id}\nolimits_{Y}$ and $g\circ f=\operatorname*{id}%
\nolimits_{X}$. (In words, the condition \textquotedblleft$f\circ
g=\operatorname*{id}\nolimits_{Y}$\textquotedblright\ means \textquotedblleft
if you start with some element $y\in Y$, then apply $g$, then apply $f$, then
you get $y$ back\textquotedblright, or equivalently \textquotedblleft the map
$f$ undoes the map $g$\textquotedblright. Similarly, the condition
\textquotedblleft$g\circ f=\operatorname*{id}\nolimits_{X}$\textquotedblright%
\ means \textquotedblleft if you start with some element $x\in X$, then apply
$f$, then apply $g$, then you get $x$ back\textquotedblright, or equivalently
\textquotedblleft the map $g$ undoes the map $f$\textquotedblright. Thus, an
inverse of $f$ means a map $g:Y\rightarrow X$ that both undoes and is undone
by $f$.)

The map $f:X\rightarrow Y$ is said to be \textit{invertible} if and only if an
inverse of $f$ exists. If an inverse of $f$ exists, then it is
unique\footnote{This is not hard to show. In fact, the situation is very
similar to inverses of matrices; in particular, we can define
\textquotedblleft left inverses\textquotedblright\ and \textquotedblleft right
inverses\textquotedblright, and prove analogues of Proposition
\ref{prop.inverses.L=R} and Corollary \ref{cor.inverses.unique} for maps
instead of matrices.}, and thus is called \textit{the inverse of }$f$, and is
denoted by $f^{-1}$.

For example:

\begin{itemize}
\item The map $\mathbb{Z}\rightarrow\mathbb{Z},\ x\mapsto x+1$ is invertible,
and its inverse is $\mathbb{Z}\rightarrow\mathbb{Z},\ x\mapsto x-1$.

\item The map $\mathbb{Q}\setminus\left\{  1\right\}  \rightarrow
\mathbb{Q}\setminus\left\{  0\right\}  ,\ x\mapsto\dfrac{1}{1-x}$ is
invertible, and its inverse is the map $\mathbb{Q}\setminus\left\{  0\right\}
\rightarrow\mathbb{Q}\setminus\left\{  1\right\}  ,\ x\mapsto1-\dfrac{1}{x}$.
\end{itemize}

A map $f:X\rightarrow Y$ is invertible if and only if it is bijective.

\item If $X$ is a set, then a \textit{permutation} of $X$ means a bijective
(i.e., invertible) map $X\rightarrow X$.
\end{itemize}

Each permutation of the set $\left\{  1,2,\ldots,n\right\}  $ (for any
$n\in\mathbb{N}$) determines a permutation matrix:

\begin{definition}
\label{def.permat.permat-from-perm}Let $n\in\mathbb{N}$. Let $w$ be a
permutation of $\left\{  1,2,\ldots,n\right\}  $ (that is, a bijection
$\left\{  1,2,\ldots,n\right\}  \rightarrow\left\{  1,2,\ldots,n\right\}  $).
Then, we define an $n\times n$-matrix $P_{w}$ by $P_{w}=\left(  \delta
_{w\left(  i\right)  ,j}\right)  _{1\leq i\leq n,\ 1\leq j\leq n}$. In other
words, $P_{w}$ is the $n\times n$-matrix with the following entries:

\begin{itemize}
\item For each $i\in\left\{  1,2,\ldots,n\right\}  $, the $i$-th row has an
entry equal to $1$ in the $w\left(  i\right)  $-th position (that is, the
$\left(  i,w\left(  i\right)  \right)  $-th entry of the matrix is $1$).

\item All other entries are $0$.
\end{itemize}
\end{definition}

\begin{example}
Let $n=4$. There are $24$ permutations of the set $\left\{  1,2,3,4\right\}
$. One of them is the map $u:\left\{  1,2,3,4\right\}  \rightarrow\left\{
1,2,3,4\right\}  $ that sends $1,2,3,4$ to $3,1,4,2$, respectively. The matrix
$P_{u}$ corresponding to this map $u$ is $\left(
\begin{array}
[c]{cccc}%
0 & 0 & 1 & 0\\
1 & 0 & 0 & 0\\
0 & 0 & 0 & 1\\
0 & 1 & 0 & 0
\end{array}
\right)  $.
\end{example}

\begin{proposition}
\label{prop.permat.permat-from-perm}Let $n\in\mathbb{N}$.

\textbf{(a)} If $w$ is a permutation of $\left\{  1,2,\ldots,n\right\}  $,
then $P_{w}$ is a permutation matrix.

\textbf{(b)} Let $P$ be a permutation matrix of size $n\times n$. Then, there
exists a \textbf{unique} permutation $w$ of $\left\{  1,2,\ldots,n\right\}  $
such that $P=P_{w}$.
\end{proposition}

\begin{proof}
[Proof of Proposition \ref{prop.permat.permat-from-perm} (sketched).]%
\textbf{(a)} Let $w$ be a permutation of $\left\{  1,2,\ldots,n\right\}  $.
Thus, $w$ is a bijective map, i.e., an injective and surjective map.

We want to show that $P_{w}$ is a permutation matrix. According to the
definition of a \textquotedblleft permutation matrix\textquotedblright, we
therefor must prove the following three statements:

\begin{statement}
\textit{Statement 1:} Each entry of $P_{w}$ is either a $0$ or a $1$.
\end{statement}

\begin{statement}
\textit{Statement 2:} Each row of $P_{w}$ has exactly one entry equal to $1$.
\end{statement}

\begin{statement}
\textit{Statement 3:} Each column of $P_{w}$ has exactly one entry equal to
$1$.
\end{statement}

Statement 1 follows immediately from the definition of $P_{w}$.

Statement 2 follows from the definition as well: For each $i\in\left\{
1,2,\ldots,n\right\}  $, the $i$-th row of $P_{w}$ has exactly one entry equal
to $1$ (namely, the entry in position $w\left(  i\right)  $).

It remains to prove Statement 3. Fix $j\in\left\{  1,2,\ldots,n\right\}  $. We
must show that the $j$-th column of $P_{w}$ has exactly one entry equal to
$1$. In other words, we must prove that there exists exactly one $i\in\left\{
1,2,\ldots,n\right\}  $ satisfying $w\left(  i\right)  =j$.

Since $w$ is surjective, there exists \textbf{at least one} such $i$. Since
$w$ is injective, there exists \textbf{at most one} such $i$ (because if
$i_{1}$ and $i_{2}$ are two such $i$, then $w\left(  i_{1}\right)  =j$ and
$w\left(  i_{2}\right)  =j$, so that $w\left(  i_{1}\right)  =w\left(
i_{2}\right)  $, and thus the injectivity of $w$ leads to $i_{1}=i_{2}$).
Combining the previous two sentences, we conclude that there exists
\textbf{exactly one} such $i$. Thus, Statement 3 is proven.

We have now proven all three Statements 1, 2 and 3. Hence, $P_{w}$ is a
permutation matrix, so that we have proven Proposition
\ref{prop.permat.permat-from-perm} \textbf{(a)}.

\textbf{(b)} We know that $P$ is a permutation matrix. In other words, the
following three statements hold:

\begin{statement}
\textit{Statement 1:} Each entry of $P$ is either a $0$ or a $1$.
\end{statement}

\begin{statement}
\textit{Statement 2:} Each row of $P$ has exactly one entry equal to $1$.
\end{statement}

\begin{statement}
\textit{Statement 3:} Each column of $P$ has exactly one entry equal to $1$.
\end{statement}

Now, define a map $w:\left\{  1,2,\ldots,n\right\}  \rightarrow\left\{
1,2,\ldots,n\right\}  $ as follows:

Let $i\in\left\{  1,2,\ldots,n\right\}  $. Then, the $i$-th row of $P$ has
exactly one entry equal to $1$ (by Statement 2). In other words, there exists
exactly one $j\in\left\{  1,2,\ldots,n\right\}  $ such that $P_{i,j}=1$.
Define $w\left(  i\right)  $ to be this $j$.

Thus, we have defined $w\left(  i\right)  $ for each $i\in\left\{
1,2,\ldots,n\right\}  $. Hence, the map $w:\left\{  1,2,\ldots,n\right\}
\rightarrow\left\{  1,2,\ldots,n\right\}  $ is defined.

Described in words, the map $w$ sends each $i\in\left\{  1,2,\ldots,n\right\}
$ to the position of the unique entry equal to $1$ in the $i$-th row of $P$.
Statement 1 shows that all the other entries are zeroes. Hence, the entries of
the matrix $P$ can be described as follows:

\begin{itemize}
\item For each $i\in\left\{  1,2,\ldots,n\right\}  $, the $\left(  i,w\left(
i\right)  \right)  $-th entry of $P$ is $1$.

\item All remaining entries of $P$ are $0$.
\end{itemize}

In other words, for each $i\in\left\{  1,2,\ldots,n\right\}  $ and
$j\in\left\{  1,2,\ldots,n\right\}  $, the $\left(  i,j\right)  $-th entry of
$P$ is $%
\begin{cases}
1, & \text{if }j=w\left(  i\right)  ;\\
0, & \text{if }j\neq w\left(  i\right)
\end{cases}
=\delta_{j,w\left(  i\right)  }=\delta_{w\left(  i\right)  ,j}$. Thus, the
matrix $P$ equals $\left(  \delta_{w\left(  i\right)  ,j}\right)  _{1\leq
i\leq n,\ 1\leq j\leq n}$.

Now, the map $w$ is injective\footnote{\textit{Proof.} Let $i_{1}$ and $i_{2}$
be two elements of $\left\{  1,2,\ldots,n\right\}  $ such that $w\left(
i_{1}\right)  =w\left(  i_{2}\right)  $. We must show that $i_{1}=i_{2}$.
\par
We have $P_{i_{1},w\left(  i_{1}\right)  }=1$ (by the definition of $w$) and
$P_{i_{2},w\left(  i_{2}\right)  }=1$ (similarly). Since $w\left(
i_{1}\right)  =w\left(  i_{2}\right)  $, we have $P_{i_{2},w\left(
i_{1}\right)  }=P_{i_{2},w\left(  i_{2}\right)  }=1$.
\par
If $i_{1}\neq i_{2}$, then the equalities $P_{i_{1},w\left(  i_{1}\right)
}=1$ and $P_{i_{2},w\left(  i_{1}\right)  }=1$ show that the $w\left(
i_{1}\right)  $-th column of $P$ has (at least) two entries equal to $1$
(namely, the entries in cells $\left(  i_{1},w\left(  i_{1}\right)  \right)  $
and $\left(  i_{2},w\left(  i_{1}\right)  \right)  $); but this flies in the
face of the fact that the $w\left(  i_{1}\right)  $-th column of $P$ has
exactly one entry equal to $1$ (which follows from Statement 3). Hence, we
cannot have $i_{1}\neq i_{2}$. We thus have $i_{1}=i_{2}$. This completes the
proof that $w$ is injective.} and surjective\footnote{\textit{Proof.} Let
$j\in\left\{  1,2,\ldots,n\right\}  $. Then, the $j$-th column of $P$ has
exactly one entry equal to $1$ (by Statement 3). In other words, there exists
exactly one $i\in\left\{  1,2,\ldots,n\right\}  $ such that $P_{i,j}=1$.
Consider this $i$. Then, $P_{i,j}=1$, so that $w\left(  i\right)  =j$ (by the
definition of $w$). Hence, we have shown that, for each $j\in\left\{
1,2,\ldots,n\right\}  $, there exists some $i\in\left\{  1,2,\ldots,n\right\}
$ satisfying $w\left(  i\right)  =j$. In other words, the map $w$ is
surjective.}. Hence, $w$ is bijective. Thus, $w$ is a permutation of $\left\{
1,2,\ldots,n\right\}  $. The definition of $P_{w}$ thus yields $P_{w}=\left(
\delta_{w\left(  i\right)  ,j}\right)  _{1\leq i\leq n,\ 1\leq j\leq n}$.
Comparing this with $P=\left(  \delta_{w\left(  i\right)  ,j}\right)  _{1\leq
i\leq n,\ 1\leq j\leq n}$, we obtain $P=P_{w}$.

We thus have shown that there exists \textbf{at least one} permutation $w$ of
$\left\{  1,2,\ldots,n\right\}  $ such that $P=P_{w}$. It remains to prove
that there exists \textbf{at most one} such permutation $w$. Fortunately, this
is easy: The requirement that $P=P_{w}$ determines $w$ uniquely, since the
values of $w$ can be read off the matrix $P_{w}$ (namely, $w\left(  i\right)
$ is the position of the entry equal to $1$ in the $i$-th row of $P_{w}$).
Hence, Proposition \ref{prop.permat.permat-from-perm} \textbf{(b)} is proven.
\end{proof}

Proposition \ref{prop.permat.permat-from-perm} shows that the permutation
matrices of size $n\times n$ (for any given $n\in\mathbb{N}$) are precisely
the matrices of the form $P_{w}$ with $w$ being a permutation of $\left\{
1,2,\ldots,n\right\}  $. Moreover, it shows that each permutation matrix can
be written in this form in a unique way. Hence, the map%
\begin{align*}
\left\{  \text{permutations of }\left\{  1,2,\ldots,n\right\}  \right\}   &
\rightarrow\left\{  \text{permutation matrices of size }n\times n\right\}  ,\\
w  &  \mapsto P_{w}%
\end{align*}
\footnote{By this, I mean: the map from the set of all permutations of
$\left\{  1,2,\ldots,n\right\}  $ to the set of all permutation matrices of
size $n\times n$ that sends each permutation $w$ to the permutation matrix
$P_{w}$.} is a bijection. Therefore, there are as many permutation matrices of
size $n\times n$ as there are permutations of $\left\{  1,2,\ldots,n\right\}
$. This allows us to prove Proposition \ref{prop.permat.n!}:

\begin{proof}
[Proof of Proposition \ref{prop.permat.n!} (sketched).]We have just seen that
there are as many permutation matrices of size $n\times n$ as there are
permutations of $\left\{  1,2,\ldots,n\right\}  $. Hence, it suffices to show
that there are precisely $n!$ permutations of $\left\{  1,2,\ldots,n\right\}
$.

How do we choose a permutation $w$ of $\left\{  1,2,\ldots,n\right\}  $ ?
Clearly, it suffices to choose each of its $n$ values $w\left(  1\right)
,w\left(  2\right)  ,\ldots,w\left(  n\right)  $ among the numbers in the set
$\left\{  1,2,\ldots,n\right\}  $. These $n$ values must be distinct (because
$w$ has to be a permutation, and thus injective). Also, these $n$ values must
cover all of the set $\left\{  1,2,\ldots,n\right\}  $ (since $w$ has to be a
permutation, and thus surjective); however, it turns out that this is already
guaranteed if we choose these $n$ values to be distinct (because $n$ distinct
values chosen among the numbers in $\left\{  1,2,\ldots,n\right\}  $ will
\textbf{always} cover all of the set $\left\{  1,2,\ldots,n\right\}  $).
Hence, we need to choose $n$ distinct numbers $w\left(  1\right)  ,w\left(
2\right)  ,\ldots,w\left(  n\right)  $ among the numbers in the set $\left\{
1,2,\ldots,n\right\}  $. Here is one way to do this:

\begin{itemize}
\item First, choose a value for $w\left(  1\right)  $. There are $n$ possible
choices for this (since $w\left(  1\right)  $ has to belong to $\left\{
1,2,\ldots,n\right\}  $).

\item Second, choose a value for $w\left(  2\right)  $. There are $n-1$
possible choices for this (since $w\left(  2\right)  $ has to belong to
$\left\{  1,2,\ldots,n\right\}  $, but must not equal $w\left(  1\right)  $,
since we want the $n$ numbers $w\left(  1\right)  ,w\left(  2\right)
,\ldots,w\left(  n\right)  $ to be distinct).

\item Third, choose a value for $w\left(  3\right)  $. There are $n-2$
possible choices for this (since $w\left(  3\right)  $ has to belong to
$\left\{  1,2,\ldots,n\right\}  $, but must not equal any of $w\left(
1\right)  $ and $w\left(  2\right)  $, since we want the $n$ numbers $w\left(
1\right)  ,w\left(  2\right)  ,\ldots,w\left(  n\right)  $ to be distinct).

\item Fourth, choose a value for $w\left(  4\right)  $. There are $n-3$
possible choices for this (since $w\left(  4\right)  $ has to belong to
$\left\{  1,2,\ldots,n\right\}  $, but must not equal any of $w\left(
1\right)  ,w\left(  2\right)  ,w\left(  3\right)  $, since we want the $n$
numbers $w\left(  1\right)  ,w\left(  2\right)  ,\ldots,w\left(  n\right)  $
to be distinct).

\item And so on.

\item At the last step, choose a value for $w\left(  n\right)  $. There are
$n-\left(  n-1\right)  $ possible choices for this (since $w\left(  n\right)
$ has to belong to $\left\{  1,2,\ldots,n\right\}  $, but must not equal any
of $w\left(  1\right)  ,w\left(  2\right)  ,\ldots,w\left(  n-1\right)  $,
since we want the $n$ numbers $w\left(  1\right)  ,w\left(  2\right)
,\ldots,w\left(  n\right)  $ to be distinct).
\end{itemize}

We thus get a total of $n\cdot\left(  n-1\right)  \cdot\left(  n-2\right)
\cdot\left(  n-3\right)  \cdot\cdots\cdot\left(  n-\left(  n-1\right)
\right)  $ possible choices. In other words, we get a total of $n!$ possible
choices\footnote{since
\begin{align*}
&  n\cdot\left(  n-1\right)  \cdot\left(  n-2\right)  \cdot\left(  n-3\right)
\cdot\cdots\cdot\left(  n-\left(  n-1\right)  \right) \\
&  =n\cdot\left(  n-1\right)  \cdot\left(  n-2\right)  \cdot\left(
n-3\right)  \cdot\cdots\cdot1=1\cdot2\cdot\cdots\cdot n=n!
\end{align*}
}. Hence, there are precisely $n!$ permutations of $\left\{  1,2,\ldots
,n\right\}  $. As we have said, this proves Proposition \ref{prop.permat.n!}.
\end{proof}

Here is a further property of permutation matrices constructed out of permutations:

\begin{proposition}
\label{prop.permat.prod-of-perms}Let $n\in\mathbb{N}$.

\textbf{(a)} If $x$ and $y$ are two permutations of $\left\{  1,2,\ldots
,n\right\}  $, then $P_{x}P_{y}=P_{y\circ x}$.

\textbf{(b)} If $w$ is a permutation of $\left\{  1,2,\ldots,n\right\}  $,
then $\left(  P_{w}\right)  ^{-1}=P_{w^{-1}}$.

\textbf{(c)} We have $P_{\operatorname*{id}\nolimits_{\left\{  1,2,\ldots
,n\right\}  }}=I_{n}$.

\textbf{(d)} Let $u$ and $v$ be two distinct elements of $\left\{
1,2,\ldots,n\right\}  $. Let $\tau_{u,v}$ be the permutation of $\left\{
1,2,\ldots,n\right\}  $ which sends $u$ to $v$, sends $v$ to $u$, and sends
each other element of $\left\{  1,2,\ldots,n\right\}  $ to itself. (For
example, if $n=7$, then $\tau_{2,5}$ sends $1,2,3,4,5,6,7$ to $1,5,3,4,2,6,7$,
respectively.) Then, $P_{\tau_{u,v}}=T_{u,v}$.
\end{proposition}

We leave the easy proof to the reader.

As we have said, Proposition \ref{prop.permat.permat-from-perm} shows that the
map%
\begin{align*}
\left\{  \text{permutations of }\left\{  1,2,\ldots,n\right\}  \right\}   &
\rightarrow\left\{  \text{permutation matrices of size }n\times n\right\}  ,\\
w  &  \mapsto P_{w}%
\end{align*}
is a bijection. This map provides a \textquotedblleft
dictionary\textquotedblright\ between permutations of $\left\{  1,2,\ldots
,n\right\}  $ and permutation matrices of size $n\times n$ (so to speak).
Proposition \ref{prop.permat.prod-of-perms} \textbf{(a)} shows that (under
this \textquotedblleft dictionary\textquotedblright) composition of
permutations \textquotedblleft corresponds\textquotedblright\ to
multiplication of permutation matrices (except that the order of the factors
is reversed). Proposition \ref{prop.permat.prod-of-perms} \textbf{(b)} shows
that the inverse of a permutation gets \textquotedblleft
translated\textquotedblright\ into the inverse of the corresponding
permutation matrix. Proposition \ref{prop.permat.prod-of-perms} \textbf{(c)}
shows that the identity permutation $\operatorname*{id}\nolimits_{\left\{
1,2,\ldots,n\right\}  }$ \textquotedblleft corresponds\textquotedblright\ to
the identity matrix $I_{n}$. Finally, Proposition
\ref{prop.permat.prod-of-perms} \textbf{(d)} shows that the permutations
$\tau_{u,v}$ \textquotedblleft correspond\textquotedblright\ to the swapping
matrices $T_{u,v}$. Using this \textquotedblleft dictionary\textquotedblright,
we can translate theorems about permutations into theorems about permutation
matrices, and vice versa. For instance, the translation of Proposition
\ref{prop.permat.Tkk+1} into the language of permutations is the following fact:

\begin{proposition}
\label{prop.permat.Tkk+1.perm}Let $n\in\mathbb{N}$. A \textit{simple
transposition} will mean a permutation in $S_{n}$ that has the form
$\tau_{k,k+1}$ for some $k\in\left\{  1,2,\ldots,n-1\right\}  $ (where
$\tau_{k,k+1}$ is defined as in Proposition \ref{prop.permat.prod-of-perms}
\textbf{(d)}).

Each permutation of $\left\{  1,2,\ldots,n\right\}  $ is a product of simple transpositions.
\end{proposition}

Arguably, in this translated form, this proposition is rather obvious. It
merely says that any rearrangement of the numbers $1,2,\ldots,n$ can be
obtained from the list $\left(  1,2,\ldots,n\right)  $ by repeatedly swapping
adjacent entries. This should be quite clear, at least intuitively
(\href{https://en.wikipedia.org/wiki/Bubble_sort}{the \textquotedblleft bubble
sort\textquotedblright\ algorithm} provides a way to obtain the list $\left(
1,2,\ldots,n\right)  $ from our rearrangement by repeatedly swapping adjacent
entries; now, all that remains to be done is to perform these swaps backwards
in order to get from $\left(  1,2,\ldots,n\right)  $ to the rearrangement).

Next, let us prove Proposition \ref{prop.permat.laction}. Let us first state a
more precise version of this proposition:

\begin{proposition}
\label{prop.permat.laction.Pw}Let $n\in\mathbb{N}$ and $m\in\mathbb{N}$. Let
$w$ be a permutation of $\left\{  1,2,\ldots,n\right\}  $. Let $P$ be the
$n\times n$-matrix $P_{w}$. Let $C$ be an $n\times m$-matrix. Then,%
\[
\operatorname*{row}\nolimits_{i}\left(  PC\right)  =\operatorname*{row}%
\nolimits_{w\left(  i\right)  }C\ \ \ \ \ \ \ \ \ \ \text{for every }%
i\in\left\{  1,2,\ldots,n\right\}  .
\]

\end{proposition}

\begin{proof}
[Proof of Proposition \ref{prop.permat.laction.Pw}.]We have $P=P_{w}=\left(
\delta_{w\left(  i\right)  ,j}\right)  _{1\leq i\leq n,\ 1\leq j\leq n}$ (by
the definition of $P_{w}$). Thus,%
\begin{equation}
P_{i,j}=\delta_{w\left(  i\right)  ,j}\ \ \ \ \ \ \ \ \ \ \text{for all }%
i\in\left\{  1,2,\ldots,n\right\}  \text{ and }j\in\left\{  1,2,\ldots
,n\right\}  . \label{pf.prop.permat.laction.Pw.Pij=}%
\end{equation}


Now, Proposition \ref{prop.AB} (applied to $n$, $m$, $P$ and $C$ instead of
$m$, $p$, $A$ and $B$) shows that%
\[
\left(  PC\right)  _{i,j}=\sum_{k=1}^{n}P_{i,k}C_{k,j}%
\ \ \ \ \ \ \ \ \ \ \text{for all }i\in\left\{  1,2,\ldots,n\right\}  \text{
and }j\in\left\{  1,2,\ldots,m\right\}  .
\]
Hence, for all $i\in\left\{  1,2,\ldots,n\right\}  $ and $j\in\left\{
1,2,\ldots,m\right\}  $, we have%
\begin{align}
\left(  PC\right)  _{i,j}  &  =\sum_{k=1}^{n}\underbrace{P_{i,k}%
}_{\substack{=\delta_{w\left(  i\right)  ,k}\\\text{(by
(\ref{pf.prop.permat.laction.Pw.Pij=}), applied}\\\text{to }k\text{ instead of
}j\text{)}}}C_{k,j}=\sum_{k=1}^{n}\underbrace{\delta_{w\left(  i\right)
,k}C_{k,j}}_{=C_{k,j}\delta_{w\left(  i\right)  ,k}}=\sum_{k=1}^{n}%
C_{k,j}\underbrace{\delta_{w\left(  i\right)  ,k}}_{\substack{=\delta
_{k,w\left(  i\right)  }\\\text{(since }\delta_{u,v}=\delta_{v,u}\\\text{for
any two}\\\text{objects }u\text{ and }v\text{)}}}\nonumber\\
&  =\sum_{k=1}^{n}C_{k,j}\delta_{k,w\left(  i\right)  }=C_{w\left(  i\right)
,j} \label{pf.prop.permat.laction.Pw.PCij=}%
\end{align}
(by Proposition \ref{prop.sum.delta}, applied to $p=1$, $q=n$, $r=w\left(
i\right)  $ and $a_{k}=C_{k,j}$).

Now, let $i\in\left\{  1,2,\ldots,n\right\}  $. Then, the definition of
$\operatorname*{row}\nolimits_{w\left(  i\right)  }C$ yields
$\operatorname*{row}\nolimits_{w\left(  i\right)  }C=\left(  C_{w\left(
i\right)  ,y}\right)  _{1\leq x\leq1,\ 1\leq y\leq m}$. But the definition of
$\operatorname*{row}\nolimits_{i}\left(  PC\right)  $ yields%
\[
\operatorname*{row}\nolimits_{i}\left(  PC\right)  =\left(
\underbrace{\left(  PC\right)  _{i,y}}_{\substack{=C_{w\left(  i\right)
,y}\\\text{(by (\ref{pf.prop.permat.laction.Pw.PCij=}), applied to
}j=y\text{)}}}\right)  _{1\leq x\leq1,\ 1\leq y\leq m}=\left(  C_{w\left(
i\right)  ,y}\right)  _{1\leq x\leq1,\ 1\leq y\leq m}.
\]
Comparing this with $\operatorname*{row}\nolimits_{w\left(  i\right)
}C=\left(  C_{w\left(  i\right)  ,y}\right)  _{1\leq x\leq1,\ 1\leq y\leq m}$,
we obtain $\operatorname*{row}\nolimits_{i}\left(  PC\right)
=\operatorname*{row}\nolimits_{w\left(  i\right)  }C$. This proves Proposition
\ref{prop.permat.laction.Pw}.
\end{proof}

\begin{proof}
[Proof of Proposition \ref{prop.permat.laction}.]Proposition
\ref{prop.permat.permat-from-perm} \textbf{(b)} shows that there exists a
\textbf{unique} permutation $w$ of $\left\{  1,2,\ldots,n\right\}  $ such that
$P=P_{w}$. Consider this $w$. Proposition \ref{prop.permat.laction.Pw} yields
that%
\[
\operatorname*{row}\nolimits_{i}\left(  PC\right)  =\operatorname*{row}%
\nolimits_{w\left(  i\right)  }C\ \ \ \ \ \ \ \ \ \ \text{for every }%
i\in\left\{  1,2,\ldots,n\right\}  .
\]
In other words, the $i$-th row of $PC$ equals the $w\left(  i\right)  $-th row
of $C$ for each $i\in\left\{  1,2,\ldots,n\right\}  $. Since $w$ is a
permutation of $\left\{  1,2,\ldots,n\right\}  $, this shows that the rows of
$PC$ are the rows of $C$, rearranged. Hence, the matrix $PC$ can be obtained
from $C$ by rearranging the rows. This proves Proposition
\ref{prop.permat.laction}.
\end{proof}

\begin{noncompile}
\begin{todo}
(*) Permutation matrices are products of $T_{k,k+1}$'s. (More detail?)
\end{todo}

\begin{todo}
Permutation matrices act by permuting rows: If $P$ is a permutation matrix and
$C$ is any matrix, then $PC$ has the same rows as $C$ but (possibly) in a
different order. Similarly, $CP$ has the same columns as $C$ but (possibly) in
a different order.
\end{todo}
\end{noncompile}

\subsection{The standard row operations}

I shall now introduce the standard row operations -- certain transformations
acting on matrices, changing some of their rows while leaving others
unchanged. We have already encountered them in some proofs above (for
instance, the \textquotedblleft downward row additions\textquotedblright\ in
the proof of Theorem \ref{thm.triangular.Alamuv} were one type of row
operations), but now we shall give them the systematic treatment they deserve
and see them collaborate on producing in Gaussian elimination.

\begin{definition}
For the rest of Chapter \ref{sect.gauss}, we shall use the word
\textquotedblleft transformation\textquotedblright\ in a rather specific
meaning: Let $n\in\mathbb{N}$. A \textit{transformation of }$n$\textit{-rowed
matrices} will mean a map%
\[
\left\{  \text{matrices with }n\text{ rows}\right\}  \rightarrow\left\{
\text{matrices with }n\text{ rows}\right\}  .
\]
In other words, a \textit{transformation of }$n$\textit{-rowed matrices} is a
map that transforms each matrix with $n$ rows into a new matrix with $n$ rows.

We shall use the arrow notation for transformations: If $\mathcal{O}$ is some
transformation of $n$-rowed matrices, and if $C$ and $D$ are two matrices,
then we will use the notation \textquotedblleft$C\overset{\mathcal{O}%
}{\longrightarrow}D$\textquotedblright\ when we want to say that the
transformation $\mathcal{O}$ transforms $C$ into $D$. For example, if
$\mathcal{M}$ denotes the transformation of $2$-rowed matrices that multiplies
each entry of a matrix by $3$, then $\left(
\begin{array}
[c]{ccc}%
a & b & c\\
a^{\prime} & b^{\prime} & c^{\prime}%
\end{array}
\right)  \overset{\mathcal{M}}{\longrightarrow}\left(
\begin{array}
[c]{ccc}%
3a & 3b & 3c\\
3a^{\prime} & 3b^{\prime} & 3c^{\prime}%
\end{array}
\right)  $.

More generally, if $\mathcal{O}_{1},\mathcal{O}_{2},\ldots,\mathcal{O}_{m}$ is
a sequence of transformations of $n$-rowed matrices, and if $C_{0}%
,C_{1},\ldots,C_{m}$ are some matrices, then we will use the notation
\textquotedblleft$C_{0}\overset{\mathcal{O}_{1}}{\longrightarrow}%
C_{1}\overset{\mathcal{O}_{2}}{\longrightarrow}\cdots\overset{\mathcal{O}%
_{m}}{\longrightarrow}C_{m}$\textquotedblright\ when we want to say that the
transformation $\mathcal{O}_{i}$ transforms $C_{i-1}$ into $C_{i}$ for each
$i\in\left\{  1,2,\ldots,m\right\}  $. Examples for this will appear below
once we have defined some actual transformations.

We will also sometimes draw arrows in two directions. Namely: If $\mathcal{O}$
and $\mathcal{P}$ are two transformations of $n$-rowed matrices, and if $C$
and $D$ are two matrices, then we will use the notation \textquotedblleft%
$C\underset{\mathcal{P}}{\overset{\mathcal{O}}{%
%TCIMACRO{\QTATOP{\longrightarrow}{\longleftarrow}}%
%BeginExpansion
\genfrac{}{}{0pt}{1}{\longrightarrow}{\longleftarrow}%
%EndExpansion
}}D$\textquotedblright\ when we want to say that the transformation
$\mathcal{O}$ transforms $C$ into $D$ while the transformation $\mathcal{P}$
transforms $D$ into $C$. This will often happen when two transformations
$\mathcal{O}$ and $\mathcal{P}$ are inverse to each other (i.e., each of them
undoes the other).
\end{definition}

\begin{definition}
\label{def.row-ops.row-add}Let $n\in\mathbb{N}$. Let $u$ and $v$ be two
distinct elements of $\left\{  1,2,\ldots,n\right\}  $. Let $\lambda$ be a number.

Consider the transformation which transforms an $n\times m$-matrix $C$ (for
some $m\in\mathbb{N}$) into the product $A_{u,v}^{\lambda}C$, where
$A_{u,v}^{\lambda}$ is the $\lambda$-addition matrix as defined in Definition
\ref{def.Alamuv}. It is the transformation that modifies an $n\times m$-matrix
$C$ by adding $\lambda\operatorname*{row}\nolimits_{v}C$ to the $u$-th row
(according to Proposition \ref{prop.Alamuv.laction}). This transformation will
be called \textit{the row addition }$A_{u,v}^{\lambda}$. (We are using the
same symbol $A_{u,v}^{\lambda}$ for the transformation and for the $\lambda
$-addition matrix, since they are so closely related; nevertheless, they are
not one and the same thing. I hope that the reader will be able to keep them apart.)

The row addition $A_{u,v}^{\lambda}$ is called a \textit{downward row
addition} if $u>v$, and is called an \textit{upward row addition} if $u<v$.

Note that two matrices $C$ and $D$ (with $n$ rows each) satisfy
$C\overset{A_{u,v}^{\lambda}}{\longrightarrow}D$ if and only if they satisfy
$D=A_{u,v}^{\lambda}C$. (This is because the row addition $A_{u,v}^{\lambda}$
transforms each $n\times m$-matrix $C$ into $A_{u,v}^{\lambda}C$.)

Note that the row addition $A_{u,v}^{\lambda}$ is inverse to the row addition
$A_{u,v}^{-\lambda}$ (since the matrices $A_{u,v}^{\lambda}$ and
$A_{u,v}^{-\lambda}$ are inverse). Hence, if two matrices $C$ and $D$ (with
$n$ rows each) satisfy $C\overset{A_{u,v}^{\lambda}}{\longrightarrow}D$, then
they satisfy $C\underset{A_{u,v}^{-\lambda}}{\overset{A_{u,v}^{\lambda}}{%
%TCIMACRO{\QTATOP{\longrightarrow}{\longleftarrow}}%
%BeginExpansion
\genfrac{}{}{0pt}{1}{\longrightarrow}{\longleftarrow}%
%EndExpansion
}}D$.
\end{definition}

\begin{example}
The row addition $A_{1,3}^{5}$ modifies a $3\times m$-matrix $C$ by adding
$5\operatorname*{row}\nolimits_{3}C$ to the $1$-st row of $C$. Thus, this row
addition transforms any $3\times4$-matrix $\left(
\begin{array}
[c]{cccc}%
a_{1} & b_{1} & c_{1} & d_{1}\\
a_{2} & b_{2} & c_{2} & d_{2}\\
a_{3} & b_{3} & c_{3} & d_{3}%
\end{array}
\right)  $ into $\left(
\begin{array}
[c]{cccc}%
a_{1}+5a_{3} & b_{1}+5b_{3} & c_{1}+5c_{3} & d_{1}+5d_{3}\\
a_{2} & b_{2} & c_{2} & d_{2}\\
a_{3} & b_{3} & c_{3} & d_{3}%
\end{array}
\right)  $. Using the arrow notation, we can write this as follows:%
\[
\left(
\begin{array}
[c]{cccc}%
a_{1} & b_{1} & c_{1} & d_{1}\\
a_{2} & b_{2} & c_{2} & d_{2}\\
a_{3} & b_{3} & c_{3} & d_{3}%
\end{array}
\right)  \overset{A_{1,3}^{5}}{\longrightarrow}\left(
\begin{array}
[c]{cccc}%
a_{1}+5a_{3} & b_{1}+5b_{3} & c_{1}+5c_{3} & d_{1}+5d_{3}\\
a_{2} & b_{2} & c_{2} & d_{2}\\
a_{3} & b_{3} & c_{3} & d_{3}%
\end{array}
\right)  .
\]
For example, $\left(
\begin{array}
[c]{cccc}%
1 & 3 & 2 & 1\\
0 & 4 & -1 & 1\\
2 & -1 & 3 & 1
\end{array}
\right)  \overset{A_{1,3}^{5}}{\longrightarrow}\left(
\begin{array}
[c]{cccc}%
11 & -2 & 17 & 6\\
0 & 4 & -1 & 1\\
2 & -1 & 3 & 1
\end{array}
\right)  $.
\end{example}

\begin{definition}
\label{def.row-ops.row-scale}Let $n\in\mathbb{N}$. Let $u\in\left\{
1,2,\ldots,n\right\}  $. Let $\lambda$ be a nonzero number.

Consider the transformation which transforms an $n\times m$-matrix $C$ (for
some $m\in\mathbb{N}$) into the product $S_{u}^{\lambda}C$, where
$S_{u}^{\lambda}$ is the $\lambda$-scaling matrix as defined in Definition
\ref{def.Slamu}. It is the transformation that modifies an $n\times m$-matrix
$C$ by scaling the $u$-th row by $\lambda$ (according to Proposition
\ref{prop.Slamu.laction}). This transformation will be called \textit{the row
scaling }$S_{u}^{\lambda}$. (We are using the same symbol $S_{u}^{\lambda}$
for the transformation and for the $\lambda$-scaling matrix. Again, this
should not lead to confusion.)

Note that two matrices $C$ and $D$ (with $n$ rows each) satisfy
$C\overset{S_{u}^{\lambda}}{\longrightarrow}D$ if and only if they satisfy
$D=S_{u}^{\lambda}C$. (This is because the row scaling $S_{u}^{\lambda}$
transforms each $n\times m$-matrix $C$ into $S_{u}^{\lambda}C$.)

Note that the row scaling $S_{u}^{\lambda}$ is inverse to the row scaling
$S_{u}^{1/\lambda}$ (since the matrices $S_{u}^{\lambda}$ and $S_{u}%
^{1/\lambda}$ are inverse). Hence, if two matrices $C$ and $D$ (with $n$ rows
each) satisfy $C\overset{S_{u}^{\lambda}}{\longrightarrow}D$, then they
satisfy $C\underset{S_{u}^{1/\lambda}}{\overset{S_{u}^{\lambda}}{%
%TCIMACRO{\QTATOP{\longrightarrow}{\longleftarrow}}%
%BeginExpansion
\genfrac{}{}{0pt}{1}{\longrightarrow}{\longleftarrow}%
%EndExpansion
}}D$.
\end{definition}

\begin{example}
The row scaling $S_{2}^{5}$ modifies a $3\times m$-matrix $C$ by scaling the
$2$-nd row by $5$. Thus, this row scaling transforms any $3\times2$-matrix
$\left(
\begin{array}
[c]{cc}%
a_{1} & b_{1}\\
a_{2} & b_{2}\\
a_{3} & b_{3}%
\end{array}
\right)  $ into $\left(
\begin{array}
[c]{cc}%
a_{1} & b_{1}\\
5a_{2} & 5b_{2}\\
a_{3} & b_{3}%
\end{array}
\right)  $.
\end{example}

\begin{definition}
\label{def.row-ops.row-swap}Let $n\in\mathbb{N}$. Let $u$ and $v$ be two
distinct elements of $\left\{  1,2,\ldots,n\right\}  $.

Consider the transformation which transforms an $n\times m$-matrix $C$ (for
some $m\in\mathbb{N}$) into the product $T_{u,v}C$, where $T_{u,v}$ is the
swapping matrix as defined in Definition \ref{def.Tuv}. It is the
transformation that modifies an $n\times m$-matrix $C$ by swapping the $u$-th
row with the $v$-th row (according to Proposition \ref{prop.Tuv.laction}).
This transformation will be called \textit{the row swap }$T_{u,v}$. (We are
using the same symbol $T_{u,v}$ for the transformation and for the swapping matrix.)

Note that two matrices $C$ and $D$ (with $n$ rows each) satisfy
$C\overset{T_{u,v}}{\longrightarrow}D$ if and only if they satisfy
$D=T_{u,v}C$. (This is because the row swap $T_{u,v}$ transforms each $n\times
m$-matrix $C$ into $T_{u,v}C$.)

Note that the row swap $T_{u,v}$ is inverse to itself (since the matrices
$T_{u,v}$ and $T_{u,v}$ are inverse). Hence, if two matrices $C$ and $D$ (with
$n$ rows each) satisfy $C\overset{T_{u,v}}{\longrightarrow}D$, then they
satisfy $C\underset{T_{u,v}}{\overset{T_{u,v}}{%
%TCIMACRO{\QTATOP{\longrightarrow}{\longleftarrow}}%
%BeginExpansion
\genfrac{}{}{0pt}{1}{\longrightarrow}{\longleftarrow}%
%EndExpansion
}}D$.
\end{definition}

\begin{example}
The row swap $T_{1,3}$ modifies a $4\times m$-matrix $C$ by swapping the
$1$-st row of $C$ with the $3$-rd row of $C$. Thus, this row swap transforms
any $4\times2$-matrix $\left(
\begin{array}
[c]{cc}%
a_{1} & b_{1}\\
a_{2} & b_{2}\\
a_{3} & b_{3}\\
a_{4} & b_{4}%
\end{array}
\right)  $ into $\left(
\begin{array}
[c]{cc}%
a_{3} & b_{3}\\
a_{2} & b_{2}\\
a_{1} & b_{1}\\
a_{4} & b_{4}%
\end{array}
\right)  $. Using the arrow notation, we can write this as follows:%
\[
\left(
\begin{array}
[c]{cc}%
a_{1} & b_{1}\\
a_{2} & b_{2}\\
a_{3} & b_{3}\\
a_{4} & b_{4}%
\end{array}
\right)  \overset{T_{1,3}}{\longrightarrow}\left(
\begin{array}
[c]{cc}%
a_{3} & b_{3}\\
a_{2} & b_{2}\\
a_{1} & b_{1}\\
a_{4} & b_{4}%
\end{array}
\right)  .
\]
For example, $\left(
\begin{array}
[c]{cc}%
1 & 2\\
3 & 4\\
5 & 6\\
7 & 8
\end{array}
\right)  \overset{T_{1,3}}{\longrightarrow}\left(
\begin{array}
[c]{cc}%
5 & 6\\
3 & 4\\
1 & 2\\
7 & 8
\end{array}
\right)  $.
\end{example}

\begin{definition}
Let $n\in\mathbb{N}$. The \textit{standard row operations} (on matrices with
$n$ rows) are:

\begin{itemize}
\item row additions (i.e., transformations of the form $A_{u,v}^{\lambda}$);

\item row scalings (i.e., transformations of the form $S_{u}^{\lambda}$);

\item row swaps (i.e., transformations of the form $T_{u,v}$).
\end{itemize}
\end{definition}

These row operations allow us to \textquotedblleft reduce\textquotedblright%
\ any matrix to a certain simple form -- called a row echelon form -- that has
many zeroes (in a sense, it is close to upper-triangular, although the two
notions are not exactly the same) and that allows for easily solving linear systems.

\subsection{\label{sect.gauss.row-echelon}Row-echelon matrices}

\begin{definition}
Let $n\in\mathbb{N}$ and $m\in\mathbb{N}$. An $n\times m$-matrix $A$ is said
to be \emph{zero} if and only if $A=0_{n\times m}$. In other words, an
$n\times m$-matrix $A$ is said to be \textit{zero} if and only if all entries
of $A$ are zero.

An $n\times m$-matrix $A$ is said to be \emph{nonzero} if $A$ is not zero.
(This does not mean that all entries of $A$ are nonzero!)
\end{definition}

\begin{definition}
\label{def.pivot}Let $m\in\mathbb{N}$. Let $v=\left(  v_{1},v_{2},\ldots
,v_{m}\right)  $ be a $1\times m$-matrix (i.e., a row vector of size $m$). If
$v$ is nonzero (i.e., if not all entries of $v$ are zero), then the smallest
$i\in\left\{  1,2,\ldots,m\right\}  $ satisfying $v_{i}\neq0$ is called the
\emph{pivot index} of $v$ and will be denoted by $\operatorname*{pivind}v$.
Furthermore, the value of $v_{i}$ for this smallest $i$ is called the
\emph{pivot entry} of $v$. (Thus, the pivot entry of $v$ is the first nonzero
entry of $v$.)
\end{definition}

\begin{example}
The row vector $\left(  0,6,7,0,9\right)  $ has pivot index $2$ (because if we
write it as $\left(  v_{1},v_{2},v_{3},v_{4},v_{5}\right)  $, then the
smallest $i\in\left\{  1,2,3,4,5\right\}  $ satisfying $v_{i}\neq0$ is $2$)
and pivot entry $6$.

The row vector $\left(  0,0,-1,-1,0\right)  $ has pivot index $3$ and pivot
entry $-1$.

The row vector $\left(  0,0,0,0,0\right)  $ has no pivot index (since it is zero).
\end{example}

\begin{definition}
\label{def.pivot-cells}Let $n\in\mathbb{N}$ and $m\in\mathbb{N}$. Let $A$ be
an $n\times m$-matrix.

\textbf{(a)} Let $i\in\left\{  1,2,\ldots,n\right\}  $ be such that
$\operatorname*{row}\nolimits_{i}A$ is nonzero. Then, the cell $\left(
i,\operatorname*{pivind}\left(  \operatorname*{row}\nolimits_{i}A\right)
\right)  $ is called the \textit{pivot cell} in the $i$-th row of $A$. (In
words, this is the leftmost cell in the $i$-th row of $A$ that has a nonzero entry.)

\textbf{(b)} The \textit{pivot cells} of $A$ are the pivot cells in the $i$-th
row of $A$, where $i$ ranges over all elements of $\left\{  1,2,\ldots
,n\right\}  $ for which $\operatorname*{row}\nolimits_{i}A$ is nonzero. The
\textit{pivot entries} of $A$ are the entries of $A$ in the pivot cells. (In
other words, the pivot entries of $A$ are the leftmost nonzero entries of all
nonzero rows of $A$.)
\end{definition}

\begin{example}
The pivot cells of the matrix $\left(
\begin{array}
[c]{ccc}%
7 & 3 & 1\\
0 & 0 & 0\\
0 & 0 & 4\\
0 & -1 & 2
\end{array}
\right)  $ are $\left(  1,1\right)  $, $\left(  3,3\right)  $ and $\left(
4,2\right)  $, and the respective pivot entries are $7$, $4$ and $-1$. There
is no pivot cell in the $2$-nd row, since this row is zero.
\end{example}

\begin{definition}
\label{def.row-echelon}Let $n\in\mathbb{N}$ and $m\in\mathbb{N}$. Let $A$ be
an $n\times m$-matrix.

We say that $A$ is a \textit{row-echelon matrix} if for each $i\in\left\{
1,2,\ldots,n-1\right\}  $, the following holds:

\begin{itemize}
\item \textbf{either} $\operatorname*{row}\nolimits_{i+1}A$ is zero,

\item \textbf{or} both rows $\operatorname*{row}\nolimits_{i}A$ and
$\operatorname*{row}\nolimits_{i+1}A$ are nonzero, and satisfy
$\operatorname*{pivind}\left(  \operatorname*{row}\nolimits_{i}A\right)
<\operatorname*{pivind}\left(  \operatorname*{row}\nolimits_{i+1}A\right)  $
(that is, the pivot cell in the $i$-th row of $A$ is strictly further left
than the pivot cell in the $\left(  i+1\right)  $-th row of $A$).
\end{itemize}

Instead of saying that \textquotedblleft$A$ is a row-echelon
matrix\textquotedblright, it is customary to say that \textquotedblleft$A$ is
in row-echelon form\textquotedblright.
\end{definition}

\begin{example}
\label{exam.row-echelon}\textbf{(a)} Let $A$ be the $3\times4$-matrix $\left(
%
\begin{array}
[c]{cccc}%
-2 & 3 & 4 & 5\\
0 & 0 & 2 & 1\\
0 & 0 & 0 & 3
\end{array}
\right)  $. Then, $A$ is a row-echelon matrix. In fact, all rows of $A$ are
nonzero, and each $i\in\left\{  1,2\right\}  $ satisfies
$\operatorname*{pivind}\left(  \operatorname*{row}\nolimits_{i}A\right)
<\operatorname*{pivind}\left(  \operatorname*{row}\nolimits_{i+1}A\right)  $
(indeed, we have $\operatorname*{pivind}\left(  \operatorname*{row}%
\nolimits_{1}A\right)  =1$, $\operatorname*{pivind}\left(  \operatorname*{row}%
\nolimits_{2}A\right)  =3$ and $\operatorname*{pivind}\left(
\operatorname*{row}\nolimits_{3}A\right)  =4$). The pivot cells of $A$ are
$\left(  1,1\right)  $, $\left(  2,3\right)  $ and $\left(  3,4\right)  $.

\textbf{(b)} Let $A$ be the $3\times4$-matrix $\left(
\begin{array}
[c]{cccc}%
-2 & 3 & 4 & 5\\
0 & 0 & 2 & 1\\
0 & 0 & 4 & 3
\end{array}
\right)  $ instead. Then, $A$ is not a row-echelon matrix. Indeed, $i=2$ fails
to satisfy the condition in Definition \ref{def.row-echelon} (because
$\operatorname*{pivind}\left(  \operatorname*{row}\nolimits_{2}A\right)  =3$
and $\operatorname*{pivind}\left(  \operatorname*{row}\nolimits_{3}A\right)
=3$, but $\operatorname*{row}\nolimits_{3}A$ is nonzero). The pivot cells of
$A$ are $\left(  1,1\right)  $, $\left(  2,3\right)  $ and $\left(
3,3\right)  $.

\textbf{(c)} Let $A$ be the $5\times4$-matrix $\left(
\begin{array}
[c]{cccc}%
7 & 3 & 1 & 0\\
0 & -1 & 2 & 2\\
0 & 0 & 3 & 0\\
0 & 0 & 0 & 0\\
0 & 0 & 0 & 0
\end{array}
\right)  $ instead. Then, $A$ is a row-echelon matrix. In fact, each
$i\in\left\{  1,2\right\}  $ satisfies $\operatorname*{pivind}\left(
\operatorname*{row}\nolimits_{i}A\right)  <\operatorname*{pivind}\left(
\operatorname*{row}\nolimits_{i+1}A\right)  $ (because $\operatorname*{pivind}%
\left(  \operatorname*{row}\nolimits_{1}A\right)  =1$, $\operatorname*{pivind}%
\left(  \operatorname*{row}\nolimits_{2}A\right)  =2$ and
$\operatorname*{pivind}\left(  \operatorname*{row}\nolimits_{3}A\right)  =3$),
whereas for each $i\in\left\{  3,4\right\}  $, the vector $\operatorname*{row}%
\nolimits_{i+1}A$ is zero (since $\operatorname*{row}\nolimits_{4}A$ and
$\operatorname*{row}\nolimits_{5}A$ are zero). The pivot cells of $A$ are
$\left(  1,1\right)  $, $\left(  2,2\right)  $ and $\left(  3,3\right)  $.

\textbf{(d)} Let $A$ be the $3\times4$-matrix $\left(
\begin{array}
[c]{cccc}%
7 & 3 & 1 & 0\\
0 & 0 & 0 & 0\\
0 & 0 & 3 & 0
\end{array}
\right)  $ instead. Then, $A$ is not a row-echelon matrix. Indeed, $i=2$ fails
to satisfy the condition in Definition \ref{def.row-echelon} (because
$\operatorname*{row}\nolimits_{i}A$ is zero, but $\operatorname*{row}%
\nolimits_{i+1}A$ is nonzero). The pivot cells of $A$ are $\left(  1,1\right)
$ and $\left(  3,3\right)  $.

\textbf{(e)} Let $A$ be an invertibly upper-triangular $n\times n$-matrix (for
some $n\in\mathbb{N}$) instead. (See Definition \ref{def.triangular.uni-up}
\textbf{(b)} for the meaning of \textquotedblleft invertibly
upper-triangular\textquotedblright.) Then, $A$ is a row-echelon matrix. In
fact, each $i\in\left\{  1,2,\ldots,n-1\right\}  $ satisfies
$\operatorname*{pivind}\left(  \operatorname*{row}\nolimits_{i}A\right)
<\operatorname*{pivind}\left(  \operatorname*{row}\nolimits_{i+1}A\right)  $
(because $\operatorname*{pivind}\left(  \operatorname*{row}\nolimits_{k}%
A\right)  =k$ for each $k\in\left\{  1,2,\ldots,n\right\}  $). The pivot cells
of $A$ are $\left(  1,1\right)  ,\left(  2,2\right)  ,\ldots,\left(
n,n\right)  $.

\textbf{(f)} On the other hand, not every upper-triangular $n\times n$-matrix
is a row-echelon matrix. For instance, $\left(
\begin{array}
[c]{ccc}%
1 & 2 & 3\\
0 & 0 & 1\\
0 & 0 & 1
\end{array}
\right)  $ is not a row-echelon matrix, since the pivot indices of its $2$-nd
and $3$-rd rows are equal.
\end{example}

From the examples just given, it shouldn't be hard to build an intuition for
row-echelon matrices. Roughly speaking, a matrix $A$ is row-echelon if and
only if

\begin{itemize}
\item a nonzero row of $A$ cannot follow a zero row of $A$, and

\item the first nonzero entry in each nonzero row of $A$ is strictly further
right than that in the row before.
\end{itemize}

Here are some first simple observations:

\begin{proposition}
\label{prop.row-echelon.pivrows}Let $A$ be an $n\times m$-matrix. Assume that
$A$ is a row-echelon matrix. Let $k$ be the number of nonzero rows of $A$.

\textbf{(a)} The first $k$ rows of $A$ are nonzero, whereas the last $n-k$
rows of $A$ are zero.

\textbf{(b)} The pivot cells of $A$ are the $k$ cells
\[
\left(  1,\operatorname*{pivind}\left(  \operatorname*{row}\nolimits_{1}%
A\right)  \right)  ,\left(  2,\operatorname*{pivind}\left(
\operatorname*{row}\nolimits_{2}A\right)  \right)  ,\ldots,\left(
k,\operatorname*{pivind}\left(  \operatorname*{row}\nolimits_{k}A\right)
\right)  .
\]


\textbf{(c)} We have
\[
\operatorname*{pivind}\left(  \operatorname*{row}\nolimits_{1}A\right)
<\operatorname*{pivind}\left(  \operatorname*{row}\nolimits_{2}A\right)
<\cdots<\operatorname*{pivind}\left(  \operatorname*{row}\nolimits_{k}%
A\right)  .
\]

\end{proposition}

\begin{proof}
[Proof of Proposition \ref{prop.row-echelon.pivrows}.]The matrix $A$ is a
row-echelon matrix. Thus, the condition in Definition \ref{def.row-echelon}
shows that, for each $i\in\left\{  1,2,\ldots,n-1\right\}  $, the following
holds:%
\begin{equation}
\text{if }\operatorname*{row}\nolimits_{i+1}A\text{ is nonzero, then
}\operatorname*{row}\nolimits_{i}A\text{ is nonzero.}
\label{pf.prop.row-echelon.pivrows.ifthen}%
\end{equation}
Hence, a nonzero row of $A$ cannot follow a zero row. Using this fact, it is
easy to see that the nonzero rows of $A$ are \textquotedblleft concentrated at
the top\textquotedblright, i.e., there exists some $\ell\in\left\{
0,1,\ldots,n\right\}  $ such that the first $\ell$ rows of $A$ are nonzero
whereas the last $n-\ell$ rows of $A$ are zero. Consider this $\ell$. (Note
that $\ell=0$ if all rows of $A$ are zero, and $\ell=n$ if all rows of $A$ are
nonzero.) Clearly, the number of nonzero rows of $A$ is $\ell$. But we have
denoted this number by $k$. Thus, $\ell=k$. So we conclude that the first $k$
rows of $A$ are nonzero whereas the last $n-k$ rows of $A$ are zero (since the
first $\ell$ rows of $A$ are nonzero whereas the last $n-\ell$ rows of $A$ are
zero). This proves Proposition \ref{prop.row-echelon.pivrows} \textbf{(a)}%
.\footnote{If you find this proof insufficiently rigorous, you can argue as
follows instead: If all rows of $A$ are zero, then $k=0$ (since $k$ is the
number of nonzero rows of $A$), and therefore Proposition
\ref{prop.row-echelon.pivrows} \textbf{(a)} is obvious in this case. Hence, we
WLOG assume that $k\neq0$. Thus, there exists at least one nonzero row of $A$.
In other words, there exists at least one $i\in\left\{  1,2,\ldots,n\right\}
$ such that $\operatorname*{row}\nolimits_{i}A$ is nonzero. Let $h$ be the
\textbf{largest} such $i$. Thus, $\operatorname*{row}\nolimits_{h}A$ is
nonzero, but all the rows $\operatorname*{row}\nolimits_{h+1}%
A,\operatorname*{row}\nolimits_{h+2}A,\ldots,\operatorname*{row}%
\nolimits_{n}A$ are zero.
\par
We know that $\operatorname*{row}\nolimits_{h}A$ is nonzero. Hence, by
applying (\ref{pf.prop.row-echelon.pivrows.ifthen}) repeatedly, we can
conclude that all the $h$ rows $\operatorname*{row}\nolimits_{h}%
A,\operatorname*{row}\nolimits_{h-1}A,\operatorname*{row}\nolimits_{h-2}%
A,\ldots,\operatorname*{row}\nolimits_{1}A$ are nonzero. (Strictly speaking,
we are saying that $\operatorname*{row}\nolimits_{h-j}A$ is nonzero for each
$j\in\left\{  0,1,\ldots,h-1\right\}  $; this can be proven by induction over
$j$.) In other words, all the $h$ rows $\operatorname*{row}\nolimits_{1}%
A,\operatorname*{row}\nolimits_{2}A,\ldots,\operatorname*{row}\nolimits_{h}A$
are nonzero (since their order does not matter for us). In other words, the
first $h$ rows of $A$ are nonzero (since the first $h$ rows of $A$ are
$\operatorname*{row}\nolimits_{1}A,\operatorname*{row}\nolimits_{2}%
A,\ldots,\operatorname*{row}\nolimits_{h}A$).
\par
But recall that $\operatorname*{row}\nolimits_{h+1}A,\operatorname*{row}%
\nolimits_{h+2}A,\ldots,\operatorname*{row}\nolimits_{n}A$ are zero. In other
words, the last $n-h$ rows of $A$ are zero (since the last $n-h$ rows of $A$
are $\operatorname*{row}\nolimits_{h+1}A,\operatorname*{row}\nolimits_{h+2}%
A,\ldots,\operatorname*{row}\nolimits_{n}A$ are zero).
\par
Thus, we know that the first $h$ rows of $A$ are nonzero, whereas the last
$n-h$ rows of $A$ are zero. Hence, the number of nonzero rows of $A$ is $h$.
But the number of nonzero rows of $A$ is $k$ (since this is how we defined
$k$). Comparing the previous two sentences, we obtain $h=k$.
\par
But recall that the first $h$ rows of $A$ are nonzero, whereas the last $n-h$
rows of $A$ are zero. Since $h=k$, this rewrites as follows: The first $k$
rows of $A$ are nonzero, whereas the last $n-k$ rows of $A$ are zero. This
proves Proposition \ref{prop.row-echelon.pivrows} \textbf{(a)} again.}

\textbf{(c)} Proposition \ref{prop.row-echelon.pivrows} \textbf{(a)} shows
that the first $k$ rows of $A$ are nonzero. In other words,
$\operatorname*{row}\nolimits_{i+1}A$ is nonzero for each $i\in\left\{
0,1,\ldots,k-1\right\}  $. The condition in Definition \ref{def.row-echelon}
thus shows that each $i\in\left\{  1,2,\ldots,k-1\right\}  $ satisfies
$\operatorname*{pivind}\left(  \operatorname*{row}\nolimits_{i}A\right)
<\operatorname*{pivind}\left(  \operatorname*{row}\nolimits_{i+1}A\right)  $
(since $A$ is a row echelon matrix). In other words,%
\begin{equation}
\operatorname*{pivind}\left(  \operatorname*{row}\nolimits_{1}A\right)
<\operatorname*{pivind}\left(  \operatorname*{row}\nolimits_{2}A\right)
<\cdots<\operatorname*{pivind}\left(  \operatorname*{row}\nolimits_{k}%
A\right)  . \label{pf.prop.row-echelon.pivcells.ineq}%
\end{equation}
This proves Proposition \ref{prop.row-echelon.pivrows} \textbf{(c)}.

\textbf{(b)} Proposition \ref{prop.row-echelon.pivrows} \textbf{(a)} shows
that the first $k$ rows of $A$ are nonzero whereas the last $n-k$ rows of $A$
are zero. Thus, the pivot cells of $A$ are the pivot cells in the first $k$
rows of $A$. In other words, they are the $k$ cells
\[
\left(  1,\operatorname*{pivind}\left(  \operatorname*{row}\nolimits_{1}%
A\right)  \right)  ,\left(  2,\operatorname*{pivind}\left(
\operatorname*{row}\nolimits_{2}A\right)  \right)  ,\ldots,\left(
k,\operatorname*{pivind}\left(  \operatorname*{row}\nolimits_{k}A\right)
\right)  .
\]
This proves Proposition \ref{prop.row-echelon.pivrows} \textbf{(b)}.
\end{proof}

\begin{proposition}
\label{prop.row-echelon.pivcells}Let $A$ be a matrix.

\textbf{(a)} Each row of $A$ has at most one pivot cell.

\textbf{(b)} Assume that $A$ is a row-echelon matrix. Each column of $A$ has
at most one pivot cell.
\end{proposition}

\begin{proof}
[Proof of Proposition \ref{prop.row-echelon.pivcells}.]\textbf{(a)} This is
clear from the definition of a pivot cell. (More precisely, each nonzero row
of $A$ has exactly one pivot cell, whereas a zero row of $A$ has no pivot cell.)

\textbf{(b)} Let $n\in\mathbb{N}$ and $m\in\mathbb{N}$ be such that $A$ is an
$n\times m$-matrix. Let $k$ be the number of nonzero rows of $A$. Proposition
\ref{prop.row-echelon.pivrows} \textbf{(c)} yields%
\begin{equation}
\operatorname*{pivind}\left(  \operatorname*{row}\nolimits_{1}A\right)
<\operatorname*{pivind}\left(  \operatorname*{row}\nolimits_{2}A\right)
<\cdots<\operatorname*{pivind}\left(  \operatorname*{row}\nolimits_{k}%
A\right)  . \label{pf.prop.row-echelon.pivcells.ineq2}%
\end{equation}


Proposition \ref{prop.row-echelon.pivrows} \textbf{(b)} shows that the pivot
cells of $A$ are the $k$ cells
\[
\left(  1,\operatorname*{pivind}\left(  \operatorname*{row}\nolimits_{1}%
A\right)  \right)  ,\left(  2,\operatorname*{pivind}\left(
\operatorname*{row}\nolimits_{2}A\right)  \right)  ,\ldots,\left(
k,\operatorname*{pivind}\left(  \operatorname*{row}\nolimits_{k}A\right)
\right)  .
\]
Therefore, (\ref{pf.prop.row-echelon.pivcells.ineq2}) shows that each of these
cells lies strictly further left than the next. Therefore, these cells lie in
distinct columns. In other words, each column of $A$ has at most one pivot
cell. This proves Proposition \ref{prop.row-echelon.pivcells} \textbf{(b)}.
\end{proof}

The main importance of row-echelon matrices is that when $A$ is a row-echelon
matrix, the equation $Ax=b$ can be solved by a simple method called
\textquotedblleft back-substitution\textquotedblright. Before explaining this
method in full generality, let me give four examples (in increasing order of
complexity):\footnote{Notice that in each of the following examples, $A$ is a
row-echelon matrix. However, the pivot cells differ:
\par
\begin{itemize}
\item In Example \ref{exam.row-echelon.backsub.1}, each row has a pivot cell,
and each column has a pivot cell.
\par
\item In Example \ref{exam.row-echelon.backsub.2}, each row has a pivot cell,
but not each column does.
\par
\item In Example \ref{exam.row-echelon.backsub.3} and in Example
\ref{exam.row-echelon.backsub.4}, not each row has a pivot cell. These two
examples differ in $b$ (specifically, in the entry of $b$ in the row in which
$A$ has no pivot cell).
\end{itemize}
}

\begin{example}
\label{exam.row-echelon.backsub.1}Let $A=\left(
\begin{array}
[c]{ccc}%
4 & 3 & 4\\
0 & -1 & 2\\
0 & 0 & 5
\end{array}
\right)  $ and $b=\left(
\begin{array}
[c]{c}%
3\\
1\\
5
\end{array}
\right)  $. We want to solve the equation $Ax=b$; in other words, we want to
find all column vectors $x$ of size $3$ that satisfy $Ax=b$.

Write the unknown vector $x$ in the form $x=\left(
\begin{array}
[c]{c}%
x_{1}\\
x_{2}\\
x_{3}%
\end{array}
\right)  $. Multiplying the equalities $A=\left(
\begin{array}
[c]{ccc}%
4 & 3 & 4\\
0 & -1 & 2\\
0 & 0 & 5
\end{array}
\right)  $ and $x=\left(
\begin{array}
[c]{c}%
x_{1}\\
x_{2}\\
x_{3}%
\end{array}
\right)  $, we obtain $Ax=\left(
\begin{array}
[c]{ccc}%
4 & 3 & 4\\
0 & -1 & 2\\
0 & 0 & 5
\end{array}
\right)  \left(
\begin{array}
[c]{c}%
x_{1}\\
x_{2}\\
x_{3}%
\end{array}
\right)  =\left(
\begin{array}
[c]{c}%
4x_{1}+3x_{2}+4x_{3}\\
-x_{2}+2x_{3}\\
5x_{3}%
\end{array}
\right)  $ and $b=\left(
\begin{array}
[c]{c}%
3\\
1\\
5
\end{array}
\right)  $. Hence, the equation $Ax=b$ rewrites as $\left(
\begin{array}
[c]{c}%
4x_{1}+3x_{2}+4x_{3}\\
-x_{2}+2x_{3}\\
5x_{3}%
\end{array}
\right)  =\left(
\begin{array}
[c]{c}%
3\\
1\\
5
\end{array}
\right)  $. This, in turn, is equivalent to the following system of linear
equations:%
\begin{equation}
\left\{
\begin{array}
[c]{c}%
4x_{1}+3x_{2}+4x_{3}=3;\\
-x_{2}+2x_{3}=1;\\
5x_{3}=5
\end{array}
\right.  . \label{eq.exam.row-echelon.backsub.1.sys}%
\end{equation}


The system (\ref{eq.exam.row-echelon.backsub.1.sys}) has a particularly simple
form: each of its equations can be solved for some variable as long as we
resolve them in the appropriate order. Namely:

\begin{enumerate}
\item We can solve the third equation in
(\ref{eq.exam.row-echelon.backsub.1.sys}) for $x_{3}$, yielding the solution
$x_{3}=1$.

\item Now that we have found the value of $x_{3}$, we can substitute it into
the second equation in (\ref{eq.exam.row-echelon.backsub.1.sys}), so that the
latter equation becomes $-x_{2}+2\cdot1=1$. We can now solve this equation for
$x_{2}$, obtaining $x_{2}=1$.

\item Now that we have found the values of $x_{2}$ and $x_{3}$, we can
substitute them into the first equation in
(\ref{eq.exam.row-echelon.backsub.1.sys}), so that the latter equation becomes
$4x_{1}+3\cdot1+4\cdot1=3$. We can now solve this equation for $x_{1}$,
obtaining $x_{1}=-1$.
\end{enumerate}

Altogether, we have now found $x_{1}=-1$, $x_{2}=1$ and $x_{3}=1$. In other
words, $x=\left(
\begin{array}
[c]{c}%
-1\\
1\\
1
\end{array}
\right)  $. Thus, we have shown that each column vector $x$ satisfying $Ax=b$
must equal $\left(
\begin{array}
[c]{c}%
-1\\
1\\
1
\end{array}
\right)  $.

Conversely, the vector $x=\left(
\begin{array}
[c]{c}%
-1\\
1\\
1
\end{array}
\right)  $ actually \textbf{does} satisfy $Ax=b$; this is because we did not
\textquotedblleft throw away any requirements\textquotedblright:

\begin{enumerate}
\item When we solved the third equation in
(\ref{eq.exam.row-echelon.backsub.1.sys}) for $x_{3}$, we ensured that our
solution $x_{3}=1$ actually does satisfy this third equation.

\item When we rewrote the second equation in
(\ref{eq.exam.row-echelon.backsub.1.sys}) as $-x_{2}+2\cdot1=1$ and solved it
for $x_{2}$, we ensured that our solution $x_{2}=1$ actually does satisfy the
second equation, as long as $x_{3}=1$.

\item When we rewrote the first equation in
(\ref{eq.exam.row-echelon.backsub.1.sys}) as $4x_{1}+3\cdot1+4\cdot1=3$ and
solved it for $x_{1}$, we ensured that our solution $x_{1}=-1$ actually does
satisfy the first equation, as long as $x_{2}=1$ and $x_{3}=1$.
\end{enumerate}

Looking back at these steps, we thus conclude that the values $x_{1}=-1$,
$x_{2}=1$ and $x_{3}=1$ (that is, $x=\left(
\begin{array}
[c]{c}%
-1\\
1\\
1
\end{array}
\right)  $) do satisfy all the three equations in
(\ref{eq.exam.row-echelon.backsub.1.sys}). (Of course, we could also have
checked this by computing $Ax$ for these values; but it is important to know
that this computation is unnecessary -- the method we used comes with a
guarantee that its result is correct!)

We can now conclude that the equation $Ax=b$ has a unique solution, namely
$x=\left(
\begin{array}
[c]{c}%
-1\\
1\\
1
\end{array}
\right)  $.
\end{example}

\begin{example}
\label{exam.row-echelon.backsub.2}Let $A=\left(
\begin{array}
[c]{cccc}%
4 & 3 & 0 & 4\\
0 & -1 & 3 & 2\\
0 & 0 & 0 & 5
\end{array}
\right)  $ and $b=\left(
\begin{array}
[c]{c}%
3\\
1\\
5
\end{array}
\right)  $. We want to solve the equation $Ax=b$; in other words, we want to
find all column vectors $x$ of size $4$ that satisfy $Ax=b$.

Write the unknown vector $x$ in the form $x=\left(
\begin{array}
[c]{c}%
x_{1}\\
x_{2}\\
x_{3}\\
x_{4}%
\end{array}
\right)  $. Then, $Ax=\left(
\begin{array}
[c]{cccc}%
4 & 3 & 0 & 4\\
0 & -1 & 3 & 2\\
0 & 0 & 0 & 5
\end{array}
\right)  \left(
\begin{array}
[c]{c}%
x_{1}\\
x_{2}\\
x_{3}\\
x_{4}%
\end{array}
\right)  =\left(
\begin{array}
[c]{c}%
4x_{1}+3x_{2}+4x_{4}\\
-x_{2}+3x_{3}+2x_{4}\\
5x_{4}%
\end{array}
\right)  $ and $b=\left(
\begin{array}
[c]{c}%
3\\
1\\
5
\end{array}
\right)  $. Hence, the equation $Ax=b$ rewrites as $\left(
\begin{array}
[c]{c}%
4x_{1}+3x_{2}+4x_{4}\\
-x_{2}+3x_{3}+2x_{4}\\
5x_{4}%
\end{array}
\right)  =\left(
\begin{array}
[c]{c}%
3\\
1\\
5
\end{array}
\right)  $. This, in turn, is equivalent to the following system of linear
equations:%
\begin{equation}
\left\{
\begin{array}
[c]{c}%
4x_{1}+3x_{2}+4x_{4}=3;\\
-x_{2}+3x_{3}+2x_{4}=1;\\
5x_{4}=5
\end{array}
\right.  . \label{eq.exam.row-echelon.backsub.2.sys}%
\end{equation}


The system (\ref{eq.exam.row-echelon.backsub.2.sys}) can be solved similarly
to (\ref{eq.exam.row-echelon.backsub.1.sys}), with one more complication:

\begin{enumerate}
\item We can solve the third equation in
(\ref{eq.exam.row-echelon.backsub.2.sys}) for $x_{4}$, yielding the solution
$x_{4}=1$.

\item There is no equation (\ref{eq.exam.row-echelon.backsub.2.sys}) that
\textquotedblleft begins with $x_{3}$\textquotedblright\ (i.e., that has
$x_{3}$ appear with nonzero coefficient, but $x_{1}$ and $x_{2}$ with zero
coefficients). Thus, we have nothing to solve for $x_{3}$. (Of course, we
could try solving the second equation in
(\ref{eq.exam.row-echelon.backsub.2.sys}) for $x_{3}$; but this would require
a value of $x_{2}$, which for now is just as unknown as $x_{3}$.)

Instead of solving an equation for $x_{3}$, let us treat $x_{3}$ as a known!
In other words, let us define a number $r$ by $r=x_{3}$, and pretend that
$x_{3}=r$ is an explicit value for $x_{3}$.

\item Now that we have \textquotedblleft found\textquotedblright\ the values
of $x_{3}$ and $x_{4}$ (the word \textquotedblleft found\textquotedblright\ is
in scare quotes because all we know about $x_{3}$ is that $x_{3}=r$), we can
substitute them into the second equation in
(\ref{eq.exam.row-echelon.backsub.2.sys}), so that the latter equation becomes
$-x_{2}+3r+2\cdot1=1$. We can now solve this equation for $x_{2}$, obtaining
$x_{2}=3r+1$.

\item Now that we have \textquotedblleft found\textquotedblright\ the values
of $x_{2}$, $x_{3}$ and $x_{4}$, we can substitute them into the first
equation in (\ref{eq.exam.row-echelon.backsub.2.sys}), so that the latter
equation becomes $4x_{1}+3\left(  3r+1\right)  +4\cdot1=3$. We can now solve
this equation for $x_{1}$, obtaining $x_{1}=-\dfrac{9}{4}r-1$.
\end{enumerate}

Altogether, we have now found $x_{1}=-\dfrac{9}{4}r-1$, $x_{2}=3r+1$,
$x_{3}=r$ and $x_{4}=1$. In other words,
\begin{equation}
x=\left(
\begin{array}
[c]{c}%
-\dfrac{9}{4}r-1\\
3r+1\\
r\\
1
\end{array}
\right)  . \label{eq.exam.row-echelon.backsub.2.sol}%
\end{equation}
Thus, we have found that each column vector $x$ satisfying $Ax=b$ must have
the form (\ref{eq.exam.row-echelon.backsub.2.sol}) for \textbf{some} number
$r\in\mathbb{R}$.

Conversely, any column vector $x$ of the form
(\ref{eq.exam.row-echelon.backsub.2.sol}) (for \textbf{any} real number
$r\in\mathbb{R}$) actually \textbf{does} satisfy $Ax=b$. The reason for this
is similar to the one we gave in Example \ref{exam.row-echelon.backsub.1}:
Namely, in the process of finding the solution
(\ref{eq.exam.row-echelon.backsub.2.sol}), we did not \textquotedblleft throw
away any requirements\textquotedblright.

So we have learned that a column vector $x$ satisfies $Ax=b$ if and only if it
has the form (\ref{eq.exam.row-echelon.backsub.2.sol}) for \textbf{some}
number $r\in\mathbb{R}$. Thus, the equation $Ax=b$ has infinitely many
solutions -- namely, its solutions are all vectors $x$ of the form
(\ref{eq.exam.row-echelon.backsub.2.sol}) with $r\in\mathbb{R}$. This explains
why we were unable to find an equation to solve for $x_{3}$: Different
solutions $x$ have different values of $x_{3}$.
\end{example}

\begin{example}
\label{exam.row-echelon.backsub.3}Let $A=\left(
\begin{array}
[c]{cccc}%
4 & 3 & 0 & 4\\
0 & -1 & 3 & 2\\
0 & 0 & 0 & 0
\end{array}
\right)  $ and $b=\left(
\begin{array}
[c]{c}%
3\\
1\\
5
\end{array}
\right)  $. We want to solve the equation $Ax=b$; in other words, we want to
find all column vectors $x$ of size $4$ that satisfy $Ax=b$.

Write the unknown vector $x$ in the form $x=\left(
\begin{array}
[c]{c}%
x_{1}\\
x_{2}\\
x_{3}\\
x_{4}%
\end{array}
\right)  $. Then, $Ax=\left(
\begin{array}
[c]{cccc}%
4 & 3 & 0 & 4\\
0 & -1 & 3 & 2\\
0 & 0 & 0 & 0
\end{array}
\right)  \left(
\begin{array}
[c]{c}%
x_{1}\\
x_{2}\\
x_{3}\\
x_{4}%
\end{array}
\right)  =\left(
\begin{array}
[c]{c}%
4x_{1}+3x_{2}+4x_{4}\\
-x_{2}+3x_{3}+2x_{4}\\
0
\end{array}
\right)  $ and $b=\left(
\begin{array}
[c]{c}%
3\\
1\\
5
\end{array}
\right)  $. Hence, the equation $Ax=b$ rewrites as $\left(
\begin{array}
[c]{c}%
4x_{1}+3x_{2}+4x_{4}\\
-x_{2}+3x_{3}+2x_{4}\\
0
\end{array}
\right)  =\left(
\begin{array}
[c]{c}%
3\\
1\\
5
\end{array}
\right)  $. This, in turn, is equivalent to the following system of linear
equations:%
\begin{equation}
\left\{
\begin{array}
[c]{c}%
4x_{1}+3x_{2}+4x_{4}=3;\\
-x_{2}+3x_{3}+2x_{4}=1;\\
0=5
\end{array}
\right.  . \label{eq.exam.row-echelon.backsub.3.sys}%
\end{equation}


The system (\ref{eq.exam.row-echelon.backsub.3.sys}) can be solved very
easily: Its third equation (that is, $0=5$) is \textbf{never} satisfied (no
matter what $x$ is). Thus, the whole system
(\ref{eq.exam.row-echelon.backsub.3.sys}) is never satisfied. In other words,
$Ax=b$ is never satisfied. Hence, $Ax=b$ has no solutions.
\end{example}

\begin{example}
\label{exam.row-echelon.backsub.4}Let $A=\left(
\begin{array}
[c]{cccc}%
4 & 3 & 0 & 4\\
0 & -1 & 3 & 2\\
0 & 0 & 0 & 0
\end{array}
\right)  $ and $b=\left(
\begin{array}
[c]{c}%
3\\
1\\
0
\end{array}
\right)  $. We want to solve the equation $Ax=b$; in other words, we want to
find all column vectors $x$ of size $4$ that satisfy $Ax=b$.

Write the unknown vector $x$ in the form $x=\left(
\begin{array}
[c]{c}%
x_{1}\\
x_{2}\\
x_{3}\\
x_{4}%
\end{array}
\right)  $. Then, $Ax=\left(
\begin{array}
[c]{cccc}%
4 & 3 & 0 & 4\\
0 & -1 & 3 & 2\\
0 & 0 & 0 & 0
\end{array}
\right)  \left(
\begin{array}
[c]{c}%
x_{1}\\
x_{2}\\
x_{3}\\
x_{4}%
\end{array}
\right)  =\left(
\begin{array}
[c]{c}%
4x_{1}+3x_{2}+4x_{4}\\
-x_{2}+3x_{3}+2x_{4}\\
0
\end{array}
\right)  $ and $b=\left(
\begin{array}
[c]{c}%
3\\
1\\
0
\end{array}
\right)  $. Hence, the equation $Ax=b$ rewrites as $\left(
\begin{array}
[c]{c}%
4x_{1}+3x_{2}+4x_{4}\\
-x_{2}+3x_{3}+2x_{4}\\
0
\end{array}
\right)  =\left(
\begin{array}
[c]{c}%
3\\
1\\
0
\end{array}
\right)  $. This, in turn, is equivalent to the following system of linear
equations:%
\begin{equation}
\left\{
\begin{array}
[c]{c}%
4x_{1}+3x_{2}+4x_{4}=3;\\
-x_{2}+3x_{3}+2x_{4}=1;\\
0=0
\end{array}
\right.  . \label{eq.exam.row-echelon.backsub.4.sys}%
\end{equation}


The system (\ref{eq.exam.row-echelon.backsub.4.sys}) can be solved similarly
to (\ref{eq.exam.row-echelon.backsub.3.sys}), after a very simple first step:

\begin{enumerate}
\item The third equation in (\ref{eq.exam.row-echelon.backsub.4.sys}) is
\textbf{always} satisfied (since it says that $0=0$), and thus places no
restrictions on $x$. Hence, it can be discarded.

\item There is no equation (\ref{eq.exam.row-echelon.backsub.4.sys}) that
\textquotedblleft begins with $x_{4}$\textquotedblright\ (i.e., that has
$x_{4}$ appear with nonzero coefficient, but $x_{1}$, $x_{2}$ and $x_{3}$ with
zero coefficients). As in Example \ref{exam.row-echelon.backsub.2}, we use
this as an opportunity to treat $x_{4}$ as a known. In other words, let us
define a number $r$ by $r=x_{4}$, and pretend that $x_{4}=r$ is an explicit
value for $x_{4}$.

\item There is no equation (\ref{eq.exam.row-echelon.backsub.4.sys}) that
\textquotedblleft begins with $x_{3}$\textquotedblright\ (i.e., that has
$x_{3}$ appear with nonzero coefficient, but $x_{1}$ and $x_{2}$ with zero
coefficients). As in Example \ref{exam.row-echelon.backsub.2}, we use this as
an opportunity to treat $x_{3}$ as a known. In other words, let us define a
number $s$ by $s=x_{3}$, and pretend that $x_{3}=s$ is an explicit value for
$x_{3}$. (Of course, we must not reuse an existing variable such as $r$ here.)

\item Now that we have \textquotedblleft found\textquotedblright\ the values
of $x_{3}$ and $x_{4}$, we can substitute them into the second equation in
(\ref{eq.exam.row-echelon.backsub.4.sys}), so that the latter equation becomes
$-x_{2}+3s+2r=1$. We can now solve this equation for $x_{2}$, obtaining
$x_{2}=2r+3s-1$.

\item Now that we have \textquotedblleft found\textquotedblright\ the values
of $x_{2}$, $x_{3}$ and $x_{4}$, we can substitute them into the first
equation in (\ref{eq.exam.row-echelon.backsub.4.sys}), so that the latter
equation becomes $4x_{1}+3\left(  2r+3s-1\right)  +4r=3$. We can now solve
this equation for $x_{1}$, obtaining $x_{1}=\dfrac{3}{2}-\dfrac{9}{4}%
s-\dfrac{5}{2}r$.
\end{enumerate}

Altogether, we have now found $x_{1}=\dfrac{3}{2}-\dfrac{9}{4}s-\dfrac{5}{2}%
r$, $x_{2}=2r+3s-1$, $x_{3}=s$ and $x_{4}=r$. In other words,
\begin{equation}
x=\left(
\begin{array}
[c]{c}%
\dfrac{3}{2}-\dfrac{9}{4}s-\dfrac{5}{2}r\\
2r+3s-1\\
s\\
r
\end{array}
\right)  . \label{eq.exam.row-echelon.backsub.4.sol}%
\end{equation}
Thus, we have found that each column vector $x$ satisfying $Ax=b$ must have
the form (\ref{eq.exam.row-echelon.backsub.4.sol}) for \textbf{some} numbers
$r,s\in\mathbb{R}$.

Conversely, any column vector $x$ of the form
(\ref{eq.exam.row-echelon.backsub.4.sol}) (for \textbf{any} real numbers
$r,s\in\mathbb{R}$) actually \textbf{does} satisfy $Ax=b$. This holds, again,
for the same reason as in Example \ref{exam.row-echelon.backsub.1}.

So we have learned that a column vector $x$ satisfies $Ax=b$ if and only if it
has the form (\ref{eq.exam.row-echelon.backsub.4.sol}) for \textbf{some}
numbers $r,s\in\mathbb{R}$. Thus, the equation $Ax=b$ has infinitely many
solutions -- namely, its solutions are all vectors $x$ of the form
(\ref{eq.exam.row-echelon.backsub.4.sol}) with $r,s\in\mathbb{R}$.
\end{example}

We have now seen four examples of an algorithm for solving equations of the
form $Ax=b$ where $A$ is a row-echelon matrix. The general form of the
algorithm should now be easy to guess, but let me pin it down:

[...] [THE\ BELOW\ IS\ INCOMPLETE]

\begin{theorem}
\label{thm.row-echelon.backsub}Let $n\in\mathbb{N}$ and $m\in\mathbb{N}$. Let
$A$ be an $n\times m$-matrix. Let $b$ be an $n\times1$-vector. Assume that $A$
is a row-echelon matrix.

Consider the problem of finding all solutions to the equation $Ax=b$. In other
words, we want to find all $m\times1$-matrices $x$ satisfying $Ax=b$. This
problem can be solved by the following algorithm (\textit{called
back-substitution}):

\begin{enumerate}
\item Set $b=\left(
\begin{array}
[c]{c}%
b_{1}\\
b_{2}\\
\vdots\\
b_{n}%
\end{array}
\right)  $ and $x=\left(
\begin{array}
[c]{c}%
x_{1}\\
x_{2}\\
\vdots\\
x_{m}%
\end{array}
\right)  $. Multiplying the equalities $A=\left(
\begin{array}
[c]{cccc}%
A_{1,1} & A_{1,2} & \cdots & A_{1,m}\\
A_{2,1} & A_{2,2} & \cdots & A_{2,m}\\
\vdots & \vdots & \ddots & \vdots\\
A_{n,1} & A_{n,2} & \cdots & A_{n,m}%
\end{array}
\right)  $ and $x=\left(
\begin{array}
[c]{c}%
x_{1}\\
x_{2}\\
\vdots\\
x_{m}%
\end{array}
\right)  $, we obtain%
\begin{align*}
Ax  &  =\left(
\begin{array}
[c]{cccc}%
A_{1,1} & A_{1,2} & \cdots & A_{1,m}\\
A_{2,1} & A_{2,2} & \cdots & A_{2,m}\\
\vdots & \vdots & \ddots & \vdots\\
A_{n,1} & A_{n,2} & \cdots & A_{n,m}%
\end{array}
\right)  \left(
\begin{array}
[c]{c}%
x_{1}\\
x_{2}\\
\vdots\\
x_{m}%
\end{array}
\right) \\
&  =\left(
\begin{array}
[c]{c}%
A_{1,1}x_{1}+A_{1,2}x_{2}+\cdots+A_{1,m}x_{m}\\
A_{2,1}x_{1}+A_{2,2}x_{2}+\cdots+A_{2,m}x_{m}\\
\vdots\\
A_{n,1}x_{1}+A_{n,2}x_{2}+\cdots+A_{n,m}x_{m}%
\end{array}
\right)  .
\end{align*}
Thus, the equation $Ax=b$ rewrites as
\[
\left(
\begin{array}
[c]{c}%
A_{1,1}x_{1}+A_{1,2}x_{2}+\cdots+A_{1,m}x_{m}\\
A_{2,1}x_{1}+A_{2,2}x_{2}+\cdots+A_{2,m}x_{m}\\
\vdots\\
A_{n,1}x_{1}+A_{n,2}x_{2}+\cdots+A_{n,m}x_{m}%
\end{array}
\right)  =\left(
\begin{array}
[c]{c}%
b_{1}\\
b_{2}\\
\vdots\\
b_{n}%
\end{array}
\right)
\]
(since $b=\left(
\begin{array}
[c]{c}%
b_{1}\\
b_{2}\\
\vdots\\
b_{n}%
\end{array}
\right)  $). Hence, it is equivalent to the system%
\begin{equation}
\left\{
\begin{array}
[c]{c}%
A_{1,1}x_{1}+A_{1,2}x_{2}+\cdots+A_{1,m}x_{m}=b_{1};\\
A_{2,1}x_{1}+A_{2,2}x_{2}+\cdots+A_{2,m}x_{m}=b_{2};\\
\vdots\\
A_{n,1}x_{1}+A_{n,2}x_{2}+\cdots+A_{n,m}x_{m}=b_{n}%
\end{array}
\right.  . \label{eq.thm.row-echelon.backsub.sys}%
\end{equation}
This is a system of $n$ linear equations in the $m$ unknowns $x_{1}%
,x_{2},\ldots,x_{m}$. We thus focus on solving this system.

\item Let $k$ be the number of nonzero rows of $A$. Then, the last $n-k$ rows
of $A$ are zero (by Proposition \ref{prop.row-echelon.pivrows} \textbf{(a)}).
Hence, the last $n-k$ equations in the system
(\ref{eq.thm.row-echelon.backsub.sys}) have the form $0=b_{i}$ for some
$i\in\left\{  k+1,k+2,\ldots,n\right\}  $. If \textbf{at least one} of the
values $b_{k+1},b_{k+2},\ldots,b_{n}$ is nonzero, then the equation $Ax=b$ has
no solution. In this case, we should stop right here, since the problem is
solved. (This is the situation we have encountered in Example
\ref{exam.row-echelon.backsub.3}.)

\item Assume now that none of the values $b_{k+1},b_{k+2},\ldots,b_{n}$ is
nonzero. Thus, all of the values $b_{k+1},b_{k+2},\ldots,b_{n}$ are zero.
Hence, the last $n-k$ equations in the system
(\ref{eq.thm.row-echelon.backsub.sys}) have the form $0=0$, and therefore can
be discarded (since they are automatically true). (Notice that if $n-k=0$,
then we are discarding nothing.)

We are thus left with the first $k$ equations:%
\begin{equation}
\left\{
\begin{array}
[c]{c}%
A_{1,1}x_{1}+A_{1,2}x_{2}+\cdots+A_{1,m}x_{m}=b_{1};\\
A_{2,1}x_{1}+A_{2,2}x_{2}+\cdots+A_{2,m}x_{m}=b_{2};\\
\vdots\\
A_{k,1}x_{1}+A_{k,2}x_{2}+\cdots+A_{k,m}x_{m}=b_{k}%
\end{array}
\right.  \ \ . \label{eq.thm.row-echelon.backsub.sysk}%
\end{equation}


\item For each $i\in\left\{  1,2,\ldots,k\right\}  $, set $\phi\left(
i\right)  =\operatorname*{pivind}\left(  \operatorname*{row}\nolimits_{i}%
A\right)  $. Then, Proposition \ref{prop.row-echelon.pivrows} \textbf{(c)}
says that%
\[
\phi\left(  1\right)  <\phi\left(  2\right)  <\cdots<\phi\left(  k\right)  .
\]
[...]
\end{enumerate}
\end{theorem}

\begin{proof}
[Proof of Theorem \ref{thm.row-echelon.backsub}.]We only need to show that in
step 2 [...]
\end{proof}

[...]

\subsection{\label{sect.gauss.gauss}%
%TCIMACRO{\TEXTsymbol{<}}%
%BeginExpansion
$<$%
%EndExpansion
TODO%
%TCIMACRO{\TEXTsymbol{>} }%
%BeginExpansion
$>$
%EndExpansion
Gaussian elimination and the row echelon form}

[...]

(to be continued)

\textbf{TO-DO LIST! (This will appear eventually:)}

\begin{todo}
Gaussian elimination: $C=E^{\ast}U$, where $E^{\ast}$ is a product of
elementary matrices ($A_{u,v}^{\lambda}$, $S_{i}^{\lambda}$, $T_{u,v}$) and
$U$ is a row-echelon matrix.
\end{todo}

\begin{todo}
How this generalizes things shown before.
\end{todo}

\begin{todo}
\textbf{Example of Gaussian elimination:} Start with the matrix $C=\left(
\begin{array}
[c]{cccc}%
3 & 2 & 1 & 6\\
3 & 2 & 1 & 7\\
0 & 1 & -1 & 0\\
-1 & 2 & -3 & 0
\end{array}
\right)  $. Want to bring it into row echelon form by using downward row
additions (i.e., row operations $A_{u,v}^{\lambda}$ with $u>v$), row scalings
(i.e., row operations $S_{u}^{\lambda}$ with $\lambda\neq0$) and row swappings
(i.e., row operations $T_{u,v}$) only.

Subtracting row $1$ from row $2$ gives new matrix $C^{\prime}=\left(
\begin{array}
[c]{cccc}%
3 & 2 & 1 & 6\\
0 & 0 & 0 & 1\\
0 & 1 & -1 & 0\\
-1 & 2 & -3 & 0
\end{array}
\right)  $. Thus $C=A_{2,1}^{1}C^{\prime}$.

Subtracting $\dfrac{-1}{3}$ times row $1$ from row $4$ gives new matrix
$C^{\prime\prime}=\left(
\begin{array}
[c]{cccc}%
3 & 2 & 1 & 6\\
0 & 0 & 0 & 1\\
0 & 1 & -1 & 0\\
0 & \dfrac{8}{3} & -\dfrac{8}{3} & 2
\end{array}
\right)  $. Thus $C^{\prime}=A_{4,1}^{-1/3}C^{\prime\prime}$.

Now, the first column looks like row echelon form.

The entry $0$ in cell $\left(  2,2\right)  $ prevents the first two columns
from being row-echelon. Thus, swap it with a nonzero entry further down.

Swapping rows $2$ and $3$ gives new matrix $C^{\prime\prime\prime}=\left(
\begin{array}
[c]{cccc}%
3 & 2 & 1 & 6\\
0 & 1 & -1 & 0\\
0 & 0 & 0 & 1\\
0 & \dfrac{8}{3} & -\dfrac{8}{3} & 2
\end{array}
\right)  $. Thus $C^{\prime\prime}=T_{2,3}C^{\prime\prime\prime}$.

Subtracting $\dfrac{8}{3}$ times row $2$ from row $4$ gives new matrix
$C^{\prime\prime\prime\prime}=\left(
\begin{array}
[c]{cccc}%
3 & 2 & 1 & 6\\
0 & 1 & -1 & 0\\
0 & 0 & 0 & 1\\
0 & 0 & 0 & 2
\end{array}
\right)  $. Thus $C^{\prime\prime\prime}=A_{4,2}^{8/3}C^{\prime\prime
\prime\prime}$.

Now, the first two columns look like row echelon form.

Usually we would have to do some operations for the third column, but this
time we are in luck: The first three columns already look like row echelon
form, so no operations are needed here.

However, the fourth column still needs an operation: It has a nonzero entry
below its pivot $1$.

Subtracting $2$ times row $3$ from row $4$ gives new matrix $C^{\prime
\prime\prime\prime\prime}=\left(
\begin{array}
[c]{cccc}%
3 & 2 & 1 & 6\\
0 & 1 & -1 & 0\\
0 & 0 & 0 & 1\\
0 & 0 & 0 & 0
\end{array}
\right)  $. Thus $C^{\prime\prime\prime\prime}=A_{4,3}^{2}C^{\prime
\prime\prime\prime\prime}$.

The matrix $C^{\prime\prime\prime\prime\prime}$ is in row echelon form; its
pivot entries are in cells $\left(  1,1\right)  $, $\left(  2,2\right)  $ and
$\left(  3,4\right)  $. Altogether,%
\begin{align*}
C  &  =A_{2,1}^{1}\underbrace{C^{\prime}}_{=A_{4,1}^{-1/3}C^{\prime\prime}%
}=A_{2,1}^{1}A_{4,1}^{-1/3}\underbrace{C^{\prime\prime}}_{=T_{2,3}%
C^{\prime\prime\prime}}=A_{2,1}^{1}A_{4,1}^{-1/3}T_{2,3}\underbrace{C^{\prime
\prime\prime}}_{=A_{4,2}^{8/3}C^{\prime\prime\prime\prime}}\\
&  =A_{2,1}^{1}A_{4,1}^{-1/3}T_{2,3}A_{4,2}^{8/3}\underbrace{C^{\prime
\prime\prime\prime}}_{=A_{4,3}^{2}C^{\prime\prime\prime\prime\prime}}%
=A_{2,1}^{1}A_{4,1}^{-1/3}T_{2,3}A_{4,2}^{8/3}A_{4,3}^{2}C^{\prime\prime
\prime\prime\prime}.
\end{align*}


We could tweak $C^{\prime\prime\prime\prime\prime}$ furthermore (by row
operations) to obtain a \textit{row-reduced echelon matrix}, which is
characterized by the properties that \textbf{(a)} each pivot entry equals $1$,
and \textbf{(b)} in each column containing a pivot entry, all other entries
are $0$. To achieve \textbf{(a)}, we merely have to apply some row scaling
operations. To achieve \textbf{(b)}, we need to apply \textit{upward row
additions}, i.e., row operations $A_{u,v}^{\lambda}$ with $u<v$; this way we
can clear out the entries above each pivot.

But if we allow ourselves column operations as well, then we can even end up
with a matrix which has entries $1$ in cells $\left(  1,1\right)  ,\left(
2,2\right)  ,\ldots,\left(  k,k\right)  $ for some $k$, and entries $0$ in all
other cells. (Think of it as a truncated identity matrix, except that it is rectangular.)
\end{todo}

\begin{todo}
History: \cite{Grcar10} reference.
\end{todo}

\begin{todo}
How to solve $Uv=b$ (back-substitution; beware of zero rows).
\end{todo}

\begin{todo}
Thus, solve $Cv=b$.
\end{todo}

\begin{todo}
How to see whether a matrix $C$ is invertible using Gauss.
\end{todo}

\subsection{%
%TCIMACRO{\TEXTsymbol{<}}%
%BeginExpansion
$<$%
%EndExpansion
TODO%
%TCIMACRO{\TEXTsymbol{>} }%
%BeginExpansion
$>$
%EndExpansion
$PLU$ decomposition}

\begin{todo}
$PLU$ decomposition (not sure). (I didn't do $PLU$ in class properly, and
doing it right is subtle. \cite[Example 1.12]{OlvSha06} does it.)
\end{todo}

\begin{todo}
Most of the time, $P=I_{n}$, because there is no need to permute rows when the
entries are nonzero.
\end{todo}

\subsection{%
%TCIMACRO{\TEXTsymbol{<}}%
%BeginExpansion
$<$%
%EndExpansion
TODO%
%TCIMACRO{\TEXTsymbol{>} }%
%BeginExpansion
$>$
%EndExpansion
Determinants (briefly)}

Determinants are not a central player in these notes. Nevertheless, they are
sufficiently important to be mentioned. I consider them to be one of the most
beautiful objects in mathematics, and they are also one of the most useful in
\textbf{pure} mathematics (particularly, in all breeds of algebra and
combinatorics). However, their use in applied mathematics is rather limited
(apart from \textquotedblleft small\textquotedblright\ cases like matrices of
size $2\times2$, $3\times3$ and $4\times4$, occurring in physics and geometry
for well-known reasons). Thus, I shall give no proofs and barely state the
most crucial results. Three good references on determinants are:

\begin{itemize}
\item \cite[Chapter 8]{LaNaSc16} does the definitions and the basic properties
really well. I highly recommend it.

\item \cite[Chapter Four]{Heffer16} also has a neat treatment of determinants,
although unfortunately it requires Gauss-Jordan elimination (which I have not
done above, and which I find out-of-place in an introduction to determinants).
It has many examples and explains the geometric meaning of $2\times
2$-determinants (as areas of triangles) and $3\times3$-determinants (as
volumes of tetrahedra).

\item \cite[Chapter 4]{BarSch73} doesn't always have the best proof (in
particular, it uses some properties of singular matrices that we have not yet
learned), but seems to be written well and with attention to detail.
\end{itemize}

Apart from that, lots of other texts define and discuss determinants: some
sloppily, some rather well.\footnote{Olver's and Shakiban's treatment of
determinants in \cite[\S 1.9]{OlvSha06} is very minimalistic and incomplete
(in particular, they never bother proving that the determinant is
well-defined). On the opposite side of the spectrum, two really rigorous and
detailed treatments are Gill Williamson's notes \cite[Chapter 3]{Gill} and my
\cite{detnotes}; but the downside of \textquotedblleft
rigorous\textquotedblright\ is \textquotedblleft unmotivated and lacking
intuition\textquotedblright, and the downside of \textquotedblleft
detailed\textquotedblright\ is \textquotedblleft extremely
long\textquotedblright. You will want to strike your own balance. As I said, I
recommend \cite[Chapter 8]{LaNaSc16}.} When reading other sources, be aware of
a possible conflict of notations: Namely, we are using the notation $A_{i,j}$
for the $\left(  i,j\right)  $-th entry of a matrix $A$; but most authors use
the notation $A_{i,j}$ for something different (namely, the $\left(
i,j\right)  $-th \textquotedblleft cofactor\textquotedblright\ of $A$).

\begin{todo}
Recall Example \ref{exam.inverses} \textbf{(e)}: The number $ad-bc$ governs
whether the matrix $\left(
\begin{array}
[c]{cc}%
a & b\\
c & d
\end{array}
\right)  $ has an inverse. Does such a number exist for larger matrices? Yes,
it does, and is called the determinant.
\end{todo}

\begin{todo}
The determinant of an $n\times n$-matrix $A$ is a number, denoted by $\det A$
and defined later. (Some authors call it $\left\vert A\right\vert $, but we
will not.)
\end{todo}

\begin{todo}
Defining the determinant of an $n\times n$-matrix is not that easy. Let us
first do it for small cases:%
\begin{align*}
\det\left(
\begin{array}
[c]{cc}%
a_{1} & a_{2}\\
b_{1} & b_{2}%
\end{array}
\right)   &  =a_{1}b_{2}-a_{2}b_{1};\\
\det\left(
\begin{array}
[c]{ccc}%
a_{1} & a_{2} & a_{3}\\
b_{1} & b_{2} & b_{3}\\
c_{1} & c_{2} & c_{3}%
\end{array}
\right)   &  =a_{1}b_{2}c_{3}-a_{1}c_{2}b_{3}-b_{1}a_{2}c_{3}+b_{1}c_{2}%
a_{3}+c_{1}a_{2}b_{3}-c_{1}b_{2}a_{3};\\
\det\left(
\begin{array}
[c]{c}%
a
\end{array}
\right)   &  =a;\\
\det\left(  {}\right)   &  =1.
\end{align*}
(The last equality says that the determinant of a $0\times0$-matrix is $1$,
and yes, that's sometimes useful.)
\end{todo}

\begin{todo}
Let me give three versions of the definition of $\det A$ for all $n$:

\begin{itemize}
\item \textit{First version} (informal and sloppy)\textit{:} Set\textit{ }%
\begin{align*}
&  \det\left(
\begin{array}
[c]{cccc}%
a_{1} & a_{2} & \cdots & a_{n}\\
b_{1} & b_{2} & \cdots & b_{n}\\
\vdots & \vdots & \ddots & \vdots\\
z_{1} & z_{2} & \cdots & z_{n}%
\end{array}
\right) \\
&  =\text{sum of terms of the form }\pm?_{1}?_{2}\cdots?_{n},
\end{align*}
where the question marks in each term are some permutation (= rearrangement)
of the letters $a,b,\ldots,z$, and the $\pm$ sign depends on the
rearrangement. How exactly? Any rearrangement can be sorted into increasing
order (i.e., into $ab\ldots z$) by switching pairs of letters; e.g. (for
$n=5$) we can sort the rearrangement $caebd$ as follows:%
\begin{align*}
caebd  &  \rightarrow cabed\ \ \ \ \ \ \ \ \ \ \left(  \text{here we switched
}e\text{ with }b\right) \\
&  \rightarrow baced\ \ \ \ \ \ \ \ \ \ \left(  \text{here we switched
}c\text{ with }b\right) \\
&  \rightarrow abced\ \ \ \ \ \ \ \ \ \ \left(  \text{here we switched
}b\text{ with }a\right) \\
&  \rightarrow abcde\ \ \ \ \ \ \ \ \ \ \left(  \text{here we switched
}e\text{ with }d\right)  .
\end{align*}
Notice that we used $4$ switches here. The rule for the $\pm$ sign is: If the
rearrangement needs an even number of switches, then it's a $+$; if an odd
number, then it's a $-$.

\item \textit{Second version} (formal): Recall that each permutation matrix
$P$ is a product of several swapping matrices (i.e., of $T_{u,v}$'s). Define
$\operatorname*{sign}\left(  P\right)  $ to be $+1$ if $P$ is the product of
an even number of swapping matrices; define $\operatorname*{sign}\left(
P\right)  $ to be $-1$ if $P$ is the product of an odd number of swapping
matrices. It is not obvious that this definition is legitimate (because if $P$
would be representable both as an even product and as an odd product, then
$\operatorname*{sign}\left(  P\right)  $ would have to be $+1$ and $-1$ at the
same time!), but it is nevertheless true (it can be shown that no $P$ can ever
be representable both as even product and as odd product). If $P$ is a
permutation matrix and $A$ is an $n\times n$-matrix, then the $P$%
\textit{-product of }$A$ shall mean the product of all entries of $A$ in those
cells in which $P$ has a $1$. For example, if $P=\left(
\begin{array}
[c]{ccc}%
1 & 0 & 0\\
0 & 0 & 1\\
0 & 1 & 0
\end{array}
\right)  $, then the $P$-product of $\left(
\begin{array}
[c]{ccc}%
a_{1} & a_{2} & a_{3}\\
b_{1} & b_{2} & b_{3}\\
c_{1} & c_{2} & c_{3}%
\end{array}
\right)  $ is $a_{1}b_{3}c_{2}$, because the cells in which $P$ has a $1$ are
the cells $\left(  1,1\right)  $, $\left(  2,3\right)  $ and $\left(
3,2\right)  $, and the entries of $A$ in these cells are $a_{1}$, $b_{3}$ and
$c_{2}$. Now, the determinant $\det A$ of $A$ is the sum of%
\[
\operatorname*{sign}\left(  P\right)  \cdot\left(  \text{the }P\text{-product
of }A\right)
\]
for all permutation matrices $P$.

\item \textit{Third version} (formal, uses Section
\ref{sect.gauss.permat.perms})\textit{:} Let%
\[
\det A=\sum_{\sigma\in S_{n}}\left(  -1\right)  ^{\sigma}A_{1,\sigma\left(
1\right)  }A_{2,\sigma\left(  2\right)  }\cdots A_{n,\sigma\left(  n\right)
},
\]
where $S_{n}$ denotes the set of all permutations of $\left\{  1,2,\ldots
,n\right\}  $, and the number $\left(  -1\right)  ^{\sigma}$ is defined as
follows: Proposition \ref{prop.permat.Tkk+1.perm} shows that the permutation
$\sigma$ can be written as a product of $k$ simple transpositions for some
$k\in\mathbb{N}$. If this $k$ is even, then set $\left(  -1\right)  ^{\sigma
}=1$; if this $k$ is odd, then set $\left(  -1\right)  ^{\sigma}=-1$. (Again,
we need to prove that this definition is legitimate, i.e., that $\left(
-1\right)  ^{\sigma}$ is never defined to be $1$ and $-1$ at the same time.)
\end{itemize}

Note that the third version of the definition is merely the formalization of
the first version. The second version is just the translation of the third
version into the language of permutation matrices.
\end{todo}

\begin{todo}
State the main properties of determinants:

\begin{itemize}
\item We have $\det\left(  0_{n\times n}\right)  =0$ for all $n>0$.

\item We have $\det\left(  I_{n}\right)  =1$ for all $n$.

\item We have $\det\left(  A^{T}\right)  =\det A$ for any square matrix $A$.

\item The determinant is multilinear in the rows (i.e., linear \textbf{in each
row}, as long as the other rows are fixed). For example,%
\begin{align*}
&  \det\left(
\begin{array}
[c]{ccc}%
a_{1} & a_{2} & a_{3}\\
b_{1}+b_{1}^{\prime} & b_{2}+b_{2}^{\prime} & b_{3}+b_{3}^{\prime}\\
c_{1} & c_{2} & c_{3}%
\end{array}
\right) \\
&  =\det\left(
\begin{array}
[c]{ccc}%
a_{1} & a_{2} & a_{3}\\
b_{1} & b_{2} & b_{3}\\
c_{1} & c_{2} & c_{3}%
\end{array}
\right)  +\det\left(
\begin{array}
[c]{ccc}%
a_{1} & a_{2} & a_{3}\\
b_{1}^{\prime} & b_{2}^{\prime} & b_{3}^{\prime}\\
c_{1} & c_{2} & c_{3}%
\end{array}
\right)
\end{align*}
and%
\[
\det\left(
\begin{array}
[c]{ccc}%
a_{1} & a_{2} & a_{3}\\
\lambda b_{1} & \lambda b_{2} & \lambda b_{3}\\
c_{1} & c_{2} & c_{3}%
\end{array}
\right)  =\lambda\det\left(
\begin{array}
[c]{ccc}%
a_{1} & a_{2} & a_{3}\\
b_{1} & b_{2} & b_{3}\\
c_{1} & c_{2} & c_{3}%
\end{array}
\right)  .
\]
See \cite[Theorem 8.2.3, 3.]{LaNaSc16} for a precise statement (but for
columns instead of rows).

\item Note that $\det\left(  A+B\right)  \neq\det A+\det B$ in general.

\item Switching two rows of a matrix $A$ negates $\det A$ (that is, if $B$ is
obtained from $A$ by switching two rows, then $\det B=-\det A$).

\item If $A$ has two equal rows, then $\det A=0$.

\item If $A$ has a row filled with zeroes, then $\det A=0$.

\item If we add a multiple of some row of $A$ to another row, then $\det A$
does not change.

\item All of the above properties that reference rows also hold for columns.

\item If $A$ is upper-triangular or lower-triangular, then $\det A$ is the
product of the diagonal entries of $A$.

\item We have $\det\left(  AB\right)  =\det A\cdot\det B$ for any two $n\times
n$-matrices $A$ and $B$. This fact is not obvious, and is one of the miracles
of mathematics. It is probably the main reason why determinants are useful!

\item An $n\times n$-matrix $A$ is invertible if and only if $\det A\neq0$.

\item For each $i\in\left\{  1,2,\ldots,n\right\}  $ and $j\in\left\{
1,2,\ldots,n\right\}  $, let $A_{\sim i,\sim j}$ denote the $\left(
n-1\right)  \times\left(  n-1\right)  $-matrix obtained from $A$ by removing
the $i$-th row and the $j$-th column. For example,%
\[
\left(
\begin{array}
[c]{ccc}%
a_{1} & a_{2} & a_{3}\\
b_{1} & b_{2} & b_{3}\\
c_{1} & c_{2} & c_{3}%
\end{array}
\right)  _{\sim1,\sim2}=\left(
\begin{array}
[c]{cc}%
b_{1} & b_{3}\\
c_{1} & c_{3}%
\end{array}
\right)  .
\]
Let $\operatorname*{adj}A$ be the $n\times n$-matrix $\left(  \left(
-1\right)  ^{i+j}\det\left(  A_{\sim j,\sim i}\right)  \right)  _{1\leq i\leq
n,\ 1\leq j\leq n}$. (This is called the \textit{adjugate} of $A$.) Then,%
\[
A\cdot\operatorname*{adj}A=\operatorname*{adj}A\cdot A=\det A\cdot I_{n}.
\]


\item For every $p\in\left\{  1,2,\ldots,n\right\}  $, we have%
\[
\det A=\sum_{q=1}^{n}\left(  -1\right)  ^{p+q}A_{p,q}\det\left(  A_{\sim
p,\sim q}\right)  .
\]
This is called \textquotedblleft Laplace expansion in the $p$-th
row\textquotedblright, and gives us a way to compute $\det A$ using
determinants of $\left(  n-1\right)  \times\left(  n-1\right)  $-matrices.

\item For every $q\in\left\{  1,2,\ldots,n\right\}  $, we have%
\[
\det A=\sum_{p=1}^{n}\left(  -1\right)  ^{p+q}A_{p,q}\det\left(  A_{\sim
p,\sim q}\right)  .
\]
This is called \textquotedblleft Laplace expansion in the $q$-th
column\textquotedblright.
\end{itemize}
\end{todo}

\subsection{%
%TCIMACRO{\TEXTsymbol{<}}%
%BeginExpansion
$<$%
%EndExpansion
TODO%
%TCIMACRO{\TEXTsymbol{>} }%
%BeginExpansion
$>$
%EndExpansion
The rest}

\begin{todo}
(*) SageMath examples.
\end{todo}

\begin{todo}
(*) What holds over rings, what over fields.
\end{todo}

\section{\label{sect.spaces}%
%TCIMACRO{\TEXTsymbol{<}}%
%BeginExpansion
$<$%
%EndExpansion
TODO%
%TCIMACRO{\TEXTsymbol{>} }%
%BeginExpansion
$>$
%EndExpansion
Vector spaces}

\begin{todo}
Introduction into vector spaces.

We shall now switch to what feels like a different subject: the theory of
vector spaces. Soon, we will see that this is closely related to the matrix
algebra that we have been doing above, and in some sense is like doing linear
algebra in a more abstract language (as opposed to doing something completely
different). But first, we introduce the relevant notions without reference to
the matrix theory done above.

(...... This all needs to be filled in. So far, this is a stenographic lecture
plan rather than a set of lecture notes ......)
\end{todo}

\subsection{%
%TCIMACRO{\TEXTsymbol{<}}%
%BeginExpansion
$<$%
%EndExpansion
DRAFT%
%TCIMACRO{\TEXTsymbol{>} }%
%BeginExpansion
$>$
%EndExpansion
Vector spaces}

\textbf{References for vector spaces:} \cite[\S 4.1-\S 4.2]{LaNaSc16}
(possibly the best one), \cite[\S 2.1]{OlvSha06} (focusses on the
analysis-related and applied stuff) and \cite[Two.I.1]{Heffer16}.

\begin{definition}
\label{def.vectspace}A \textit{vector space (over }$\mathbb{R}$\textit{)} is a
set $V$ equipped with two binary operations:

\begin{itemize}
\item a binary operation called \textquotedblleft$+$\textquotedblright, which
takes as its input two elements $v$ and $w$ of $V$, and yields an element of
$V$ called $v+w$;

\item a binary operation called \textquotedblleft$\cdot$\textquotedblright,
which takes as its input a number $\lambda\in\mathbb{R}$ and an element $v$ of
$V$, and yields an element of $V$ called $\lambda\cdot v$ or $\lambda v$,
\end{itemize}

as well as a chosen element of $V$ called $\overrightarrow{0}$, with the
following properties:

\textbf{(a)} We have $v+w=w+v$ for all $v\in V$ and $w\in V$. (This is called
\textquotedblleft commutativity of addition\textquotedblright.)

\textbf{(b)} We have $u+\left(  v+w\right)  =\left(  u+v\right)  +w$ for all
$u\in V$, $v\in V$ and $w\in V$. (This is called \textquotedblleft
associativity of addition\textquotedblright.)

\textbf{(c)} We have $v+\overrightarrow{0}=\overrightarrow{0}+v=v$ for all
$v\in V$. (This is called \textquotedblleft neutrality of $\overrightarrow{0}%
$\textquotedblright.)

\textbf{(d)} For each $v\in V$, there exists some element $w\in V$ satisfying
$v+w=w+v=\overrightarrow{0}$. (This is called \textquotedblleft existence of
additive inverses\textquotedblright. The element $w$ is called the
\textit{additive inverse of }$v$, and is usually called $-v$; it is uniquely
determined by $v$ (this is not hard to check).)

\textbf{(e)} We have $\left(  \lambda+\mu\right)  v=\lambda v+\mu v$ for all
$\lambda\in\mathbb{R}$, $\mu\in\mathbb{R}$ and $v\in V$. (This is called
\textquotedblleft right distributivity\textquotedblright.)

\textbf{(f)} We have $\lambda\left(  v+w\right)  =\lambda v+\lambda w$ for all
$\lambda\in\mathbb{R}$, $v\in V$ and $w\in V$. (This is called
\textquotedblleft left distributivity\textquotedblright.)

\textbf{(g)} We have $\left(  \lambda\mu\right)  v=\lambda\left(  \mu
v\right)  $ for all $\lambda\in\mathbb{R}$, $\mu\in\mathbb{R}$ and $v\in V$.
(This is called \textquotedblleft associativity of scaling\textquotedblright.
It allows us to write $\lambda\mu v$ for both $\left(  \lambda\mu\right)  v$
and $\lambda\left(  \mu v\right)  $.)

\textbf{(h)} We have $1v=v$ for all $v\in V$. (This is called
\textquotedblleft neutrality of $1$\textquotedblright.)

\textbf{(i)} We have $0v=\overrightarrow{0}$ for all $v\in V$.

\textbf{(j)} We have $\lambda\overrightarrow{0}=\overrightarrow{0}$ for all
$\lambda\in\mathbb{R}$.

The operation $+$ is called the \textit{addition} of the vector space; the
operation $\cdot$ is called the \textit{scaling} of the vector space; the
element $\overrightarrow{0}$ is called the \textit{zero vector} (or the
\textit{origin}) of the vector space. The elements of $V$ are called
\textit{vectors}. (These are familiar-sounding names for abstract things. The
operation $+$ may and may not have anything to do with addition of numbers.
The operation $\cdot$ may and may not be related to scaling of numbers. The
element $\overrightarrow{0}$ may and may not be the number $0$. The elements
of $V$, which we call \textquotedblleft vectors\textquotedblright, may and may
not be the kinds of vectors we are used to seeing (i.e., row vectors and
column vectors); they can just as well be polynomials or functions or numbers
or matrices. You should think of the word \textquotedblleft
vector\textquotedblright\ as meaning \textquotedblleft element of a vector
space\textquotedblright, not as meaning \textquotedblleft a list of
numbers\textquotedblright; the latter meaning is too restrictive. The
properties \textbf{(a)}, \textbf{(b)}, $\ldots$, \textbf{(j)} are requiring
that the operations $+$ and $\cdot$ and the element $\overrightarrow{0}$
\textquotedblleft behave like\textquotedblright\ the addition and scaling of
matrices and the zero matrix, at least as far as their most basic properties
are concerned; however, they still leave a lot of freedom to decide what these
operations and this element should be.)

We shall often speak of \textquotedblleft the vector space $V$%
\textquotedblright\ or say that \textquotedblleft$V$ is a vector
space\textquotedblright, but of course, the vector space is not just the set
$V$; it is (as we have defined it) the set $V$ endowed with the two operations
$+$ and $\cdot$ and the element $\overrightarrow{0}$. The two operations and
the element $\overrightarrow{0}$ are part of the data; if you modify them,
then you end up with a different vector space, even if the set $V$ is the
same! However, most of the time, we will not have several different vector
spaces sharing one and the same set $V$; therefore, we will be able to speak
of \textquotedblleft the vector space $V$\textquotedblright\ and assume that
the reader knows what operations $+$ and $\cdot$ and what element
$\overrightarrow{0}$ we mean.
\end{definition}

We have just defined the notion of a \textquotedblleft vector space over
$\mathbb{R}$\textquotedblright\ (also known as a \textquotedblleft real vector
space\textquotedblright). We can similarly define a \textquotedblleft vector
space over $\mathbb{Q}$\textquotedblright\ (by replacing each
\textquotedblleft$\mathbb{R}$\textquotedblright\ in the above definition by
\textquotedblleft$\mathbb{Q}$\textquotedblright) and a \textquotedblleft
vector space over $\mathbb{C}$\textquotedblright\ (by replacing each
\textquotedblleft$\mathbb{R}$\textquotedblright\ in the above definition by
\textquotedblleft$\mathbb{C}$\textquotedblright). Most of the vector spaces we
shall be studying below are vector spaces over $\mathbb{R}$, but the other
options are also useful. (If you have read Definition \ref{def.field}, you
will have no trouble defining a \textquotedblleft vector space over
$\mathbb{K}$\textquotedblright\ for every field $\mathbb{K}$.)

Note that our definition of a vector space is somewhat similar to the
definition of a commutative ring (Definition \ref{def.commring}). At least as
far as the operation $+$ alone is concerned, the requirements on it are
literally the same in the two definitions\footnote{assuming that we identify
the $\overrightarrow{0}$ in Definition \ref{def.vectspace} with the $0$ in
Definition \ref{def.commring}} (commutativity of addition, associativity of
addition, neutrality of $\overrightarrow{0}$ and existence of additive
inverses). However, the similarity ends here: The binary operation $\cdot$
(\textquotedblleft multiplication\textquotedblright) in Definition
\ref{def.commring} works differently from the binary operation $\cdot$
(\textquotedblleft scaling\textquotedblright) in Definition
\ref{def.vectspace}. The former takes two inputs in the commutative ring,
whereas the latter takes one input in $\mathbb{R}$ and one input in the vector
space. The axioms still have certain similarities, but they should not fool
you into believing that commutative rings are vector spaces (or vice versa).

If you compare our Definition \ref{def.vectspace} with other definitions of a
\textquotedblleft vector space\textquotedblright\ you find in the literature
(for example, \cite[Definition 4.1.1]{LaNaSc16}, \cite[Definition
2.1]{OlvSha06} or \cite[Definition Two.I.1]{Heffer16}), you will notice that
they are slightly different: For example, \cite[Definition 4.1.1]{LaNaSc16},
\cite[Definition 2.1]{OlvSha06} or \cite[Definition Two.I.1]{Heffer16} are
lacking our properties \textbf{(i)} and \textbf{(j)}, whereas \cite[Definition
2.3.1]{Kowals16} is missing our properties \textbf{(d)} and \textbf{(j)}.
However, the definitions are nevertheless \textbf{equivalent} (i.e., they
define precisely the same notion of a vector space). The reason for that is
some of the properties we required in Definition \ref{def.vectspace} are
\textbf{redundant} (i.e., they follow from the other properties, so that
nothing changes if we leave them out). For example:

\begin{proposition}
\label{prop.vectspace.redundant.d}Property \textbf{(d)} in Definition
\ref{def.vectspace} follows from properties \textbf{(e)}, \textbf{(h)} and
\textbf{(i)}. Thus, we could leave out property \textbf{(d)} from the definition.
\end{proposition}

\begin{proof}
[Proof of Proposition \ref{prop.vectspace.redundant.d}.]Assume that properties
\textbf{(e)}, \textbf{(h)} and \textbf{(i)} hold. We must now prove property
\textbf{(d)}.

Let $v\in V$. Then,
\begin{align*}
\left(  -1\right)  v+\underbrace{v}_{\substack{=1v\\\text{(by property
\textbf{(h)})}}}  &  =\left(  -1\right)  v+1v=\underbrace{\left(  \left(
-1\right)  +1\right)  }_{=0}v\\
&  \ \ \ \ \ \ \ \ \ \ \left(  \text{by property \textbf{(e)}, applied to
}\lambda=-1\text{ and }\mu=1\right) \\
&  =0v=\overrightarrow{0}\ \ \ \ \ \ \ \ \ \ \left(  \text{by property
\textbf{(i)}}\right)
\end{align*}
and%
\begin{align*}
\underbrace{v}_{\substack{=1v\\\text{(by property \textbf{(h)})}}}+\left(
-1\right)  v  &  =1v+\left(  -1\right)  v=\underbrace{\left(  1+\left(
-1\right)  \right)  }_{=0}v\\
&  \ \ \ \ \ \ \ \ \ \ \left(  \text{by property \textbf{(e)}, applied to
}\lambda=1\text{ and }\mu=-1\right) \\
&  =0v=\overrightarrow{0}\ \ \ \ \ \ \ \ \ \ \left(  \text{by property
\textbf{(i)}}\right)  .
\end{align*}
Hence, there exists some element $w\in V$ satisfying $v+w=w+v=0$ (namely,
$w=\left(  -1\right)  v$). Thus, property \textbf{(d)} is proven. Hence, we
have shown that property \textbf{(d)} follows from properties \textbf{(e)},
\textbf{(h)} and \textbf{(i)}. This proves Proposition
\ref{prop.vectspace.redundant.d}.
\end{proof}

\begin{proposition}
\label{prop.vectspace.redundant.j}Property \textbf{(j)} in Definition
\ref{def.vectspace} follows from properties \textbf{(g)} and \textbf{(i)}.
Thus, we could omit property \textbf{(j)} from the definition.
\end{proposition}

\begin{proof}
[Proof of Proposition \ref{prop.vectspace.redundant.j}.]Assume that properties
\textbf{(g)} and \textbf{(i)} hold. We must now prove property \textbf{(j)}.

Let $\lambda\in\mathbb{R}$. Then, property \textbf{(i)} (applied to
$v=\overrightarrow{0}$) yields $0\cdot\overrightarrow{0}=\overrightarrow{0}$.
Thus, $\lambda\left(  0\cdot\overrightarrow{0}\right)  =\lambda
\overrightarrow{0}$, so that%
\begin{align*}
\lambda\overrightarrow{0}  &  =\lambda\left(  0\cdot\overrightarrow{0}\right)
=\underbrace{\left(  \lambda\cdot0\right)  }_{=0}\overrightarrow{0}%
\ \ \ \ \ \ \ \ \ \ \left(  \text{by property \textbf{(g)}, applied to }%
\mu=0\text{ and }v=\overrightarrow{0}\right) \\
&  =0\cdot\overrightarrow{0}=\overrightarrow{0}.
\end{align*}
Thus, property \textbf{(j)} is proven. Hence, we have shown that property
\textbf{(j)} follows from properties \textbf{(g)} and \textbf{(i)}. This
proves Proposition \ref{prop.vectspace.redundant.j}.
\end{proof}

\begin{proposition}
\label{prop.vectspace.redundant.i}Property \textbf{(i)} in Definition
\ref{def.vectspace} follows from properties \textbf{(b)}, \textbf{(c)},
\textbf{(d)} and \textbf{(e)}. Thus, we could omit property \textbf{(i)} from
the definition.
\end{proposition}

\begin{proof}
[Proof of Proposition \ref{prop.vectspace.redundant.i}.]Assume that properties
\textbf{(b)}, \textbf{(c)}, \textbf{(d)} and \textbf{(e)} hold. We must now
prove property \textbf{(i)}.

Let $v\in V$. Property \textbf{(e)} (applied to $\lambda=0$ and $\mu=0$)
yields $\left(  0+0\right)  v=0v+0v$. Hence, $0v+0v=\underbrace{\left(
0+0\right)  }_{=0}v=0v$.

But we can apply property \textbf{(d)} to $0v$ instead of $v$. We thus
conclude that there exists some element $w\in V$ satisfying
$0v+w=w+0v=\overrightarrow{0}$. Consider this $w$.

Property \textbf{(b)} (applied to $0v$, $0v$ and $w$ instead of $u$, $v$ and
$w$) yields $0v+\left(  0v+w\right)  =\underbrace{\left(  0v+0v\right)
}_{=0v}+w=0v+w$. Since $0v+w=\overrightarrow{0}$, this rewrites as
$0v+\overrightarrow{0}=\overrightarrow{0}$.

Property \textbf{(c)} (applied to $0v$ instead of $v$) yields
$0v+\overrightarrow{0}=\overrightarrow{0}+0v=0v$. Comparing
$0v+\overrightarrow{0}=\overrightarrow{0}$ with $0v+\overrightarrow{0}=0v$, we
obtain $0v=\overrightarrow{0}$. Thus, property \textbf{(i)} is proven. Hence,
we have shown that property \textbf{(i)} follows from properties \textbf{(b)},
\textbf{(c)}, \textbf{(d)} and \textbf{(e)}. This proves Proposition
\ref{prop.vectspace.redundant.i}.
\end{proof}

However, we \textbf{cannot} simultaneously omit \textbf{both} properties
\textbf{(d)} and \textbf{(i)} from the definition: Each of them is redundant
provided the other stays in, but without the other they are not redundant.

The vector $\overrightarrow{0}$ in Definition \ref{def.vectspace} is often
written as $0$. This is an \textquotedblleft abuse of
notation\textquotedblright\ (because the vector $\overrightarrow{0}$ is not
literally the number $0$), but mostly harmless as there is rarely a
possibility of confusion.

\subsection{%
%TCIMACRO{\TEXTsymbol{<}}%
%BeginExpansion
$<$%
%EndExpansion
DRAFT%
%TCIMACRO{\TEXTsymbol{>} }%
%BeginExpansion
$>$
%EndExpansion
Examples and constructions of vector spaces}

Before we do anything interesting with vector spaces, let me show various
examples of them:

\begin{example}
\label{exam.vectspace.Rnm}Let $n\in\mathbb{N}$ and $m\in\mathbb{N}$. The set
of all $n\times m$-matrices with real entries shall be denoted by
$\mathbb{R}^{n\times m}$. This set $\mathbb{R}^{n\times m}$ is a vector space
-- or, more precisely, it becomes a vector space if we define the operation
\textquotedblleft$+$\textquotedblright\ as addition of matrices (conveniently,
we have already been denoting this addition by \textquotedblleft%
$+$\textquotedblright\ throughout these notes), define the operation
\textquotedblleft$\cdot$\textquotedblright\ as scaling of matrices (again, we
have fortunately always been calling it \textquotedblleft$\cdot$%
\textquotedblright), and define the zero vector $\overrightarrow{0}$ as the
zero matrix $0_{n\times m}$. Proving that the properties in Definition
\ref{def.vectspace} are satisfied is easy; in fact, they all boil down to
simple facts about matrices.

Thus, $n\times m$-matrices are vectors (in the vector space $\mathbb{R}%
^{n\times m}$). This does not mean that they are row vectors or column
vectors; we are instead using the word \textquotedblleft
vector\textquotedblright\ in its general meaning of \textquotedblleft element
of a vector space\textquotedblright\ here.

Of course, row vectors and column vectors are vectors as well (because they
are matrices). The vector space of all column vectors of size $n$ is
$\mathbb{R}^{n\times1}$ (since column vectors of size $n$ are $n\times
1$-matrices). We will denote this vector space by $\mathbb{R}^{n}$. (Beware
that some authors also use the notation $\mathbb{R}^{n}$ for the vector space
of all row vectors of size $n$, which is $\mathbb{R}^{1\times n}$. Some
authors go as far as pretend that column vectors and row vectors are the same,
but this is dangerous, since matrix multiplication treats them differently!)

Notice that multiplication of matrices does not matter for the vector space
$\mathbb{R}^{n\times m}$. Even if there was no such thing as a product of two
matrices, $\mathbb{R}^{n\times m}$ would still be a vector space, since vector
spaces only need addition and scaling (and $\overrightarrow{0}$), but no
multiplication of vectors.
\end{example}

\begin{definition}
The notations $\mathbb{R}^{n\times m}$ and $\mathbb{R}^{n}$ introduced in
Example \ref{exam.vectspace.Rnm}, as well as the vector space structure on
$\mathbb{R}^{n\times m}$, will be used throughout these notes.
\end{definition}

\begin{example}
\label{exam.vectspace.RC0}Some simpler examples:

\begin{itemize}
\item The set $\mathbb{R}$ of all real numbers itself is a vector space (with
addition being addition, scaling being multiplication, and $\overrightarrow{0}%
$ being the number $0$.)

\item The set $\mathbb{C}$ of all complex numbers is a vector space (with
addition being addition, scaling being multiplication, and $\overrightarrow{0}%
$ being the number $0$.)

\item The one-element set $\left\{  0\right\}  $ is also a vector space (with
addition, scaling and $\overrightarrow{0}$ being defined in the only possible
way -- there is only one choice).
\end{itemize}
\end{example}

\begin{example}
\label{exam.vectspace.Rinf}Consider the set $\mathbb{R}^{\infty}$ of all
infinite sequences $\left(  a_{1},a_{2},a_{3},\ldots\right)  $ of real
numbers. This set $\mathbb{R}^{\infty}$ is also a vector space, with addition
defined by%
\[
\left(  a_{1},a_{2},a_{3},\ldots\right)  +\left(  b_{1},b_{2},b_{3}%
,\ldots\right)  =\left(  a_{1}+b_{1},a_{2}+b_{2},a_{3}+b_{3},\ldots\right)
\]
(that is, entry by entry), scaling defined by%
\[
\lambda\left(  a_{1},a_{2},a_{3},\ldots\right)  =\left(  \lambda a_{1},\lambda
a_{2},\lambda a_{3},\ldots\right)
\]
(thus, again, entry by entry), and $\overrightarrow{0}$ defined by%
\[
\overrightarrow{0}=\left(  0,0,0,\ldots\right)  .
\]
(It makes sense to think of infinite sequences as $1\times\infty$-matrices;
thus, the above definitions of $+$, $\cdot$ and $\overrightarrow{0}$ are
precisely the rules we set for matrices.)
\end{example}

\begin{example}
\label{exam.vectspace.RS}Let $S$ be \textbf{any} set. Consider the set
$\mathbb{R}^{S}$ of all maps from $S$ to $\mathbb{R}$. Then, $\mathbb{R}^{S}$
becomes a vector space, if we define $+$, $\cdot$ and $\overrightarrow{0}$ as follows:

\begin{itemize}
\item If $f\in\mathbb{R}^{S}$ and $g\in\mathbb{R}^{S}$ are two maps, then
their sum $f+g$ is defined to be the map from $S$ to $\mathbb{R}$ that sends
each $s\in S$ to $f\left(  s\right)  +g\left(  s\right)  $. Thus,%
\[
\left(  f+g\right)  \left(  s\right)  =f\left(  s\right)  +g\left(  s\right)
\ \ \ \ \ \ \ \ \ \ \text{for every }s\in S.
\]
This is called \textit{pointwise addition} (because it means that we add two
maps by adding their values at each point).

\item If $f\in\mathbb{R}^{S}$ is a map and $\lambda$ is a number, then the map
$\lambda f$ is defined to be the map from $S$ to $\mathbb{R}$ that sends each
$s\in S$ to $\lambda\cdot f\left(  s\right)  $. Thus,%
\[
\left(  \lambda f\right)  \left(  s\right)  =\lambda\cdot f\left(  s\right)
\ \ \ \ \ \ \ \ \ \ \text{for every }s\in S.
\]
This is called \textit{pointwise scaling} (because it means that we scale a
map by scaling its value at each point).

\item We define $\overrightarrow{0}\in\mathbb{R}^{S}$ to be the map from $S$
to $\mathbb{R}$ that sends each $s\in S$ to $0$. This is called the
\textit{constant-}$0$\textit{ map}, since it is constant and all its values
are $0$.
\end{itemize}

This example can be viewed as a generalization of the previous examples. Namely:

\begin{itemize}
\item An $n\times m$-matrix $A$ with real entries can be viewed as a map from
$\left\{  1,2,\ldots,n\right\}  \times\left\{  1,2,\ldots,m\right\}  $ to
$\mathbb{R}$ (namely, as the map which sends each $\left(  i,j\right)
\in\left\{  1,2,\ldots,n\right\}  \times\left\{  1,2,\ldots,m\right\}  $ to
the $\left(  i,j\right)  $-th entry $A_{i,j}$ of the matrix $A$). Conversely,
each map from $\left\{  1,2,\ldots,n\right\}  \times\left\{  1,2,\ldots
,m\right\}  $ to $\mathbb{R}$ can be encoded by an $n\times m$-matrix (whose
entries are the values of the map). Thus, we can identify the $n\times
m$-matrices (with real entries) with the maps from $\left\{  1,2,\ldots
,n\right\}  \times\left\{  1,2,\ldots,m\right\}  $ to $\mathbb{R}$. In other
words, we can identify the set $\mathbb{R}^{n\times m}$ with the set
$\mathbb{R}^{\left\{  1,2,\ldots,n\right\}  \times\left\{  1,2,\ldots
,m\right\}  }$. It is easily seen that the addition $+$, the scaling $\cdot$
and the zero vector $\overrightarrow{0}$ of the vector space $\mathbb{R}%
^{n\times m}$ (as defined in Example \ref{exam.vectspace.Rnm}) are identical
with the addition $+$, the scaling $\cdot$ and the zero vector
$\overrightarrow{0}$ of the vector space $\mathbb{R}^{\left\{  1,2,\ldots
,n\right\}  \times\left\{  1,2,\ldots,m\right\}  }$ (as defined in the current
example), once we identify the two sets. Hence, the \textbf{vector space}
$\mathbb{R}^{n\times m}$ can be identified with the \textbf{vector space}
$\mathbb{R}^{\left\{  1,2,\ldots,n\right\}  \times\left\{  1,2,\ldots
,m\right\}  }$.

\item The vector space $\mathbb{R}$ (from Example \ref{exam.vectspace.RC0})
can be identified with the vector space $\mathbb{R}^{\left\{  1\right\}  }$.

\item The vector space $\mathbb{C}$ (from Example \ref{exam.vectspace.RC0})
can be identified with the vector space $\mathbb{R}^{\left\{  1,2\right\}  }$
(since a complex number is defined as a pair of two real numbers, and since
the addition of two complex numbers is componentwise, and so is the scaling of
a complex number by a real number).

\item Recall that $\varnothing$ denotes the empty set (i.e., the set $\left\{
{}\right\}  $). The vector space $\left\{  0\right\}  $ (from Example
\ref{exam.vectspace.RC0}) can be identified with the vector space
$\mathbb{R}^{\varnothing}$. Indeed, the only element $0$ of $\left\{
0\right\}  $ can be equated with the only map from $\varnothing$ to
$\mathbb{R}$.

\item The vector space $\mathbb{R}^{\infty}$ (from Example
\ref{exam.vectspace.Rinf}) can be identified with the vector space
$\mathbb{R}^{\left\{  1,2,3,\ldots\right\}  }$.
\end{itemize}

I have not explained what it really means to \textquotedblleft
identify\textquotedblright\ two sets or vector spaces. Roughly speaking,
identifying two sets $S$ and $T$ means matching up each element of $S$ with an
element of $T$ (in such a way that each element of $T$ ends up matched with
exactly one element of $S$) and pretending that each element is identical to
its match. When $S$ and $T$ are vector spaces, one additionally wants to
ensure that this matching has the property that adding two elements of $S$ has
the same result as adding their matches in $T$, and similarly for scaling, and
finally that the zero vector of $S$ is matched with the zero vector of $T$.
This is all satisfied in our above examples. Of course, this is still vague
and imprecise. A rigorous way to formalize the idea of \textquotedblleft
identifying\textquotedblright\ is the concept of \textit{isomorphisms} (i.e.,
invertible \textit{linear maps}) between vector spaces; we shall introduce
this concept later.
\end{example}

\begin{example}
The set of continuous functions from $\mathbb{R}$ to $\mathbb{R}$ is a vector
space as well. Again, addition and scaling are pointwise (i.e., defined in the
same way as in Example \ref{exam.vectspace.RS}), and again the zero vector is
the constant-$0$ map. What makes this definition work is that:

\begin{itemize}
\item the sum of two continuous functions is continuous;

\item the result of scaling a continuous function by a real $\lambda$ is again continuous;

\item the constant-$0$ map is continuous.
\end{itemize}
\end{example}

\begin{example}
The set of all polynomial functions from $\mathbb{R}$ to $\mathbb{R}$ is a
vector space, too. (A function $f:\mathbb{R}\rightarrow\mathbb{R}$ is said to
be a \textit{polynomial function} if there exist some $n\in\mathbb{N}$ and
some $a_{0},a_{1},\ldots,a_{n}\in\mathbb{R}$ such that every $x\in\mathbb{R}$
satisfies $f\left(  x\right)  =a_{0}+a_{1}x+a_{2}x^{2}+\cdots+a_{n}x^{n}$. For
instance, the function $f$ that sends each $x\in\mathbb{R}$ to $2\left(
x+3\right)  ^{2}x^{3}+2$ is a polynomial function, because it satisfies
$f\left(  x\right)  =2\left(  x+3\right)  ^{2}x^{3}+2=2+0x+0x^{2}%
+18x^{3}+12x^{4}+2x^{5}$ for all $x\in\mathbb{R}$.)
\end{example}

\begin{example}
The set of all constant functions from $\mathbb{R}$ to $\mathbb{R}$ is a
vector space as well. But it can be identified with the vector space
$\mathbb{R}$ (see Example \ref{exam.vectspace.RS} for the meaning of
\textquotedblleft identify\textquotedblright); in fact, any real number
$r\in\mathbb{R}$ corresponds to the constant function that sends everything to
$r$.
\end{example}

\begin{example}
You probably have seen at least two notions of \textquotedblleft
vectors\textquotedblright\ in analytic geometry: vectors in the plane, and
vectors in space. The former kind of vectors can be identified with elements
of $\mathbb{R}^{2}$, and the latter can be identified with elements of
$\mathbb{R}^{3}$. Namely:

\begin{itemize}
\item In the plane, a vector that starts at the origin and ends at the point
$\left(  a,b\right)  $ is identified with the column vector $\left(
\begin{array}
[c]{c}%
a\\
b
\end{array}
\right)  \in\mathbb{R}^{2}$. (Every vector in the plane can be made to start
at the origin, and its ending point is then uniquely determined; thus, this
definition is legitimate.)

\item In space, a vector that starts at the origin and ends at the point
$\left(  a,b,c\right)  $ is identified with the column vector $\left(
\begin{array}
[c]{c}%
a\\
b\\
c
\end{array}
\right)  \in\mathbb{R}^{3}$.
\end{itemize}
\end{example}

Let me show two ways to construct new vector spaces from given vector spaces;
this, of course, can be used to obtain lots of further examples.

\begin{definition}
\label{def.vectspace.product}Let $V$ and $W$ be two sets. Recall that $V\times
W$ denotes the set of all pairs $\left(  v,w\right)  $ with $v\in V$ and $w\in
W$. This set $V\times W$ is called the \textit{Cartesian product} (or simply
the \textit{product}) of the two sets $V$ and $W$.

Now, assume that $V$ and $W$ are two vector spaces. Then, we make the
Cartesian product $V\times W$ into a vector space as well. Namely, we define

\begin{itemize}
\item its addition by%
\begin{equation}
\left(  v_{1},w_{1}\right)  +\left(  v_{2},w_{2}\right)  =\left(  v_{1}%
+v_{2},w_{1}+w_{2}\right)  \label{eq.def.vectspace.product.add}%
\end{equation}
(that is, entry by entry),

\item its scaling by%
\begin{equation}
\lambda\left(  v,w\right)  =\left(  \lambda v,\lambda w\right)
\label{eq.def.vectspace.product.scal}%
\end{equation}
(that is, entry by entry), and

\item its zero vector by%
\begin{equation}
\overrightarrow{0}=\left(  \overrightarrow{0},\overrightarrow{0}\right)
\label{eq.def.vectspace.product.0}%
\end{equation}
(that is, entry by entry again).
\end{itemize}

\noindent(Let me dwell on the precise meaning of the equalities
(\ref{eq.def.vectspace.product.add}), (\ref{eq.def.vectspace.product.scal})
and (\ref{eq.def.vectspace.product.0}). The sign \textquotedblleft%
$+$\textquotedblright\ appears three times in
(\ref{eq.def.vectspace.product.add}), and each time it has a slightly
different meaning. Namely, the sign \textquotedblleft$+$\textquotedblright\ on
the left hand side of (\ref{eq.def.vectspace.product.add}) stands for the
addition of the vector space $V\times W$ (which we are defining). The sign
\textquotedblleft$+$\textquotedblright\ in \textquotedblleft$v_{1}+v_{2}%
$\textquotedblright\ stands for the addition of the vector space $V$. The sign
\textquotedblleft$+$\textquotedblright\ in \textquotedblleft$w_{1}+w_{2}%
$\textquotedblright\ stands for the addition of the vector space $W$.
Similarly, the meanings of the expressions \textquotedblleft$\lambda\left(
v,w\right)  $\textquotedblright, \textquotedblleft$\lambda v$%
\textquotedblright\ and \textquotedblleft$\lambda w$\textquotedblright\ in
(\ref{eq.def.vectspace.product.scal}) are slightly different, since each of
these refers to the scaling of a different vector space. Finally, the three
terms \textquotedblleft$\overrightarrow{0}$\textquotedblright\ in
(\ref{eq.def.vectspace.product.0}) are the zero vectors of three different
vector spaces (namely, of $V\times W$, of $V$ and of $W$, in order of appearance).)

The vector space $V\times W$ is called the \textit{Cartesian product} of the
vector spaces $V$ and $W$.

We can similarly define the Cartesian product of several vector spaces
$V_{1},V_{2},\ldots,V_{n}$. It is again a vector space; its elements are
$n$-tuples $\left(  v_{1},v_{2},\ldots,v_{n}\right)  $ for which each $v_{i}$
belongs to the corresponding $V_{i}$; its addition, scaling and zero vector
are again defined entry by entry.
\end{definition}

\begin{definition}
\label{def.vectspace.StoV}Let $V$ be a vector space. Let $S$ be a set. Recall
that $V^{S}$ denotes the set of all maps from $S$ to $V$. We make $V^{S}$ into
a vector space as well. Namely, we define its addition $+$, its scaling
$\cdot$, and its zero vector $\overrightarrow{0}$ as follows:

\begin{itemize}
\item If $f\in V^{S}$ and $g\in V^{S}$ are two maps, then their sum $f+g$ is
defined to be the map from $S$ to $V$ that sends each $s\in S$ to $f\left(
s\right)  +g\left(  s\right)  $ (where the \textquotedblleft$+$%
\textquotedblright\ in \textquotedblleft$f\left(  s\right)  +g\left(
s\right)  $\textquotedblright\ refers to the addition of $V$). Thus,%
\[
\left(  f+g\right)  \left(  s\right)  =f\left(  s\right)  +g\left(  s\right)
\ \ \ \ \ \ \ \ \ \ \text{for every }s\in S.
\]
This is called \textit{pointwise addition} (because it means that we add two
maps by adding their values at each point).

\item If $f\in V^{S}$ is a map and $\lambda$ is a number, then the map
$\lambda f$ is defined to be the map from $S$ to $V$ that sends each $s\in S$
to $\lambda\cdot f\left(  s\right)  $ (where the \textquotedblleft$\cdot
$\textquotedblright\ in \textquotedblleft$\lambda\cdot f\left(  s\right)
$\textquotedblright\ refers to the scaling of $V$). Thus,%
\[
\left(  \lambda f\right)  \left(  s\right)  =\lambda\cdot f\left(  s\right)
\ \ \ \ \ \ \ \ \ \ \text{for every }s\in S.
\]
This is called \textit{pointwise scaling} (because it means that we scale a
map by scaling its value at each point).

\item We define $\overrightarrow{0}\in V^{S}$ to be the map from $S$ to $V$
that sends each $s\in S$ to $\overrightarrow{0}$ (the zero vector of $V$).
This is called the \textit{constant-}$\overrightarrow{0}$\textit{ map}, since
it is constant and all its values are $\overrightarrow{0}$.
\end{itemize}

This example, of course, generalizes Example \ref{exam.vectspace.RS}: If we
set $V=\mathbb{R}$, then $V^{S}$ becomes precisely the vector space
$\mathbb{R}^{S}$ defined in Example \ref{exam.vectspace.RS}.
\end{definition}

\subsection{%
%TCIMACRO{\TEXTsymbol{<}}%
%BeginExpansion
$<$%
%EndExpansion
DRAFT%
%TCIMACRO{\TEXTsymbol{>} }%
%BeginExpansion
$>$
%EndExpansion
(*) The summation sign for vectors}

In Section \ref{sect.intro.sum}, we have introduced the summation sign $\sum$
for sums of numbers; in Section \ref{sect.intro.sum-matrices}, we have
introduced the analogous summation sign $\sum$ for sums of matrices. In the
same way, we can define the summation sign $\sum$ for sums of vectors in an
arbitrary vector space:

\begin{definition}
\label{def.sum.vectors}Let $V$ be a vector space. Let $p$ and $q$ be two
integers such that $p\leq q+1$. Let $v_{p},v_{p+1},\ldots,v_{q}$ be some
vectors in $V$. Then, $\sum_{k=p}^{q}v_{k}$ means the sum $v_{p}%
+v_{p+1}+\cdots+v_{q}$. (This sum is well-defined, since each vector space
satisfies a \textquotedblleft general associativity law\textquotedblright%
\ similar to Proposition \ref{prop.matrix-laws.ga-add}. Moreover, the order of
its addends does not matter, as can be proven using property \textbf{(a)} in
Definition \ref{def.vectspace}.)

The notation $\sum_{k=p}^{q}v_{k}$ is analogous to the notation $\sum
_{k=p}^{q}a_{k}$ introduced in Definition \ref{def.sum} (and to the notation
$\sum_{k=p}^{q}A_{k}$ introduced in Definition \ref{def.sum.matrices}). The
same remarks and clarifications done for the latter notation in Definition
\ref{def.sum.2} apply to the former notation. There is only one difference:
When the sum $\sum_{k=p}^{q}v_{k}$ has no addends (i.e., when $p\geq q+1$),
its value is defined to be the zero vector $\overrightarrow{0}$ (of the vector
space $V$) rather than the number $0$.
\end{definition}

The summation sign $\sum$ for vectors (that we have just defined) generalizes
both the summation sign $\sum$ for numbers and the summation sign $\sum$ for
matrices (since numbers and matrices can both be viewed as vectors).

Most properties of the summation sign $\sum$ that hold for numbers can be
straightforwardly generalized to vectors (in an arbitrary vector space). Some
of them can even be generalized in several ways: For example, in Proposition
\ref{prop.sum.dist}, we can replace the numbers $a_{p},a_{p+1},\ldots,a_{q}$
by vectors (while $b$ remains a number), \textbf{or} we can replace the number
$b$ by a vector (while $a_{p},a_{p+1},\ldots,a_{q}$ remain numbers; for this
generalization, we should rewrite the products $ba_{k}$ and $b\sum_{k=p}%
^{q}a_{k}$ as $a_{k}b$ and $\left(  \sum_{k=p}^{q}a_{k}\right)  b$). (We
cannot replace both $a_{p},a_{p+1},\ldots,a_{q}$ and $b$ by vectors, since the
product of two vectors is not defined!) Let me state the two resulting
generalizations explicitly (with slightly modified names):

\begin{proposition}
\label{prop.sum.vectors.dist1}Let $V$ be a vector space. Let $p$ and $q$ be
two integers such that $p\leq q+1$. Let $v_{p},v_{p+1},\ldots,v_{q}$ be some
vectors in $V$. Let $b$ be a number. Then,
\[
\sum_{k=p}^{q}bv_{k}=b\sum_{k=p}^{q}v_{k}.
\]
(The expression $\sum_{k=p}^{q}bv_{k}$ has to be read as $\sum_{k=p}%
^{q}\left(  bv_{k}\right)  $.)
\end{proposition}

\begin{proposition}
\label{prop.sum.vectors.dist2}Let $V$ be a vector space. Let $p$ and $q$ be
two integers such that $p\leq q+1$. Let $a_{p},a_{p+1},\ldots,a_{q}$ be some
numbers. Let $v\in V$ be a vector. Then,
\[
\sum_{k=p}^{q}a_{k}v=\left(  \sum_{k=p}^{q}a_{k}\right)  v.
\]
(The expression $\sum_{k=p}^{q}a_{k}v$ has to be read as $\sum_{k=p}%
^{q}\left(  a_{k}v\right)  $.)
\end{proposition}

Let me also state the straightforward generalization of Proposition
\ref{prop.sum.delta} to vectors:

\begin{proposition}
\label{prop.sum.vectors.delta}Let $V$ be a vector space. Let $p$ and $q$ be
two integers such that $p\leq q$. Let $r\in\left\{  p,p+1,\ldots,q\right\}  $.
Let $v_{p},v_{p+1},\ldots,v_{q}$ be some vectors in $V$. Then,%
\[
\sum_{k=p}^{q}\delta_{k,r}v_{k}=v_{r}.
\]

\end{proposition}

\subsection{%
%TCIMACRO{\TEXTsymbol{<}}%
%BeginExpansion
$<$%
%EndExpansion
TODO%
%TCIMACRO{\TEXTsymbol{>} }%
%BeginExpansion
$>$
%EndExpansion
Subspaces}

\textbf{References for subspaces:} \cite[\S 4.3]{LaNaSc16} (possibly the best
one), \cite[\S 2.2]{OlvSha06} (focusses on the analysis-related and applied
stuff) and \cite[Two.I.2]{Heffer16}.

When $U$ is a subset of a vector space $V$, we can try to make $U$ itself into
a vector space by \textquotedblleft inheriting\textquotedblright\ the
addition, the scaling and the zero vector from $V$: That is, we define the sum
$v+w$ of two vectors $v\in U$ and $w\in U$ to be the result of adding $v$ and
$w$ as elements of $V$. Similarly, we define $\lambda v$ for $\lambda
\in\mathbb{R}$ and $v\in U$ to be the result of scaling $v$ by $\lambda$ as
element of $V$. Finally, we define the zero vector $\overrightarrow{0}$ of $U$
as the zero vector $\overrightarrow{0}$ of $V$. However, this all works (i.e.,
makes $U$ into a vector space) only if the vectors appearing in these
definitions end up in $U$: For example, our definition of $v+w$ actually
defines an addition on $U$ only if the sum $v+w$ actually belongs to $U$ for
all $v\in U$ and $w\in U$. Similarly, scaling on $U$ is well-defined only if
$\lambda v\in U$ for all $\lambda\in\mathbb{R}$ and $v\in U$. Finally, the
zero vector $\overrightarrow{0}$ of $U$ is well-defined only if the zero
vector $\overrightarrow{0}$ of $V$ actually belongs to $U$. Let me collect all
these conditions in one notion:

\begin{definition}
\label{def.subspace}Let $U$ be a subset of a vector space $V$. We say that $U$
is a \textit{subspace} (or \textit{vector subspace}) of $V$ if the following
three conditions hold:

\textbf{(a)} We have $\overrightarrow{0}\in U$ (where $\overrightarrow{0}$ is
the zero vector of $V$).

\textbf{(b)} We have $v+w\in U$ for all $v\in U$ and $w\in U$.

\textbf{(c)} We have $\lambda v\in U$ for all $\lambda\in\mathbb{R}$ and $v\in
U$.

Note that condition \textbf{(b)} is often put into words as follows:
\textquotedblleft The subset $U$ is closed under addition\textquotedblright%
\ (since it says, roughly speaking, that if you start with elements of $U$,
then you cannot escape $U$ by applying addition). Similarly, condition
\textbf{(c)} is often stated as follows: \textquotedblleft The subset $U$ is
closed under scaling\textquotedblright. Hence, the subset $U$ is a subspace of
$V$ if and only if $U$ contains the zero vector, is closed under addition and
is closed under scaling.
\end{definition}

\begin{proposition}
\label{prop.subspace.space}Let $U$ be a subspace of a vector space $V$. Then,
$U$ becomes a vector space, if we let it \textquotedblleft
inherit\textquotedblright\ the addition $+$, the scaling $\cdot$ and the zero
vector $\overrightarrow{0}$ from $V$. Here, \textquotedblleft
inheriting\textquotedblright\ means that:

\begin{itemize}
\item we define the sum $v+w$ of two vectors $v\in U$ and $w\in U$ to be the
result of adding $v$ and $w$ as elements of $V$.

\item we define $\lambda v$ for $\lambda\in\mathbb{R}$ and $v\in U$ to be the
result of scaling $v$ by $\lambda$ as element of $V$.

\item we define the zero vector $\overrightarrow{0}$ of $U$ as the zero vector
$\overrightarrow{0}$ of $V$.
\end{itemize}
\end{proposition}

\begin{proof}
[Proof of Proposition \ref{prop.subspace.space}.]The \textquotedblleft
inherited\textquotedblright\ operations on $U$ are clearly well-defined. (For
example, the condition \textbf{(b)} in Definition \ref{def.subspace} shows
that the \textquotedblleft inherited\textquotedblright\ addition is actually
an addition on $U$. Similarly, condition \textbf{(c)} shows that the
\textquotedblleft inherited\textquotedblright\ scaling is actually a scaling,
and condition \textbf{(a)} shows that the \textquotedblleft
inherited\textquotedblright\ zero vector actually is in $U$.)

It remains to prove that the ten properties \textbf{(a)}, \textbf{(b)},
\textbf{(c)}, \textbf{(d)}, \textbf{(e)}, \textbf{(f)}, \textbf{(g)},
\textbf{(h)}, \textbf{(i)} and \textbf{(j)} from Definition
\ref{def.vectspace} are satisfied for $U$ instead of $V$. This is almost
completely straightforward: We know that these ten properties are satisfied
for $V$ (since $V$ is a vector space). Since the operations on $U$ are
\textquotedblleft inherited\textquotedblright\ from $V$, this immediately
yields that the nine properties \textbf{(a)}, \textbf{(b)}, \textbf{(c)},
\textbf{(e)}, \textbf{(f)}, \textbf{(g)}, \textbf{(h)}, \textbf{(i)} and
\textbf{(j)} are also satisfied for $U$ instead of $V$. However, this argument
does not prove that property \textbf{(d)} is satisfied for $U$ instead of $V$:
We know that it holds for $V$, but we cannot immediately conclude that it must
also hold for $U$ instead of $V$ (because there is no obvious reason why the
element $w$ whose existence is claimed in property \textbf{(d)} must belong to
$U$). Fortunately, we can ignore property \textbf{(d)}, since Proposition
\ref{prop.vectspace.redundant.d} shows that this property is redundant. Hence,
$U$ is a vector space (since the nine properties \textbf{(a)}, \textbf{(b)},
\textbf{(c)}, \textbf{(e)}, \textbf{(f)}, \textbf{(g)}, \textbf{(h)},
\textbf{(i)} and \textbf{(j)} are satisfied for $U$ instead of $V$). This
proves Proposition \ref{prop.subspace.space}.
\end{proof}

\begin{remark}
Some authors replace condition \textbf{(a)} in Definition \ref{def.subspace}
by the seemingly weaker condition that $U$ be nonempty. However, this weaker
condition is actually equivalent to condition \textbf{(a)} (as long as
condition \textbf{(c)} is assumed). Indeed, if $U$ is nonempty, then we can
pick any $v\in U$, and then we can observe that $\overrightarrow{0}=0v\in U$
(by condition \textbf{(c)}, applied to $\lambda=0$); thus, condition
\textbf{(a)} holds.
\end{remark}

\subsection{%
%TCIMACRO{\TEXTsymbol{<}}%
%BeginExpansion
$<$%
%EndExpansion
DRAFT%
%TCIMACRO{\TEXTsymbol{>} }%
%BeginExpansion
$>$
%EndExpansion
Examples and constructions of subspaces}

\subsubsection{$\left\{  \protect\overrightarrow{0}\right\}  $ and $V$}

Examples of subspaces, and general ways to construct subspaces, abound. Let us
first go for the very lowest-hanging fruits:

\begin{proposition}
\label{prop.subspace.0V}Let $V$ be a vector space.

\textbf{(a)} The subset $\left\{  \overrightarrow{0}\right\}  $ of $V$ is a
subspace of $V$.

\textbf{(b)} The subset $V$ of $V$ is a subspace of $V$.
\end{proposition}

\begin{proof}
[Proof of Proposition \ref{prop.subspace.0V}.]Straightforward, and left to the
reader. (For instance, $\left\{  \overrightarrow{0}\right\}  $ is closed under
addition because the only possible sum of two elements of $\left\{
\overrightarrow{0}\right\}  $ is $\overrightarrow{0}+\overrightarrow{0}%
=\overrightarrow{0}$, which of course belongs to $\left\{  \overrightarrow{0}%
\right\}  $.)
\end{proof}

Please note that the subset $\left\{  \overrightarrow{0}\right\}  $ in
Proposition \ref{prop.subspace.0V} \textbf{(a)} is \textbf{not} the empty set!
(It contains the zero vector $\overrightarrow{0}$, whereas the empty set
contains nothing.) That said, it is \textquotedblleft as close to the empty
set as a subspace of $V$ can get\textquotedblright: It contains only the zero
vector $\overrightarrow{0}$, which any subspace of $V$ is forced to contain.
It is easy to see that $\left\{  \overrightarrow{0}\right\}  $ is the
intersection of all subspaces of $V$. (In contrast, the empty set
$\varnothing$ is the intersection of all \textbf{subsets} of $V$.)

\subsubsection{Some examples of subspaces of $\mathbb{R}^{3}$}

Let us now get our hands dirty and verify some examples of subspaces and non-subspaces:

\begin{example}
\label{exam.subspace.R3}Recall that $\mathbb{R}^{n}$ denotes the vector space
of all column vectors of size $n$. (We defined this in Example
\ref{exam.vectspace.Rnm}.)

\textbf{(a)} Let $A$ be the subset
\[
\left\{  \left(  x_{1},x_{2},x_{3}\right)  ^{T}\in\mathbb{R}^{3}\ \mid
\ x_{1}-x_{2}+2x_{3}=0\right\}
\]
of $\mathbb{R}^{3}$. Then, $A$ is a subspace of $\mathbb{R}^{3}$.

\textbf{(b)} Let $B$ be the subset%
\[
\left\{  \left(  x_{1},x_{2},x_{3}\right)  ^{T}\in\mathbb{R}^{3}\ \mid
\ x_{1}=2x_{2}=3x_{3}\right\}
\]
of $\mathbb{R}^{3}$. Then, $B$ is a subspace of $\mathbb{R}^{3}$.

\textbf{(c)} Let $C$ be the subset%
\[
\left\{  \left(  x_{1},x_{2},x_{3}\right)  ^{T}\in\mathbb{R}^{3}\ \mid
\ x_{1}-x_{2}+2x_{3}=1\right\}
\]
of $\mathbb{R}^{3}$. Then, $C$ is \textbf{not} a subspace of $\mathbb{R}^{3}$.

\textbf{(d)} Let $D$ be the subset%
\[
\left\{  \left(  u,0,2u+v\right)  ^{T}\ \mid\ u,v\in\mathbb{R}\right\}
\]
of $\mathbb{R}^{3}$. (Note that the letter $v$ stands for a number here, not
for a vector.) Then, $D$ is a subspace of $\mathbb{R}^{3}$.

\textbf{(e)} Let $E$ be the subset%
\[
\left\{  \left(  u,0,u+1\right)  ^{T}\ \mid\ u\in\mathbb{R}\right\}
\]
of $\mathbb{R}^{3}$. Then, $E$ is \textbf{not} a subspace of $\mathbb{R}^{3}$.

\textbf{(f)} Let $F$ be the subset%
\[
\left\{  \left(  x_{1},x_{2},x_{3}\right)  ^{T}\ \mid\ x_{1}x_{2}%
x_{3}=0\right\}
\]
of $\mathbb{R}^{3}$. Then, $F$ is \textbf{not} a subspace of $\mathbb{R}^{3}$.

\textbf{(g)} Let $G$ be the subset%
\[
\left\{  \left(  x_{1},x_{2},x_{3}\right)  ^{T}\ \mid\ x_{1},x_{2},x_{3}%
\in\mathbb{Q}\right\}
\]
of $\mathbb{R}^{3}$. (Recall that $\mathbb{Q}$ is the set of all rational
numbers.) Then, $G$ is \textbf{not} a subspace of $\mathbb{R}^{3}$.

\textbf{(h)} Let $H$ be the subset%
\[
\left\{  \left(  u,0,2u+v+1\right)  ^{T}\ \mid\ u,v\in\mathbb{R}\right\}
\]
of $\mathbb{R}^{3}$. Then, $H$ is a subspace of $\mathbb{R}^{3}$. (Actually,
$H=D$.)
\end{example}

\begin{proof}
[Proof of Example \ref{exam.subspace.R3}.]Recall how subspaces are defined
(see Definition \ref{def.subspace}). Thus, in order to prove that some subset
$U$ of $\mathbb{R}^{3}$ is a subspace of $\mathbb{R}^{3}$, we must prove that
it satisfies the three conditions \textbf{(a)}, \textbf{(b)} and \textbf{(c)}
of Definition \ref{def.subspace} (i.e., that it contains the zero vector, is
closed under addition, and is closed under scaling). On the other hand, in
order to prove that some subset $U$ of $\mathbb{R}^{3}$ is \textbf{not} a
subspace of $\mathbb{R}^{3}$, it suffices to prove that it fails \textbf{at
least one} of these three conditions \textbf{(a)}, \textbf{(b)} and
\textbf{(c)}. (Often, a subset $U$ will fail two or all three of these
conditions, but we do not need to check them all: Failing one is enough.) Let
us now apply this strategy to the subsets $A$, $B$, $C$, $D$, $E$, $F$ and $G$:

\textbf{(a)} We want to show that $A$ is a subspace of $\mathbb{R}^{3}$. Thus,
we need to show that $A$ contains the zero vector, is closed under addition,
and is closed under scaling. Let us do this:

\textit{Proof that }$A$\textit{ contains the zero vector:} Recall that the
zero vector of $\mathbb{R}^{3}$ is $0_{3\times1}=\left(  0,0,0\right)  ^{T}$.
This vector $0_{3\times1}$ lies in $A$ if and only if $0-0+2\cdot0=0$ (by the
definition of $A$). Thus, $0_{3\times1}$ lies in $A$ (since $0-0+2\cdot0=0$
holds). In other words, $A$ contains the zero vector.

\textit{Proof that }$A$ \textit{is closed under addition:} Let $v\in A$ and
$w\in A$. We must prove that $v+w\in A$.

Write the vector $v\in A\subseteq\mathbb{R}^{3}$ in the form $v=\left(
v_{1},v_{2},v_{3}\right)  ^{T}$ (for three real numbers $v_{1},v_{2},v_{3}%
\in\mathbb{R}$). Since $\left(  v_{1},v_{2},v_{3}\right)  ^{T}=v\in A$, we
have $v_{1}-v_{2}+2v_{3}=0$.

Write the vector $w\in A\subseteq\mathbb{R}^{3}$ in the form $w=\left(
w_{1},w_{2},w_{3}\right)  ^{T}$ (for three real numbers $w_{1},w_{2},w_{3}%
\in\mathbb{R}$). Since $\left(  w_{1},w_{2},w_{3}\right)  ^{T}=w\in A$, we
have $w_{1}-w_{2}+2w_{3}=0$.

From $v=\left(  v_{1},v_{2},v_{3}\right)  ^{T}$ and $w=\left(  w_{1}%
,w_{2},w_{3}\right)  ^{T}$, we obtain%
\[
v+w=\left(  v_{1},v_{2},v_{3}\right)  ^{T}+\left(  w_{1},w_{2},w_{3}\right)
^{T}=\left(  v_{1}+w_{1},v_{2}+w_{2},v_{3}+w_{3}\right)  ^{T}%
\]
(since column vectors are added entry by entry). Thus, in order to prove that
$v+w\in A$, we must show that $\left(  v_{1}+w_{1}\right)  -\left(
v_{2}+w_{2}\right)  +2\left(  v_{3}+w_{3}\right)  =0$ (by the definition of
$A$). But showing this is easy: Just notice that%
\begin{align*}
&  \left(  v_{1}+w_{1}\right)  -\left(  v_{2}+w_{2}\right)  +2\left(
v_{3}+w_{3}\right) \\
&  =\underbrace{\left(  v_{1}-v_{2}+2v_{3}\right)  }_{=0}+\underbrace{\left(
w_{1}-w_{2}+2w_{3}\right)  }_{=0}=0+0=0.
\end{align*}
Thus, we have proven that $v+w\in A$. This completes the proof of the fact
that $A$ is closed under addition.

\textit{Proof that }$A$ \textit{is closed under scaling:} Let $\lambda
\in\mathbb{R}$ and $v\in A$. We must prove that $\lambda v\in A$.

Write the vector $v\in A\subseteq\mathbb{R}^{3}$ in the form $v=\left(
v_{1},v_{2},v_{3}\right)  ^{T}$ (for three real numbers $v_{1},v_{2},v_{3}%
\in\mathbb{R}$). Since $\left(  v_{1},v_{2},v_{3}\right)  ^{T}=v\in A$, we
have $v_{1}-v_{2}+2v_{3}=0$.

From $v=\left(  v_{1},v_{2},v_{3}\right)  ^{T}$, we obtain%
\[
\lambda v=\lambda\left(  v_{1},v_{2},v_{3}\right)  ^{T}=\left(  \lambda
v_{1},\lambda v_{2},\lambda v_{3}\right)  ^{T}%
\]
(since column vectors are scaled entry by entry). Thus, in order to prove that
$\lambda v\in A$, we must show that $\lambda v_{1}-\lambda v_{2}+2\lambda
v_{3}=0$ (by the definition of $A$). But showing this is easy: Just notice
that%
\[
\lambda v_{1}-\lambda v_{2}+2\lambda v_{3}=\lambda\underbrace{\left(
v_{1}-v_{2}+2v_{3}\right)  }_{=0}=\lambda0=0.
\]
Thus, we have proven that $\lambda v\in A$. This completes the proof of the
fact that $A$ is closed under scaling.

We now have shown that $A$ contains the zero vector, is closed under addition,
and is closed under scaling. Hence, $A$ is a subspace of $\mathbb{R}^{3}$.
Example \ref{exam.subspace.R3} \textbf{(a)} is proven.

\textbf{(b)} The chain of equations $x_{1}=2x_{2}=3x_{3}$ in the definition of
$B$ is equivalent to \textquotedblleft$x_{1}=2x_{2}$ and $2x_{2}=3x_{3}%
$\textquotedblright. The proof of Example \ref{exam.subspace.R3} \textbf{(b)}
proceeds similarly to the above proof of Example \ref{exam.subspace.R3}
\textbf{(a)}, with the only difference that instead of the single equation
$x_{1}-x_{2}+2x_{3}=0$, we now have to keep track of the two equations
$x_{1}=2x_{2}$ and $2x_{2}=3x_{3}$. Let me only show one part of the proof,
namely the verification that $B$ is closed under addition:

\textit{Proof that }$B$ \textit{is closed under addition:} Let $v\in B$ and
$w\in B$. We must prove that $v+w\in B$.

Write the vector $v\in B\subseteq\mathbb{R}^{3}$ in the form $v=\left(
v_{1},v_{2},v_{3}\right)  ^{T}$ (for three real numbers $v_{1},v_{2},v_{3}%
\in\mathbb{R}$). Since $\left(  v_{1},v_{2},v_{3}\right)  ^{T}=v\in B$, we
have $v_{1}=2v_{2}$ and $2v_{2}=3v_{3}$.

Write the vector $w\in B\subseteq\mathbb{R}^{3}$ in the form $w=\left(
w_{1},w_{2},w_{3}\right)  ^{T}$ (for three real numbers $w_{1},w_{2},w_{3}%
\in\mathbb{R}$). Since $\left(  w_{1},w_{2},w_{3}\right)  ^{T}=w\in B$, we
have $w_{1}=2w_{2}$ and $2w_{2}=3w_{3}$.

From $v=\left(  v_{1},v_{2},v_{3}\right)  ^{T}$ and $w=\left(  w_{1}%
,w_{2},w_{3}\right)  ^{T}$, we obtain%
\[
v+w=\left(  v_{1},v_{2},v_{3}\right)  ^{T}+\left(  w_{1},w_{2},w_{3}\right)
^{T}=\left(  v_{1}+w_{1},v_{2}+w_{2},v_{3}+w_{3}\right)  ^{T}%
\]
(since column vectors are added entry by entry). Thus, in order to prove that
$v+w\in B$, we must show that $v_{1}+w_{1}=2\left(  v_{2}+w_{2}\right)  $ and
$2\left(  v_{2}+w_{2}\right)  =3\left(  v_{3}+w_{3}\right)  $ (by the
definition of $B$). But showing this is easy: Just notice that%
\begin{align*}
\underbrace{v_{1}}_{=2v_{2}}+\underbrace{w_{1}}_{=2w_{2}}  &  =2v_{2}%
+2w_{2}=2\left(  v_{2}+w_{2}\right)  \ \ \ \ \ \ \ \ \ \ \text{and}\\
2\left(  v_{2}+w_{2}\right)   &  =\underbrace{2v_{2}}_{=3v_{3}}%
+\underbrace{2w_{2}}_{=3w_{3}}=3v_{3}+3w_{3}=3\left(  v_{3}+w_{3}\right)  .
\end{align*}
Thus, we have proven that $v+w\in B$. This completes the proof of the fact
that $B$ is closed under addition.

Proving that $B$ contains the zero vector and is closed under scaling is left
to the reader. (As I have said, the proofs are similar to the corresponding
proofs for $A$, and the changes that need to be done are essentially the same
as we did in the proof that $B$ is closed under addition.) Thus, Example
\ref{exam.subspace.R3} \textbf{(b)} is proven.

\textbf{(c)} The zero vector $\overrightarrow{0}=0_{3\times1}=\left(
0,0,0\right)  ^{T}$ of $\mathbb{R}^{3}$ does not belong to $C$ (since it does
not satisfy $0-0+2\cdot0=1$). Hence, $C$ does not contain the zero vector. As
a consequence, $C$ cannot be a subspace of $\mathbb{R}^{3}$. Thus, Example
\ref{exam.subspace.R3} \textbf{(c)} is proven.

(Notice that $C$ furthermore fails to be closed under addition, and also fails
to be closed under scaling. Each of these failures gives another reason why
$C$ is not a subspace of $\mathbb{R}^{3}$. We just chose to use the simplest
reason instead, namely that it fails to contain the zero vector.)

\textbf{(d)} We want to show that $D$ is a subspace of $\mathbb{R}^{3}$. Thus,
we need to show that $D$ contains the zero vector, is closed under addition,
and is closed under scaling. Before we do this, let us rewrite the definition
of $D$ as follows:%
\begin{align}
D  &  =\left\{  \left(  u,0,2u+v\right)  ^{T}\ \mid\ u,v\in\mathbb{R}\right\}
\nonumber\\
&  =\left\{  \left(  x,0,2x+y\right)  ^{T}\ \mid\ x,y\in\mathbb{R}\right\}
\label{pf.exam.subspace.R3.d.D}%
\end{align}
(here, we have renamed the indices $u$ and $v$ as $x$ and $y$). (The reason
for rewriting $D$ this way was to get rid of the letters $u$ and $v$; indeed,
we are going to reuse these letters for vectors.)

\textit{Proof that }$D$\textit{ contains the zero vector:} Recall that the
zero vector of $\mathbb{R}^{3}$ is $0_{3\times1}=\left(  0,0,0\right)  ^{T}$.
This vector $0_{3\times1}$ has the form $\left(  u,0,2u+v\right)  ^{T}$ for
some $u,v\in\mathbb{R}$ (namely, for $u=0$ and $v=0$). Thus, it belongs to $D$
(by the definition of $D$). In other words, $D$ contains the zero vector.

\textit{Proof that }$D$ \textit{is closed under addition:} Let $v\in D$ and
$w\in D$. We must prove that $v+w\in D$.

We have $v\in D=\left\{  \left(  x,0,2x+y\right)  ^{T}\ \mid\ x,y\in
\mathbb{R}\right\}  $ (by (\ref{pf.exam.subspace.R3.d.D})). In other words,
$v$ has the form $\left(  x,0,2x+y\right)  ^{T}$ for some $x,y\in\mathbb{R}$.
Fix two such $x,y$, and denote them by $x_{v},y_{v}$. (I do not want to simply
call them $x,y$, because I will introduce two other such $x,y$ shortly.) Thus,
$x_{v},y_{v}$ are real numbers and satisfy $v=\left(  x_{v},0,2x_{v}%
+y_{v}\right)  ^{T}$.

We have $w\in D=\left\{  \left(  x,0,2x+y\right)  ^{T}\ \mid\ x,y\in
\mathbb{R}\right\}  $ (by (\ref{pf.exam.subspace.R3.d.D})). In other words,
$w$ has the form $\left(  x,0,2x+y\right)  ^{T}$ for some $x,y\in\mathbb{R}$.
Fix two such $x,y$, and denote them by $x_{w},y_{w}$. Thus, $x_{w},y_{w}$ are
real numbers and satisfy $w=\left(  x_{w},0,2x_{w}+y_{w}\right)  ^{T}$.

From $v=\left(  x_{v},0,2x_{v}+y_{v}\right)  ^{T}$ and $w=\left(
x_{w},0,2x_{w}+y_{w}\right)  ^{T}$, we obtain%
\begin{align*}
v+w  &  =\left(  x_{v},0,2x_{v}+y_{v}\right)  ^{T}+\left(  x_{w}%
,0,2x_{w}+y_{w}\right)  ^{T}\\
&  =\left(  x_{v}+x_{w},\underbrace{0+0}_{=0},\underbrace{\left(  2x_{v}%
+y_{v}\right)  +\left(  2x_{w}+y_{w}\right)  }_{=2\left(  x_{v}+x_{w}\right)
+\left(  y_{v}+y_{w}\right)  }\right)  ^{T}\\
&  \ \ \ \ \ \ \ \ \ \ \left(  \text{since column vectors are added entry by
entry}\right) \\
&  =\left(  x_{v}+x_{w},0,2\left(  x_{v}+x_{w}\right)  +\left(  y_{v}%
+y_{w}\right)  \right)  ^{T}.
\end{align*}
Thus, the vector $v+w$ has the form $\left(  x,0,2x+y\right)  ^{T}$ for some
$x,y\in\mathbb{R}$ (namely, for $x=x_{v}+x_{w}$ and $y=y_{v}+y_{w}$). In other
words, $v+w\in\left\{  \left(  x,0,2x+y\right)  ^{T}\ \mid\ x,y\in
\mathbb{R}\right\}  $. In view of (\ref{pf.exam.subspace.R3.d.D}), this
rewrites as $v+w\in D$. This completes the proof of the fact that $D$ is
closed under addition.

\textit{Proof that }$D$ \textit{is closed under scaling:} Let $\lambda
\in\mathbb{R}$ and $v\in D$. We must prove that $\lambda v\in D$.

We have $v\in D=\left\{  \left(  x,0,2x+y\right)  ^{T}\ \mid\ x,y\in
\mathbb{R}\right\}  $ (by (\ref{pf.exam.subspace.R3.d.D})). In other words,
$v$ has the form $\left(  x,0,2x+y\right)  ^{T}$ for some $x,y\in\mathbb{R}$.
Fix two such $x,y$, and denote them by $x_{v},y_{v}$. Thus, $x_{v},y_{v}$ are
real numbers and satisfy $v=\left(  x_{v},0,2x_{v}+y_{v}\right)  ^{T}$.

Hence,%
\begin{align*}
\lambda v  &  =\lambda\left(  x_{v},0,2x_{v}+y_{v}\right)  ^{T}=\left(
\lambda x_{v},\underbrace{\lambda0}_{=0},\underbrace{\lambda\left(
2x_{v}+y_{v}\right)  }_{=2\cdot\lambda x_{v}+\lambda y_{v}}\right)  ^{T}\\
&  \ \ \ \ \ \ \ \ \ \ \left(  \text{since column vectors are scaled entry by
entry}\right) \\
&  =\left(  \lambda x_{v},0,2\cdot\lambda x_{v}+\lambda y_{v}\right)  ^{T}.
\end{align*}


Thus, the vector $\lambda v$ has the form $\left(  x,0,2x+y\right)  ^{T}$ for
some $x,y\in\mathbb{R}$ (namely, for $x=\lambda x_{v}$ and $y=\lambda y_{v}$).
In other words, $\lambda v\in\left\{  \left(  x,0,2x+y\right)  ^{T}%
\ \mid\ x,y\in\mathbb{R}\right\}  $. In view of (\ref{pf.exam.subspace.R3.d.D}%
), this rewrites as $\lambda v\in D$. This completes the proof of the fact
that $D$ is closed under scaling.

We now have shown that $D$ contains the zero vector, is closed under addition,
and is closed under scaling. Hence, $D$ is a subspace of $\mathbb{R}^{3}$.
Example \ref{exam.subspace.R3} \textbf{(d)} is proven.

\textbf{(e)} There exists no $u\in\mathbb{R}$ satisfying $\left(
0,0,0\right)  ^{T}=\left(  u,0,u+1\right)  ^{T}$ (because any such $u$ would
have to satisfy $0=u$ and $0=u+1$ at the same time; but this is clearly
impossible). In other words, $\left(  0,0,0\right)  ^{T}$ is \textbf{not} a
vector of the form $\left(  u,0,u+1\right)  ^{T}$ with $u\in\mathbb{R}$. In
other words, $\left(  0,0,0\right)  ^{T}\notin\left\{  \left(  u,0,u+1\right)
^{T}\ \mid\ u\in\mathbb{R}\right\}  $. Since $E=\left\{  \left(
u,0,u+1\right)  ^{T}\ \mid\ u\in\mathbb{R}\right\}  $, this rewrites as
$\left(  0,0,0\right)  ^{T}\notin E$.

But recall that the zero vector of the vector space $\mathbb{R}^{3}$ is
$\left(  0,0,0\right)  ^{T}$. Hence, this zero vector does not lie in $E$
(since $\left(  0,0,0\right)  ^{T}\notin E$). In other words, $E$ does not
contain the zero vector. Consequently, $E$ cannot be a subspace of
$\mathbb{R}^{3}$. Thus, Example \ref{exam.subspace.R3} \textbf{(e)} is proven.

\textbf{(f)} The subset $F$ is not closed under addition. In fact, if we set
$v=\left(  1,0,0\right)  ^{T}$ and $w=\left(  0,1,1\right)  ^{T}$, then $v\in
F$ (since $1\cdot0\cdot0=0$) and $w\in F$ (since $0\cdot1\cdot1=0$), but
$v+w=\left(  1,1,1\right)  ^{T}\notin F$ (since $1\cdot1\cdot1\neq0$).

Since $F$ is not closed under addition, $F$ is not a subspace of
$\mathbb{R}^{3}$ (even though, as the reader can easily check, $F$ contains
the zero vector and is closed under scaling). This proves Example
\ref{exam.subspace.R3} \textbf{(f)}.

\textbf{(g)} The subset $G$ is not closed under scaling. In fact, if we set
$\lambda=\sqrt{2}$ and $v=\left(  1,0,0\right)  ^{T}$, then $v\in G$ (since
$1,0,0\in\mathbb{Q}$), but $\lambda v=\left(  \sqrt{2},0,0\right)  ^{T}\notin
G$ (since not all of $\sqrt{2},0,0$ belong to $\mathbb{Q}$).

Since $G$ is not closed under scaling, $G$ is not a subspace of $\mathbb{R}%
^{3}$ (even though, as the reader can easily check, $G$ contains the zero
vector and is closed under addition). This proves Example
\ref{exam.subspace.R3} \textbf{(g)}.

\textbf{(h)} I claim that $H=D$.

In order to prove this, it suffices to show that $H\subseteq D$ and
$D\subseteq H$.

\textit{Proof of }$H\subseteq D$\textit{:} Let $h\in H$. Thus, $h\in
H=\left\{  \left(  u,0,2u+v+1\right)  ^{T}\ \mid\ u,v\in\mathbb{R}\right\}  $.
In other words, $h$ has the form $\left(  u,0,2u+v+1\right)  ^{T}$ for some
$u,v\in\mathbb{R}$. Consider these $u,v$.

Now, $h=\left(  u,0,2u+v+1\right)  ^{T}$. Hence, $h$ has the form $\left(
x,0,2x+y\right)  ^{T}$ for some $x,y\in\mathbb{R}$ (namely, for $x=u$ and
$y=v+1$). In other words, $h\in\left\{  \left(  x,0,2x+y\right)  ^{T}%
\ \mid\ x,y\in\mathbb{R}\right\}  $. In view of (\ref{pf.exam.subspace.R3.d.D}%
), this rewrites as $h\in D$.

Now, we have proven that $h\in D$ for each $h\in H$. In other words,
$H\subseteq D$.

\textit{Proof of }$D\subseteq H$\textit{:} Let $d\in D$. Then, $d\in
D=\left\{  \left(  x,0,2x+y\right)  ^{T}\ \mid\ x,y\in\mathbb{R}\right\}  $
(by (\ref{pf.exam.subspace.R3.d.D})). In other words, $d$ has the form
$d=\left(  x,0,2x+y\right)  ^{T}$ for some $x,y\in\mathbb{R}$. Consider these
$x,y$.

Now, $d=\left(  x,0,2x+\underbrace{y}_{=\left(  y-1\right)  +1}\right)
^{T}=\left(  x,0,2x+\left(  y-1\right)  +1\right)  ^{T}$. Hence, $d$ has the
form $\left(  u,0,2u+v+1\right)  ^{T}$ for some $u,v\in\mathbb{R}$ (namely,
for $u=x$ and $v=y-1$). In other words, $d\in\left\{  \left(
u,0,2u+v+1\right)  ^{T}\ \mid\ u,v\in\mathbb{R}\right\}  $. In light of
\newline$\left\{  \left(  u,0,2u+v+1\right)  ^{T}\ \mid\ u,v\in\mathbb{R}%
\right\}  =H$, this rewrites as $d\in H$.

We have now shown that $d\in H$ for each $d\in D$. In other words, $D\subseteq
H$.

Combining the relations $H\subseteq D$ and $D\subseteq H$, we obtain $H=D$.
Since $D$ is a subspace of $\mathbb{R}^{3}$ (this was proven in Example
\ref{exam.subspace.R3} \textbf{(d)}), we thus know that $H$ is a subspace of
$\mathbb{R}^{3}$. Example \ref{exam.subspace.R3} \textbf{(h)} is proven.
\end{proof}

\subsubsection{The kernel of a matrix}

You might have spotted some patterns in Example \ref{exam.subspace.R3}. For
instance, you have noticed that the subsets $A$ and $B$ were \textquotedblleft
carved out\textquotedblright\ of $\mathbb{R}^{3}$ by systems of linear
equations with no constant terms (i.e., linear equations of the form
$a_{1}x_{1}+a_{2}x_{2}+a_{3}x_{3}=0$, or equivalent to such equations). This
is a general pattern: Any subset of $\mathbb{R}^{n}$ \textquotedblleft carved
out\textquotedblright\ by a system of linear equations with no constant
terms\footnote{Such systems are called \textit{homogeneous systems} in
\cite[\S 1.8]{OlvSha06}.} is a subspace of $\mathbb{R}^{n}$. Let me state (and
prove) this fact in a slightly cleaner form, replacing the system of linear
equations by a matrix equation\footnote{In fact, recall (from Proposition
\ref{prop.systems.Ax=b}) that a system of linear equations in $n$ unknowns
$x_{1},x_{2},\ldots,x_{n}$ can be rewritten as a single equation $Ax=b$ for
the column vector $x=\left(
\begin{array}
[c]{c}%
x_{1}\\
x_{2}\\
\vdots\\
x_{n}%
\end{array}
\right)  \in\mathbb{R}^{n}$. When the system has no constant terms, the vector
$b$ is the zero vector $0_{m\times1}$, and so the single equation $Ax=b$
becomes $Ax=0_{m\times1}$.}:

\begin{proposition}
\label{prop.subspace.kerA}Let $n\in\mathbb{N}$ and $m\in\mathbb{N}$. Let $A$
be an $m\times n$-matrix. Let $\operatorname*{Ker}A$ denote the subset%
\[
\left\{  x\in\mathbb{R}^{n}\ \mid\ Ax=0_{m\times1}\right\}
\]
of $\mathbb{R}^{n}$. (Recall that $\mathbb{R}^{n}$ stands for the vector space
$\mathbb{R}^{n\times1}$ of all column vectors of size $n$.) Then,
$\operatorname*{Ker}A$ is a subspace of $\mathbb{R}^{n}$.
\end{proposition}

Proposition \ref{prop.subspace.kerA} generalizes both Example
\ref{exam.subspace.R3} \textbf{(a)} and Example \ref{exam.subspace.R3}
\textbf{(b)}.\footnote{Indeed, you can easily check that the subset $A$ of
$\mathbb{R}^{3}$ in Example \ref{exam.subspace.R3} \textbf{(a)} is
$\operatorname*{Ker}\left(
\begin{array}
[c]{ccc}%
1 & -1 & 2
\end{array}
\right)  $, whereas the subset $B$ of $\mathbb{R}^{3}$ in Example
\ref{exam.subspace.R3} \textbf{(b)} is $\operatorname*{Ker}\left(
\begin{array}
[c]{ccc}%
1 & -2 & 0\\
0 & 2 & -3
\end{array}
\right)  $ (since the condition $x_{1}=2x_{2}=3x_{3}$ defining $B$ is
equivalent to the system $\left\{
\begin{array}
[c]{c}%
x_{1}-2x_{2}=0;\\
2x_{2}-3x_{3}=0
\end{array}
\right.  $).} The proof of Proposition \ref{prop.subspace.kerA} is essentially
like the proofs given for those examples, but shorter because working with
matrices is simpler than dealing with lots of single entries:

\begin{proof}
[Proof of Proposition \ref{prop.subspace.kerA}.]We want to show that
$\operatorname*{Ker}A$ is a subspace of $\mathbb{R}^{n}$. Thus, we need to
show that $\operatorname*{Ker}A$ contains the zero vector, is closed under
addition, and is closed under scaling. Let us do this:

\textit{Proof that }$\operatorname*{Ker}A$\textit{ contains the zero vector:}
We have $A0_{n\times1}=0_{m\times1}$. Hence, $0_{n\times1}\in\left\{
x\in\mathbb{R}^{n}\ \mid\ Ax=0_{m\times1}\right\}  =\operatorname*{Ker}A$. In
other words, $\operatorname*{Ker}A$ contains the zero vector (since
$0_{n\times1}$ is the zero vector of the vector space $\mathbb{R}^{n}$).

\textit{Proof that }$\operatorname*{Ker}A$ \textit{is closed under addition:}
Let $v\in\operatorname*{Ker}A$ and $w\in\operatorname*{Ker}A$. We must prove
that $v+w\in\operatorname*{Ker}A$.

We have $v\in\operatorname*{Ker}A=\left\{  x\in\mathbb{R}^{n}\ \mid
\ Ax=0_{m\times1}\right\}  $. In other words, $Av=0_{m\times1}$. Similarly,
$Aw=0_{m\times1}$. Now, $A\left(  v+w\right)  =\underbrace{Av}_{=0_{m\times1}%
}+\underbrace{Aw}_{=0_{m\times1}}=0_{m\times1}+0_{m\times1}=0_{m\times1}$.
Hence, $v+w\in\left\{  x\in\mathbb{R}^{n}\ \mid\ Ax=0_{m\times1}\right\}
=\operatorname*{Ker}A$. This completes the proof of the fact that
$\operatorname*{Ker}A$ is closed under addition.

\textit{Proof that }$\operatorname*{Ker}A$ \textit{is closed under scaling:}
Let $\lambda\in\mathbb{R}$ and $v\in\operatorname*{Ker}A$. We must prove that
$\lambda v\in\operatorname*{Ker}A$.

We have $v\in\operatorname*{Ker}A=\left\{  x\in\mathbb{R}^{n}\ \mid
\ Ax=0_{m\times1}\right\}  $. In other words, $Av=0_{m\times1}$. Now,
$A\left(  \lambda v\right)  =\lambda\underbrace{Av}_{=0_{m\times1}}%
=\lambda0_{m\times1}=0_{m\times1}$. Hence, $\lambda v\in\left\{
x\in\mathbb{R}^{n}\ \mid\ Ax=0_{m\times1}\right\}  =\operatorname*{Ker}A$.
This completes the proof of the fact that $\operatorname*{Ker}A$ is closed
under scaling.

We now have shown that $\operatorname*{Ker}A$ contains the zero vector, is
closed under addition, and is closed under scaling. Hence,
$\operatorname*{Ker}A$ is a subspace of $\mathbb{R}^{n}$. This proves
Proposition \ref{prop.subspace.kerA}.
\end{proof}

\begin{definition}
\label{def.subspace.kerA}Let $n\in\mathbb{N}$ and $m\in\mathbb{N}$. Let $A$ be
an $m\times n$-matrix. The subspace $\operatorname*{Ker}A$ of $\mathbb{R}^{n}$
introduced in Proposition \ref{prop.subspace.kerA} is called the
\textit{kernel} (or the \textit{nullspace}) of $A$. (Some authors also denote
it by $\ker A$.)
\end{definition}

We will see some examples of kernels later; for now, let me only show one
particularly simple example, namely the kernel of a zero matrix:

\begin{proposition}
\label{prop.kernel.zero}Let $n\in\mathbb{N}$ and $m\in\mathbb{N}$. Then,
$\operatorname*{Ker}\left(  0_{m\times n}\right)  =\mathbb{R}^{n}$.
\end{proposition}

\begin{proof}
[Proof of Proposition \ref{prop.kernel.zero}.]The definition of
$\operatorname*{Ker}\left(  0_{m\times n}\right)  $ yields
\[
\operatorname*{Ker}\left(  0_{m\times n}\right)  =\left\{  x\in\mathbb{R}%
^{n}\ \mid\ 0_{m\times n}x=0_{m\times1}\right\}  .
\]
But \textbf{every} $x\in\mathbb{R}^{n}$ satisfies $0_{m\times n}x=0_{m\times
1}$. Hence, $\left\{  x\in\mathbb{R}^{n}\ \mid\ 0_{m\times n}x=0_{m\times
1}\right\}  =\left\{  x\in\mathbb{R}^{n}\right\}  =\mathbb{R}^{n}$. Thus,
$\operatorname*{Ker}\left(  0_{m\times n}\right)  =\left\{  x\in\mathbb{R}%
^{n}\ \mid\ 0_{m\times n}x=0_{m\times1}\right\}  =\mathbb{R}^{n}$. This proves
Proposition \ref{prop.kernel.zero}.
\end{proof}

\subsubsection{The span of $k$ vectors}

Here is another fairly general construction of a subspace:

\begin{definition}
\label{def.subspace.span}Let $V$ be a vector space. Let $v_{1},v_{2}%
,\ldots,v_{k}$ be finitely many vectors in $V$.

\textbf{(a)} A \textit{linear combination} of $v_{1},v_{2},\ldots,v_{k}$ means
a vector of the form%
\[
\lambda_{1}v_{1}+\lambda_{2}v_{2}+\cdots+\lambda_{k}v_{k}%
\ \ \ \ \ \ \ \ \ \ \text{with }\lambda_{1},\lambda_{2},\ldots,\lambda_{k}%
\in\mathbb{R}.
\]
(In words: A \textit{linear combination} of $v_{1},v_{2},\ldots,v_{k}$ means a
vector obtained by first scaling each of $v_{1},v_{2},\ldots,v_{k}$ by some
real number, and then adding the results.)

\textbf{(b)} The \textit{span} of $v_{1},v_{2},\ldots,v_{k}$ is the set of all
linear combinations of $v_{1},v_{2},\ldots,v_{k}$. This set is denoted by
$\operatorname*{span}\left(  v_{1},v_{2},\ldots,v_{k}\right)  $. Many authors
also denote it by $\operatorname{span}\left(  v_{1},v_{2},\ldots,v_{k}\right)
$, but this notation risks confusion with the inner product of two vectors $v$
and $w$ (which will be introduced later, and which will be denoted by $\left<
v,w\right>  $).

Note that $\operatorname{span}\left(  v_{1},v_{2},\ldots,v_{k}\right)  $ is
defined as the set of all linear combinations of $v_{1},v_{2},\ldots,v_{k}$;
but the latter linear combinations are the vectors of the form $\lambda
_{1}v_{1}+\lambda_{2}v_{2}+\cdots+\lambda_{k}v_{k}$ with $\lambda_{1}%
,\lambda_{2},\ldots,\lambda_{k}\in\mathbb{R}$. Hence, $\operatorname{span}%
\left(  v_{1},v_{2},\ldots,v_{k}\right)  $ is the set of all vectors of the
form $\lambda_{1}v_{1}+\lambda_{2}v_{2}+\cdots+\lambda_{k}v_{k}$ with
$\lambda_{1},\lambda_{2},\ldots,\lambda_{k}\in\mathbb{R}$. In other words,%
\begin{equation}
\operatorname{span}\left(  v_{1},v_{2},\ldots,v_{k}\right)  =\left\{
\lambda_{1}v_{1}+\lambda_{2}v_{2}+\cdots+\lambda_{k}v_{k}\ \mid\ \lambda
_{1},\lambda_{2},\ldots,\lambda_{k}\in\mathbb{R}\right\}  .
\label{eq.def.subspace.span.explicit}%
\end{equation}

\end{definition}

\begin{remark}
The span $\operatorname{span}\left(  {}\right)  $ (that is, the span of no
vectors) is the subspace $\left\{  \overrightarrow{0}\right\}  $ of $V$ (not
the empty set!). This is because $\overrightarrow{0}$ is a linear combination
of an empty list of vectors (being the empty sum). This might appear somewhat
counterintuitive, but is natural and important.
\end{remark}

\begin{proposition}
\label{prop.subspace.span}Let $V$ be a vector space. Let $v_{1},v_{2}%
,\ldots,v_{k}$ be finitely many vectors in $V$. Then, $\operatorname{span}%
\left(  v_{1},v_{2},\ldots,v_{k}\right)  $ (that is, the span of $v_{1}%
,v_{2},\ldots,v_{k}$) is a subspace of $V$.
\end{proposition}

\begin{proof}
[Proof of Proposition \ref{prop.subspace.span}.]We want to show that
$\operatorname{span}\left(  v_{1},v_{2},\ldots,v_{k}\right)  $ is a subspace
of $V$. Thus, we need to show that $\operatorname{span}\left(  v_{1}%
,v_{2},\ldots,v_{k}\right)  $ contains the zero vector, is closed under
addition, and is closed under scaling. Let us do this:

\textit{Proof that }$\operatorname{span}\left(  v_{1},v_{2},\ldots
,v_{k}\right)  $\textit{ contains the zero vector:} We have $0v_{1}%
+0v_{2}+\cdots+0v_{k}=\overrightarrow{0}+\overrightarrow{0}+\cdots
+\overrightarrow{0}=\overrightarrow{0}$. Thus, $\overrightarrow{0}%
=0v_{1}+0v_{2}+\cdots+0v_{k}$. Hence, the vector $\overrightarrow{0}$ has the
form $\lambda_{1}v_{1}+\lambda_{2}v_{2}+\cdots+\lambda_{k}v_{k}$ for some
$\lambda_{1},\lambda_{2},\ldots,\lambda_{k}\in\mathbb{R}$ (namely, for
$\lambda_{i}=0$). In other words, $\overrightarrow{0}$ is a linear combination
of $v_{1},v_{2},\ldots,v_{k}$. In other words, $\overrightarrow{0}%
\in\operatorname{span}\left(  v_{1},v_{2},\ldots,v_{k}\right)  $ (since
$\operatorname{span}\left(  v_{1},v_{2},\ldots,v_{k}\right)  $ is the set of
all linear combinations of $v_{1},v_{2},\ldots,v_{k}$). In other words,
$\operatorname{span}\left(  v_{1},v_{2},\ldots,v_{k}\right)  $ contains the
zero vector (since $\overrightarrow{0}$ is the zero vector of the vector space
$V$).

\textit{Proof that }$\operatorname{span}\left(  v_{1},v_{2},\ldots
,v_{k}\right)  $ \textit{is closed under addition:} Let $v\in
\operatorname{span}\left(  v_{1},v_{2},\ldots,v_{k}\right)  $ and
$w\in\operatorname{span}\left(  v_{1},v_{2},\ldots,v_{k}\right)  $. We must
prove that $v+w\in\operatorname{span}\left(  v_{1},v_{2},\ldots,v_{k}\right)
$.

We have
\[
v\in\operatorname{span}\left(  v_{1},v_{2},\ldots,v_{k}\right)  =\left\{
\lambda_{1}v_{1}+\lambda_{2}v_{2}+\cdots+\lambda_{k}v_{k}\ \mid\ \lambda
_{1},\lambda_{2},\ldots,\lambda_{k}\in\mathbb{R}\right\}
\]
(by (\ref{eq.def.subspace.span.explicit})). In other words, there exist some
$\lambda_{1},\lambda_{2},\ldots,\lambda_{k}\in\mathbb{R}$ satisfying
$v=\lambda_{1}v_{1}+\lambda_{2}v_{2}+\cdots+\lambda_{k}v_{k}$. Fix these
$\lambda_{1},\lambda_{2},\ldots,\lambda_{k}$, and denote them by $\mu_{1}%
,\mu_{2},\ldots,\mu_{k}$. Thus, $\mu_{1},\mu_{2},\ldots,\mu_{k}$ are elements
of $\mathbb{R}$ and satisfy $v=\mu_{1}v_{1}+\mu_{2}v_{2}+\cdots+\mu_{k}v_{k}$.

We have
\[
w\in\operatorname{span}\left(  v_{1},v_{2},\ldots,v_{k}\right)  =\left\{
\lambda_{1}v_{1}+\lambda_{2}v_{2}+\cdots+\lambda_{k}v_{k}\ \mid\ \lambda
_{1},\lambda_{2},\ldots,\lambda_{k}\in\mathbb{R}\right\}
\]
(by (\ref{eq.def.subspace.span.explicit})). In other words, there exist some
$\lambda_{1},\lambda_{2},\ldots,\lambda_{k}\in\mathbb{R}$ satisfying
$w=\lambda_{1}v_{1}+\lambda_{2}v_{2}+\cdots+\lambda_{k}v_{k}$. Fix these
$\lambda_{1},\lambda_{2},\ldots,\lambda_{k}$, and denote them by $\kappa
_{1},\kappa_{2},\ldots,\kappa_{k}$. Thus, $\kappa_{1},\kappa_{2},\ldots
,\kappa_{k}$ are elements of $\mathbb{R}$ and satisfy $w=\kappa_{1}%
v_{1}+\kappa_{2}v_{2}+\cdots+\kappa_{k}v_{k}$.

Adding the equalities $v=\mu_{1}v_{1}+\mu_{2}v_{2}+\cdots+\mu_{k}v_{k}$ and
$w=\kappa_{1}v_{1}+\kappa_{2}v_{2}+\cdots+\kappa_{k}v_{k}$, we obtain%
\begin{align*}
v+w  &  =\left(  \mu_{1}v_{1}+\mu_{2}v_{2}+\cdots+\mu_{k}v_{k}\right)
+\left(  \kappa_{1}v_{1}+\kappa_{2}v_{2}+\cdots+\kappa_{k}v_{k}\right) \\
&  =\left(  \mu_{1}v_{1}+\kappa_{1}v_{1}\right)  +\left(  \mu_{2}v_{2}%
+\kappa_{2}v_{2}\right)  +\cdots+\left(  \mu_{k}v_{k}+\kappa_{k}v_{k}\right)
\\
&  =\left(  \mu_{1}+\kappa_{1}\right)  v_{1}+\left(  \mu_{2}+\kappa
_{2}\right)  v_{2}+\cdots+\left(  \mu_{k}+\kappa_{k}\right)  v_{k}.
\end{align*}
Hence, the vector $v+w$ has the form $\lambda_{1}v_{1}+\lambda_{2}v_{2}%
+\cdots+\lambda_{k}v_{k}$ for some $\lambda_{1},\lambda_{2},\ldots,\lambda
_{k}\in\mathbb{R}$ (namely, for $\lambda_{i}=\mu_{i}+\kappa_{i}$). In other
words,
\[
v+w\in\left\{  \lambda_{1}v_{1}+\lambda_{2}v_{2}+\cdots+\lambda_{k}v_{k}%
\ \mid\ \lambda_{1},\lambda_{2},\ldots,\lambda_{k}\in\mathbb{R}\right\}  .
\]
In view of (\ref{eq.def.subspace.span.explicit}), this rewrites as
$v+w\in\operatorname{span}\left(  v_{1},v_{2},\ldots,v_{k}\right)  $. This
completes the proof of the fact that $\operatorname{span}\left(  v_{1}%
,v_{2},\ldots,v_{k}\right)  $ is closed under addition.

\textit{Proof that }$\operatorname{span}\left(  v_{1},v_{2},\ldots
,v_{k}\right)  $ \textit{is closed under scaling:} Let $\lambda\in\mathbb{R}$
and $v\in\operatorname{span}\left(  v_{1},v_{2},\ldots,v_{k}\right)  $. We
must prove that $\lambda v\in\operatorname{span}\left(  v_{1},v_{2}%
,\ldots,v_{k}\right)  $.

We have
\[
v\in\operatorname{span}\left(  v_{1},v_{2},\ldots,v_{k}\right)  =\left\{
\lambda_{1}v_{1}+\lambda_{2}v_{2}+\cdots+\lambda_{k}v_{k}\ \mid\ \lambda
_{1},\lambda_{2},\ldots,\lambda_{k}\in\mathbb{R}\right\}
\]
(by (\ref{eq.def.subspace.span.explicit})). In other words, there exist some
$\lambda_{1},\lambda_{2},\ldots,\lambda_{k}\in\mathbb{R}$ satisfying
$v=\lambda_{1}v_{1}+\lambda_{2}v_{2}+\cdots+\lambda_{k}v_{k}$. Fix these
$\lambda_{1},\lambda_{2},\ldots,\lambda_{k}$, and denote them by $\mu_{1}%
,\mu_{2},\ldots,\mu_{k}$. Thus, $\mu_{1},\mu_{2},\ldots,\mu_{k}$ are elements
of $\mathbb{R}$ and satisfy $v=\mu_{1}v_{1}+\mu_{2}v_{2}+\cdots+\mu_{k}v_{k}$.

Multiplying both sides of the equality $v=\mu_{1}v_{1}+\mu_{2}v_{2}+\cdots
+\mu_{k}v_{k}$ by $\lambda$, we find%
\[
\lambda v=\lambda\left(  \mu_{1}v_{1}+\mu_{2}v_{2}+\cdots+\mu_{k}v_{k}\right)
=\lambda\mu_{1}v_{1}+\lambda\mu_{2}v_{2}+\cdots+\lambda\mu_{k}v_{k}.
\]
Hence, the vector $\lambda v$ has the form $\lambda_{1}v_{1}+\lambda_{2}%
v_{2}+\cdots+\lambda_{k}v_{k}$ for some $\lambda_{1},\lambda_{2}%
,\ldots,\lambda_{k}\in\mathbb{R}$ (namely, for $\lambda_{i}=\lambda\mu_{i}$).
In other words,
\[
\lambda v\in\left\{  \lambda_{1}v_{1}+\lambda_{2}v_{2}+\cdots+\lambda_{k}%
v_{k}\ \mid\ \lambda_{1},\lambda_{2},\ldots,\lambda_{k}\in\mathbb{R}\right\}
.
\]
In view of (\ref{eq.def.subspace.span.explicit}), this rewrites as $\lambda
v\in\operatorname{span}\left(  v_{1},v_{2},\ldots,v_{k}\right)  $. This
completes the proof of the fact that $\operatorname{span}\left(  v_{1}%
,v_{2},\ldots,v_{k}\right)  $ is closed under scaling.

We now have shown that $\operatorname{span}\left(  v_{1},v_{2},\ldots
,v_{k}\right)  $ contains the zero vector, is closed under addition, and is
closed under scaling. Hence, $\operatorname{span}\left(  v_{1},v_{2}%
,\ldots,v_{k}\right)  $ is a subspace of $V$. This proves Proposition
\ref{prop.subspace.span}.
\end{proof}

\begin{example}
It is easy to construct examples of spans. Here is one:

Let $x=\left(  1,0,2\right)  ^{T}$ and $y=\left(  0,0,1\right)  ^{T}$. Then,
$x$ and $y$ are two vectors in $\mathbb{R}^{3}$. What is their span
$\operatorname{span}\left(  x,y\right)  $ ? The equality
(\ref{eq.def.subspace.span.explicit}) (applied to $V=\mathbb{R}^{3}$, $k=2$
and $\left(  v_{1},v_{2},\ldots,v_{k}\right)  =\left(  x,y\right)  $) yields%
\begin{align*}
\operatorname{span}\left(  x,y\right)   &  =\left\{  \lambda_{1}x+\lambda
_{2}y\ \mid\ \lambda_{1},\lambda_{2}\in\mathbb{R}\right\}  =\left\{
u\underbrace{x}_{=\left(  1,0,2\right)  ^{T}}+v\underbrace{y}_{=\left(
0,0,1\right)  ^{T}}\ \mid\ u,v\in\mathbb{R}\right\} \\
&  \ \ \ \ \ \ \ \ \ \ \left(  \text{here, we have renamed the indices
}\lambda_{1}\text{ and }\lambda_{2}\text{ as }u\text{ and }v\right) \\
&  =\left\{  \underbrace{u\left(  1,0,2\right)  ^{T}+v\left(  0,0,1\right)
^{T}}_{\substack{=\left(  u\cdot1+v\cdot0,u\cdot0+v\cdot0,u\cdot
2+v\cdot1\right)  ^{T}\\=\left(  u,0,2u+v\right)  ^{T}}}\ \mid\ u,v\in
\mathbb{R}\right\} \\
&  =\left\{  \left(  u,0,2u+v\right)  ^{T}\ \mid\ u,v\in\mathbb{R}\right\}  .
\end{align*}
This is precisely the set $D$ defined in Example \ref{exam.subspace.R3}
\textbf{(d)}. In particular, this shows once again that the latter set $D$ is
a subspace of $\mathbb{R}^{3}$ (since Proposition \ref{prop.subspace.span}
shows that $\operatorname{span}\left(  x,y\right)  $ is a subspace of
$\mathbb{R}^{3}$). Thus, we have found a new proof of the claim of Example
\ref{exam.subspace.R3} \textbf{(d)}.
\end{example}

The following property of spans is fundamental:

\begin{proposition}
\label{prop.span.uniprop}Let $V$ be a vector space. Let $v_{1},v_{2}%
,\ldots,v_{k}$ be finitely many vectors in $V$.

\textbf{(a)} The vectors $v_{1},v_{2},\ldots,v_{k}$ belong to the span
$\operatorname{span}\left(  v_{1},v_{2},\ldots,v_{k}\right)  $.

\textbf{(b)} If $U$ is a subspace of $V$ that contains the vectors
$v_{1},v_{2},\ldots,v_{k}$, then $\operatorname{span}\left(  v_{1}%
,v_{2},\ldots,v_{k}\right)  \subseteq U$.
\end{proposition}

Propositions \ref{prop.subspace.span} and \ref{prop.span.uniprop} are often
stated together in one single laconic sentence: \textquotedblleft The span
$\operatorname{span}\left(  v_{1},v_{2},\ldots,v_{k}\right)  $ of $k$ vectors
$v_{1},v_{2},\ldots,v_{k}$ is the smallest subspace of $V$ that contains
$v_{1},v_{2},\ldots,v_{k}$\textquotedblright\ (where \textquotedblleft
smallest\textquotedblright\ means that any other subspace of $V$ that contains
$v_{1},v_{2},\ldots,v_{k}$ must contain $\operatorname{span}\left(
v_{1},v_{2},\ldots,v_{k}\right)  $ as a subset). This sentence includes the
claim of Proposition \ref{prop.subspace.span} (since it says that the span
$\operatorname{span}\left(  v_{1},v_{2},\ldots,v_{k}\right)  $ is a subspace
of $V$), the claim of Proposition \ref{prop.span.uniprop} \textbf{(a)} (since
it says that this span contains $v_{1},v_{2},\ldots,v_{k}$), and the claim of
Proposition \ref{prop.span.uniprop} \textbf{(b)} (since it says that this span
is the \textbf{smallest} subspace of $V$ that contains $v_{1},v_{2}%
,\ldots,v_{k}$).

We shall prove Proposition \ref{prop.span.uniprop} later (in Section
\ref{sect.spaces.more-on-subs}).

How to actually compute the kernel of a matrix? The next three examples
illustrate a method:

\begin{example}
\label{exam.kernel.1}Let $n=4$ and $m=3$. Let $A$ be the $m\times n$-matrix
$\left(
\begin{array}
[c]{cccc}%
1 & 3 & 0 & 1\\
2 & 0 & 1 & 2\\
0 & 6 & -2 & 0
\end{array}
\right)  $. Let us find the kernel $\operatorname*{Ker}A$ of $A$. Proposition
\ref{prop.subspace.kerA} tells us that $\operatorname*{Ker}A$ will be a
subspace of $\mathbb{R}^{n}=\mathbb{R}^{4}$; but it does not tell us which
one. To find out, we will have to do some work.

The definition of $\operatorname*{Ker}A$ yields%
\[
\operatorname*{Ker}A=\left\{  x\in\mathbb{R}^{n}\ \mid\ Ax=0_{m\times
1}\right\}  =\left\{  x\in\mathbb{R}^{4}\ \mid\ Ax=0_{3\times1}\right\}
\]
(since $n=4$ and $m=3$). Hence, finding $\operatorname*{Ker}A$ means finding
all vectors $x\in\mathbb{R}^{4}$ satisfying $Ax=0_{3\times1}$. In other words,
it means solving the equation $Ax=0_{3\times1}$. We know how to do this using
Gaussian elimination:

Let $x=\left(  x_{1},x_{2},x_{3},x_{4}\right)  ^{T}\in\mathbb{R}^{4}$. Then,
$Ax=0_{3\times1}$ rewrites as%
\[
\left(
\begin{array}
[c]{cccc}%
1 & 3 & 0 & 1\\
2 & 0 & 1 & 2\\
0 & 6 & -2 & 0
\end{array}
\right)  \left(
\begin{array}
[c]{c}%
x_{1}\\
x_{2}\\
x_{3}\\
x_{4}%
\end{array}
\right)  =\left(
\begin{array}
[c]{c}%
0\\
0\\
0
\end{array}
\right)  .
\]
This is equivalent to the system of equations%
\[
\left\{
\begin{array}
[c]{c}%
1x_{1}+3x_{2}+0x_{3}+1x_{4}=0;\\
2x_{1}+0x_{2}+1x_{3}+2x_{4}=0;\\
0x_{1}+6x_{2}+\left(  -2\right)  x_{3}+0x_{4}=0
\end{array}
\right.  .
\]
The solutions $x$ of this system are precisely the vectors of the form%
\[
x=\left(
\begin{array}
[c]{c}%
-r\\
0\\
0\\
r
\end{array}
\right)  \ \ \ \ \ \ \ \ \ \ \text{for }r\in\mathbb{R}.
\]
Thus,%
\begin{align}
\operatorname*{Ker}A  &  =\left\{  x\in\mathbb{R}^{4}\ \mid\ Ax=0_{3\times
1}\right\} \nonumber\\
&  =\left\{  \left(
\begin{array}
[c]{c}%
-r\\
0\\
0\\
r
\end{array}
\right)  \ \mid\ r\in\mathbb{R}\right\} \label{eq.exam.kernel.1.6a}\\
&  =\left\{  r\left(
\begin{array}
[c]{c}%
-1\\
0\\
0\\
1
\end{array}
\right)  \ \mid\ r\in\mathbb{R}\right\} \label{eq.exam.kernel.1.6b}\\
&  \ \ \ \ \ \ \ \ \ \ \left(  \text{since }\left(
\begin{array}
[c]{c}%
-r\\
0\\
0\\
r
\end{array}
\right)  =r\left(
\begin{array}
[c]{c}%
-1\\
0\\
0\\
1
\end{array}
\right)  \text{ for each }r\in\mathbb{R}\right) \nonumber\\
&  =\operatorname{span}\left(  \left(
\begin{array}
[c]{c}%
-1\\
0\\
0\\
1
\end{array}
\right)  \right)  .\nonumber
\end{align}
We have thus written $\operatorname*{Ker}A$ as a span of vectors (in our case:
one vector) in $\mathbb{R}^{4}$. This is not the only way to do so, but it is
probably the most explicit and simplest way.
\end{example}

\begin{example}
\label{exam.kernel.2}Let $n=3$ and $m=3$. Let $A$ be the $m\times n$-matrix
$\left(
\begin{array}
[c]{ccc}%
1 & 2 & 2\\
2 & 4 & 4\\
3 & 6 & 6
\end{array}
\right)  $. Let us find the kernel $\operatorname*{Ker}A$ of $A$.

The method is the same as in Example \ref{exam.kernel.1}, but the last few
steps are a bit more complicated. Again, finding $\operatorname*{Ker}A$ means
solving the equation $Ax=0_{3\times1}$ (but this time, $x$ is to lie in
$\mathbb{R}^{3}$).

Let $x=\left(  x_{1},x_{2},x_{3}\right)  ^{T}\in\mathbb{R}^{3}$. Then,
$Ax=0_{3\times1}$ rewrites as%
\[
\left(
\begin{array}
[c]{ccc}%
1 & 2 & 2\\
2 & 4 & 4\\
3 & 6 & 6
\end{array}
\right)  \left(
\begin{array}
[c]{c}%
x_{1}\\
x_{2}\\
x_{3}%
\end{array}
\right)  =\left(
\begin{array}
[c]{c}%
0\\
0\\
0
\end{array}
\right)  .
\]
This is equivalent to the system of equations%
\[
\left\{
\begin{array}
[c]{c}%
1x_{1}+2x_{2}+2x_{3}=0;\\
2x_{1}+4x_{2}+4x_{3}=0;\\
3x_{1}+6x_{2}+6x_{3}=0
\end{array}
\right.  .
\]
The solutions $x$ of this system are precisely the vectors of the form%
\[
x=\left(
\begin{array}
[c]{c}%
-2r-2s\\
r\\
s
\end{array}
\right)  \ \ \ \ \ \ \ \ \ \ \text{for }r,s\in\mathbb{R}.
\]
Thus,%
\begin{align}
\operatorname*{Ker}A  &  =\left\{  x\in\mathbb{R}^{3}\ \mid\ Ax=0_{3\times
1}\right\} \nonumber\\
&  =\left\{  \left(
\begin{array}
[c]{c}%
-2r-2s\\
r\\
s
\end{array}
\right)  \ \mid\ r,s\in\mathbb{R}\right\} \label{eq.exam.kernel.2.6a}\\
&  =\left\{  r\left(
\begin{array}
[c]{c}%
-2\\
1\\
0
\end{array}
\right)  +s\left(
\begin{array}
[c]{c}%
-2\\
0\\
1
\end{array}
\right)  \ \mid\ r,s\in\mathbb{R}\right\} \label{eq.exam.kernel.2.6b}\\
&  \ \ \ \ \ \ \ \ \ \ \left(  \text{since }\left(
\begin{array}
[c]{c}%
-2r-2s\\
r\\
s
\end{array}
\right)  =r\left(
\begin{array}
[c]{c}%
-2\\
1\\
0
\end{array}
\right)  +s\left(
\begin{array}
[c]{c}%
-2\\
0\\
1
\end{array}
\right)  \text{ for each }r,s\in\mathbb{R}\right) \nonumber\\
&  =\operatorname{span}\left(  \left(
\begin{array}
[c]{c}%
-2\\
1\\
0
\end{array}
\right)  ,\left(
\begin{array}
[c]{c}%
-2\\
0\\
1
\end{array}
\right)  \right)  .\nonumber
\end{align}
We have thus written $\operatorname*{Ker}A$ as a span of vectors (in our case:
two vectors) in $\mathbb{R}^{3}$. Again, there are various other ways to do so.

Let me explain how I got from (\ref{eq.exam.kernel.2.6a}) to
(\ref{eq.exam.kernel.2.6b}). I have rewritten $\left(
\begin{array}
[c]{c}%
-2r-2s\\
r\\
s
\end{array}
\right)  $ as $r\left(
\begin{array}
[c]{c}%
-2\\
1\\
0
\end{array}
\right)  +s\left(
\begin{array}
[c]{c}%
-2\\
0\\
1
\end{array}
\right)  $. The purpose of this step was to \textquotedblleft push the
variables $r$ and $s$ out of the column vector\textquotedblright, so that the
result would be of the form \textquotedblleft$r$ times a \textbf{constant}
column vector plus $s$ times a \textbf{constant} column
vector\textquotedblright. And the ultimate purpose of this was to write
$\operatorname*{Ker}A$ as a span of two column vectors.

You should be able to do such transformations yourself! To illustrate the
method in a bit more detail, let me state it as a two-step procedure:

\begin{enumerate}
\item First, decompose the vector $\left(
\begin{array}
[c]{c}%
-2r-2s\\
r\\
s
\end{array}
\right)  $ into a sum of two vectors by isolating all multiples of $r$ into
one vector and all multiples of $s$ into another. Thus,%
\[
\left(
\begin{array}
[c]{c}%
-2r-2s\\
r\\
s
\end{array}
\right)  =\left(
\begin{array}
[c]{c}%
-2r\\
r\\
0
\end{array}
\right)  +\left(
\begin{array}
[c]{c}%
-2s\\
0\\
s
\end{array}
\right)  .
\]


\item Now, push the factor $r$ out of the first vector (i.e., rewrite $\left(
%
\begin{array}
[c]{c}%
-2r\\
r\\
0
\end{array}
\right)  $ as $r\left(
\begin{array}
[c]{c}%
-2\\
1\\
0
\end{array}
\right)  $), and push the factor $s$ out of the second vector (i.e., rewrite
$\left(
\begin{array}
[c]{c}%
-2s\\
0\\
s
\end{array}
\right)  $ as $s\left(
\begin{array}
[c]{c}%
-2\\
0\\
1
\end{array}
\right)  $). The result is $r\left(
\begin{array}
[c]{c}%
-2\\
1\\
0
\end{array}
\right)  +s\left(
\begin{array}
[c]{c}%
-2\\
0\\
1
\end{array}
\right)  $.
\end{enumerate}

In hindsight, the passage from (\ref{eq.exam.kernel.1.6a}) to
(\ref{eq.exam.kernel.1.6b}) has been another instance of such a
transformation. Namely, I have rewritten $\left(
\begin{array}
[c]{c}%
-r\\
0\\
0\\
r
\end{array}
\right)  $ as $r\left(
\begin{array}
[c]{c}%
-1\\
0\\
0\\
1
\end{array}
\right)  $ in order to \textquotedblleft push the $r$ out of the column
vector\textquotedblright. Of course, this was simpler because we had only one
variable (hence, no need to isolate) and therefore only one constant vector.

Here is another example of such a transformation:%
\[
\left(
\begin{array}
[c]{c}%
5r+2s+3t\\
t\\
-2r+3s\\
s\\
-3r\\
r\\
0
\end{array}
\right)  =r\left(
\begin{array}
[c]{c}%
5\\
0\\
-2\\
0\\
-3\\
1\\
0
\end{array}
\right)  +s\left(
\begin{array}
[c]{c}%
2\\
0\\
3\\
1\\
0\\
0\\
0
\end{array}
\right)  +t\left(
\begin{array}
[c]{c}%
3\\
1\\
0\\
0\\
0\\
0\\
0
\end{array}
\right)  .
\]

\end{example}

\begin{example}
\label{exam.kernel.3}Let $n=2$ and $m=2$. Let $A$ be the $m\times n$-matrix
$\left(
\begin{array}
[c]{cc}%
1 & 2\\
2 & 3
\end{array}
\right)  $. Let us find the kernel $\operatorname*{Ker}A$ of $A$.

The method is the same as in Example \ref{exam.kernel.1}, and this time we
will actually have it a lot easier, as long as we do not get confused by empty
lists and zero vectors. Finding $\operatorname*{Ker}A$ means solving the
equation $Ax=0_{2\times1}$ for $x\in\mathbb{R}^{2}$.

Let $x=\left(  x_{1},x_{2}\right)  ^{T}\in\mathbb{R}^{2}$. Then,
$Ax=0_{2\times1}$ rewrites as%
\[
\left(
\begin{array}
[c]{cc}%
1 & 2\\
2 & 3
\end{array}
\right)  \left(
\begin{array}
[c]{c}%
x_{1}\\
x_{2}%
\end{array}
\right)  =\left(
\begin{array}
[c]{c}%
0\\
0
\end{array}
\right)  .
\]
This is equivalent to the system of equations%
\[
\left\{
\begin{array}
[c]{c}%
1x_{1}+2x_{2}=0;\\
2x_{1}+3x_{2}=0
\end{array}
\right.  .
\]
The solutions $x$ of this system are precisely the vectors of the form%
\[
x=\left(
\begin{array}
[c]{c}%
0\\
0
\end{array}
\right)  .
\]
Thus,%
\[
\operatorname*{Ker}A=\left\{  x\in\mathbb{R}^{2}\ \mid\ Ax=0_{2\times
1}\right\}  =\left\{  \underbrace{\left(
\begin{array}
[c]{c}%
0\\
0
\end{array}
\right)  }_{=0_{2\times1}}\right\}  =\left\{  0_{2\times1}\right\}  .
\]
This is as simple a result as we can get, but if we actually want to rewrite
$\operatorname*{Ker}A$ as a span, we can achieve this by proceeding as in
Examples \ref{exam.kernel.1} and \ref{exam.kernel.2}:%
\begin{align*}
\operatorname*{Ker}A  &  =\left\{  x\in\mathbb{R}^{2}\ \mid\ Ax=0_{2\times
1}\right\}  =\left\{  \underbrace{\left(
\begin{array}
[c]{c}%
0\\
0
\end{array}
\right)  }_{=\left(  \text{empty sum}\right)  }\right\} \\
&  =\left\{  \left(  \text{empty sum}\right)  \right\}  =\operatorname{span}%
\left(  {}\right)  \ \ \ \ \ \ \ \ \ \ \left(  \text{the span of no
vectors}\right)  .
\end{align*}
Here, we have \textquotedblleft decomposed\textquotedblright\ the zero vector
$\left(
\begin{array}
[c]{c}%
0\\
0
\end{array}
\right)  $ into an empty sum, and realized that the set consisting of the
empty sum is precisely the set of all linear combinations of no vectors, i.e.,
the span of no vectors. Thus, $\operatorname*{Ker}A$ is the span of no vectors.
\end{example}

\subsubsection{The image of a matrix}

A further way to construct subspaces is the following:

\begin{definition}
\label{def.subspace.Im}Let $A$ be an $n\times m$-matrix. We define
$A\mathbb{R}^{m}$ to be the subset $\left\{  Ax\ \mid\ x\in\mathbb{R}%
^{m}\right\}  $ of $\mathbb{R}^{n}$. This subset $A\mathbb{R}^{m}$ is called
the \textit{column space} (or the \textit{image}, or the \textit{range}) of
$A$.
\end{definition}

The name \textquotedblleft column space\textquotedblright\ for $A\mathbb{R}%
^{m}$ in Definition \ref{def.subspace.Im} might appear strange at first, but
we will soon see why it is justified: In Proposition \ref{prop.subspace.Im}
\textbf{(a)} below, we will show that it is the span of the columns of $A$.
The name \textquotedblleft image\textquotedblright\ is due to the fact that
$A\mathbb{R}^{m}$ is the image of the map $\mathbb{R}^{m}\rightarrow
\mathbb{R}^{n},\ x\mapsto Ax$.

Various authors have different notations for the column space $A\mathbb{R}%
^{m}$ of a matrix $A$. It is called $\operatorname*{rng}A$ in \cite{OlvSha06};
other notations for it include $\operatorname*{Range}A$, $\operatorname*{Im}%
A$, and $A\left(  \mathbb{R}^{m}\right)  $.

\begin{proposition}
\label{prop.subspace.Im}Let $A$ be an $n\times m$-matrix.

\textbf{(a)} We have $A\mathbb{R}^{m}=\operatorname{span}\left(
\operatorname*{col}\nolimits_{1}A,\operatorname*{col}\nolimits_{2}%
A,\ldots,\operatorname*{col}\nolimits_{m}A\right)  $.

\textbf{(b)} The set $A\mathbb{R}^{m}$ is a subspace of $\mathbb{R}^{n}$.
\end{proposition}

Before we prove this, let us show a simple lemma:

\begin{lemma}
\label{lem.subspace.Im.Ax}Let $A$ be an $n\times m$-matrix. Let $\left(
x_{1},x_{2},\ldots,x_{m}\right)  ^{T}\in\mathbb{R}^{m}$. Then,%
\[
A\left(  x_{1},x_{2},\ldots,x_{m}\right)  ^{T}=x_{1}\operatorname*{col}%
\nolimits_{1}A+x_{2}\operatorname*{col}\nolimits_{2}A+\cdots+x_{m}%
\operatorname*{col}\nolimits_{m}A.
\]

\end{lemma}

\begin{proof}
[Proof of Lemma \ref{lem.subspace.Im.Ax}.]For each $i\in\left\{
1,2,\ldots,n\right\}  $ and $j\in\left\{  1,2,\ldots,m\right\}  $, we have%
\begin{equation}
\left(  \operatorname*{col}\nolimits_{j}A\right)  _{i,1}=A_{i,j}
\label{pf.lem.subspace.Im.Ax.colji}%
\end{equation}
(because $\operatorname*{col}\nolimits_{j}A$ is the $j$-th column of $A$, and
thus its $i$-th entry is the $\left(  i,j\right)  $-th entry of $A$). Now, for
each $i\in\left\{  1,2,\ldots,n\right\}  $, we have%
\begin{align}
&  \left(  x_{1}\operatorname*{col}\nolimits_{1}A+x_{2}\operatorname*{col}%
\nolimits_{2}A+\cdots+x_{m}\operatorname*{col}\nolimits_{m}A\right)
_{i,1}\nonumber\\
&  =\underbrace{\left(  x_{1}\operatorname*{col}\nolimits_{1}A\right)  _{i,1}%
}_{\substack{=x_{1}\left(  \operatorname*{col}\nolimits_{1}A\right)
_{i,1}\\\text{(since matrices are}\\\text{scaled entry by entry)}%
}}+\underbrace{\left(  x_{2}\operatorname*{col}\nolimits_{2}A\right)  _{i,1}%
}_{\substack{=x_{2}\left(  \operatorname*{col}\nolimits_{2}A\right)
_{i,1}\\\text{(since matrices are}\\\text{scaled entry by entry)}}%
}+\cdots+\underbrace{\left(  x_{m}\operatorname*{col}\nolimits_{m}A\right)
_{i,1}}_{\substack{=x_{m}\left(  \operatorname*{col}\nolimits_{m}A\right)
_{i,1}\\\text{(since matrices are}\\\text{scaled entry by entry)}}}\nonumber\\
&  \ \ \ \ \ \ \ \ \ \ \left(
\begin{array}
[c]{c}%
\text{since matrices are added entry by entry}\\
\text{(more precisely, we are using Proposition \ref{prop.sum.matrices.multi}
here)}%
\end{array}
\right) \nonumber\\
&  =x_{1}\underbrace{\left(  \operatorname*{col}\nolimits_{1}A\right)  _{i,1}%
}_{\substack{=A_{i,1}\\\text{(by (\ref{pf.lem.subspace.Im.Ax.colji}%
),}\\\text{applied to }j=1\text{)}}}+x_{2}\underbrace{\left(
\operatorname*{col}\nolimits_{2}A\right)  _{i,1}}_{\substack{=A_{i,2}%
\\\text{(by (\ref{pf.lem.subspace.Im.Ax.colji}),}\\\text{applied to
}j=2\text{)}}}+\cdots+x_{m}\underbrace{\left(  \operatorname*{col}%
\nolimits_{m}A\right)  _{i,1}}_{\substack{=A_{i,m}\\\text{(by
(\ref{pf.lem.subspace.Im.Ax.colji}),}\\\text{applied to }j=m\text{)}%
}}\nonumber\\
&  =x_{1}A_{i,1}+x_{2}A_{i,2}+\cdots+x_{m}A_{i,m}.
\label{pf.lem.subspace.Im.Ax.0}%
\end{align}


Set $B=\left(  x_{1},x_{2},\ldots,x_{m}\right)  ^{T}$. Then, $B$ is an
$m\times1$-matrix, and its entries are%
\begin{equation}
B_{k,1}=x_{k}\ \ \ \ \ \ \ \ \ \ \text{for each }k\in\left\{  1,2,\ldots
,m\right\}  . \label{pf.lem.subspace.Im.Ax.1}%
\end{equation}


Now, $AB$ is an $n\times1$-matrix (since $A$ is an $n\times m$-matrix and $B$
is an $m\times1$-matrix), i.e., a column vector of size $n$. For each
$i\in\left\{  1,2,\ldots,n\right\}  $, we have%
\begin{align*}
\left(  AB\right)  _{i,1}  &  =A_{i,1}\underbrace{B_{1,1}}_{\substack{=x_{1}%
\\\text{(by (\ref{pf.lem.subspace.Im.Ax.1}),}\\\text{applied to }k=1\text{)}%
}}+A_{i,2}\underbrace{B_{2,1}}_{\substack{=x_{2}\\\text{(by
(\ref{pf.lem.subspace.Im.Ax.1}),}\\\text{applied to }k=2\text{)}}%
}+\cdots+A_{i,m}\underbrace{B_{m,1}}_{\substack{=x_{m}\\\text{(by
(\ref{pf.lem.subspace.Im.Ax.1}),}\\\text{applied to }k=m\text{)}}}\\
&  \ \ \ \ \ \ \ \ \ \ \left(  \text{by Proposition \ref{prop.matrix-prod.rc}
\textbf{(a)}, applied to }p=1\text{ and }j=1\right) \\
&  =A_{i,1}x_{1}+A_{i,2}x_{2}+\cdots+A_{i,m}x_{m}\\
&  =x_{1}A_{i,1}+x_{2}A_{i,2}+\cdots+x_{m}A_{i,m}\\
&  =\left(  x_{1}\operatorname*{col}\nolimits_{1}A+x_{2}\operatorname*{col}%
\nolimits_{2}A+\cdots+x_{m}\operatorname*{col}\nolimits_{m}A\right)  _{i,1}%
\end{align*}
(by (\ref{pf.lem.subspace.Im.Ax.0})). In other words, for each $i\in\left\{
1,2,\ldots,n\right\}  $, the $i$-th entry of the column vector $AB$ equals the
$i$-th entry of the column vector $x_{1}\operatorname*{col}\nolimits_{1}%
A+x_{2}\operatorname*{col}\nolimits_{2}A+\cdots+x_{m}\operatorname*{col}%
\nolimits_{m}A$ (because the $i$-th entry of any column vector $v$ is
$v_{i,1}$). Thus, the column vector $AB$ equals $x_{1}\operatorname*{col}%
\nolimits_{1}A+x_{2}\operatorname*{col}\nolimits_{2}A+\cdots+x_{m}%
\operatorname*{col}\nolimits_{m}A$. In other words,%
\[
AB=x_{1}\operatorname*{col}\nolimits_{1}A+x_{2}\operatorname*{col}%
\nolimits_{2}A+\cdots+x_{m}\operatorname*{col}\nolimits_{m}A.
\]
Since $B=\left(  x_{1},x_{2},\ldots,x_{m}\right)  ^{T}$, this rewrites as%
\[
A\left(  x_{1},x_{2},\ldots,x_{m}\right)  ^{T}=x_{1}\operatorname*{col}%
\nolimits_{1}A+x_{2}\operatorname*{col}\nolimits_{2}A+\cdots+x_{m}%
\operatorname*{col}\nolimits_{m}A.
\]


Lemma \ref{lem.subspace.Im.Ax} is proven.
\end{proof}

\begin{proof}
[Proof of Proposition \ref{prop.subspace.Im}.]\textbf{(a)} Applying
(\ref{eq.def.subspace.span.explicit}) to $V=\mathbb{R}^{n}$, $k=m$ and
$v_{i}=\operatorname*{col}\nolimits_{i}A$, we obtain%
\begin{align}
&  \operatorname{span}\left(  \operatorname*{col}\nolimits_{1}%
A,\operatorname*{col}\nolimits_{2}A,\ldots,\operatorname*{col}\nolimits_{m}%
A\right) \nonumber\\
&  =\left\{  \lambda_{1}\operatorname*{col}\nolimits_{1}A+\lambda
_{2}\operatorname*{col}\nolimits_{2}A+\cdots+\lambda_{m}\operatorname*{col}%
\nolimits_{m}A\ \mid\ \lambda_{1},\lambda_{2},\ldots,\lambda_{m}\in
\mathbb{R}\right\} \nonumber\\
&  =\left\{  x_{1}\operatorname*{col}\nolimits_{1}A+x_{2}\operatorname*{col}%
\nolimits_{2}A+\cdots+x_{m}\operatorname*{col}\nolimits_{m}A\ \mid
\ x_{1},x_{2},\ldots,x_{m}\in\mathbb{R}\right\}
\label{pf.prop.subspace.Im.a.1}%
\end{align}
(here, we have renamed the summation indices $\lambda_{1},\lambda_{2}%
,\ldots,\lambda_{m}$ as $x_{1},x_{2},\ldots,x_{m}$). But%
\begin{align*}
A\mathbb{R}^{m}  &  =\left\{  Ax\ \mid\ x\in\mathbb{R}^{m}\right\}
\ \ \ \ \ \ \ \ \ \ \left(  \text{by the definition of }A\mathbb{R}^{m}\right)
\\
&  =\left\{  \underbrace{A\left(  x_{1},x_{2},\ldots,x_{m}\right)  ^{T}%
}_{\substack{=x_{1}\operatorname*{col}\nolimits_{1}A+x_{2}\operatorname*{col}%
\nolimits_{2}A+\cdots+x_{m}\operatorname*{col}\nolimits_{m}A\\\text{(by Lemma
\ref{lem.subspace.Im.Ax})}}}\ \mid\ x_{1},x_{2},\ldots,x_{m}\in\mathbb{R}%
\right\} \\
&  \ \ \ \ \ \ \ \ \ \ \left(
\begin{array}
[c]{c}%
\text{here, we have substituted }\left(  x_{1},x_{2},\ldots,x_{m}\right)
^{T}\text{ for }x\text{,}\\
\text{since the elements }x\text{ of }\mathbb{R}^{m}\text{ are exactly the
column vectors}\\
\text{of the form }\left(  x_{1},x_{2},\ldots,x_{m}\right)  ^{T}\text{ with
}x_{1},x_{2},\ldots,x_{m}\in\mathbb{R}%
\end{array}
\right) \\
&  =\left\{  x_{1}\operatorname*{col}\nolimits_{1}A+x_{2}\operatorname*{col}%
\nolimits_{2}A+\cdots+x_{m}\operatorname*{col}\nolimits_{m}A\ \mid
\ x_{1},x_{2},\ldots,x_{m}\in\mathbb{R}\right\}  .
\end{align*}
Comparing this with (\ref{pf.prop.subspace.Im.a.1}), we obtain $A\mathbb{R}%
^{m}=\operatorname{span}\left(  \operatorname*{col}\nolimits_{1}%
A,\operatorname*{col}\nolimits_{2}A,\ldots,\operatorname*{col}\nolimits_{m}%
A\right)  $. This proves Proposition \ref{prop.subspace.Im} \textbf{(a)}.

\textbf{(b)} Proposition \ref{prop.subspace.span} (applied to $V=\mathbb{R}%
^{n}$, $k=m$ and $v_{i}=\operatorname*{col}\nolimits_{i}A$) shows that
$\operatorname{span}\left(  \operatorname*{col}\nolimits_{1}%
A,\operatorname*{col}\nolimits_{2}A,\ldots,\operatorname*{col}\nolimits_{m}%
A\right)  $ is a subspace of $\mathbb{R}^{n}$. In other words, $A\mathbb{R}%
^{m}$ is a subspace of $\mathbb{R}^{n}$ (since Proposition
\ref{prop.subspace.Im} \textbf{(a)} yields $A\mathbb{R}^{m}%
=\operatorname{span}\left(  \operatorname*{col}\nolimits_{1}%
A,\operatorname*{col}\nolimits_{2}A,\ldots,\operatorname*{col}\nolimits_{m}%
A\right)  $). This proves Proposition \ref{prop.subspace.Im} \textbf{(b)}.
\end{proof}

The probably simplest example of an image is the following:

\begin{proposition}
\label{prop.Im.0}Let $n\in\mathbb{N}$ and $m\in\mathbb{N}$. Then, $0_{n\times
m}\mathbb{R}^{m}=\left\{  0_{n\times1}\right\}  $.
\end{proposition}

\begin{proof}
[Proof of Proposition \ref{prop.Im.0}.]The definition of $0_{n\times
m}\mathbb{R}^{m}$ yields
\begin{equation}
0_{n\times m}\mathbb{R}^{m}=\left\{  \underbrace{0_{n\times m}x}%
_{=0_{n\times1}}\ \mid\ x\in\mathbb{R}^{m}\right\}  =\left\{  0_{n\times
1}\ \mid\ x\in\mathbb{R}^{m}\right\}  \subseteq\left\{  0_{n\times1}\right\}
. \label{pf.prop.Im.0.dir1}%
\end{equation}
On the other hand, $0_{n\times1}=0_{n\times m}0_{m\times1}\in0_{n\times
m}\mathbb{R}^{m}$ (since $0_{m\times1}\in\mathbb{R}^{m}$), and thus $\left\{
0_{n\times1}\right\}  \subseteq0_{n\times m}\mathbb{R}^{m}$. Combining this
with (\ref{pf.prop.Im.0.dir1}), we obtain $0_{n\times m}\mathbb{R}%
^{m}=\left\{  0_{n\times1}\right\}  $. This proves Proposition \ref{prop.Im.0}.
\end{proof}

\subsubsection{Subspaces from subspaces}

Next, let us see two ways to construct subspaces from other subspaces:

\begin{proposition}
\label{prop.subspace.inter}Let $V$ be a vector space. Let $U_{1}$ and $U_{2}$
be two subspaces of $V$. Then, $U_{1}\cap U_{2}$ is a subspace of $V$.
\end{proposition}

\begin{proof}
[Proof of Proposition \ref{prop.subspace.inter}.]Clearly, $U_{1}\cap U_{2}$ is
a subset of $V$ (since $U_{1}\cap U_{2}\subseteq U_{1}\subseteq V$). We want
to show that $U_{1}\cap U_{2}$ is a subspace of $V$. Thus, we need to show
that $U_{1}\cap U_{2}$ contains the zero vector, is closed under addition, and
is closed under scaling. Let us do this:

\textit{Proof that }$U_{1}\cap U_{2}$\textit{ contains the zero vector:} The
set $U_{1}$ is a subspace of $V$, and thus contains the zero vector. In other
words, $\overrightarrow{0}\in U_{1}$. Similarly, $\overrightarrow{0}\in U_{2}%
$. Combining $\overrightarrow{0}\in U_{1}$ with $\overrightarrow{0}\in U_{2}$,
we obtain $\overrightarrow{0}\in U_{1}\cap U_{2}$. In other words, $U_{1}\cap
U_{2}$ contains the zero vector (since $\overrightarrow{0}$ is the zero vector
of the vector space $V$).

\textit{Proof that }$U_{1}\cap U_{2}$ \textit{is closed under addition:} Let
$v\in U_{1}\cap U_{2}$ and $w\in U_{1}\cap U_{2}$. We must prove that $v+w\in
U_{1}\cap U_{2}$.

We have $v\in U_{1}\cap U_{2}\subseteq U_{1}$ and $w\in U_{1}\cap
U_{2}\subseteq U_{1}$. But $U_{1}$ is a subspace of $V$, and thus is closed
under addition. Hence, from $v\in U_{1}$ and $w\in U_{1}$, we can conclude
that $v+w\in U_{1}$. The same argument (but with the roles of $U_{1}$ and
$U_{2}$ interchanged) shows that $v+w\in U_{2}$.

We now know that the element $v+w$ lies in both sets $U_{1}$ and $U_{2}$;
therefore, $v+w$ lies in their intersection $U_{1}\cap U_{2}$. In other words,
$v+w\in U_{1}\cap U_{2}$. This completes the proof of the fact that $U_{1}\cap
U_{2}$ is closed under addition.

\textit{Proof that }$U_{1}\cap U_{2}$ \textit{is closed under scaling:} This
proof is left to the reader. (It is similar to the above proof that $U_{1}\cap
U_{2}$ is closed under addition.)

We now have shown that $U_{1}\cap U_{2}$ contains the zero vector, is closed
under addition, and is closed under scaling. Hence, $U_{1}\cap U_{2}$ is a
subspace of $V$. This proves Proposition \ref{prop.subspace.inter}.
\end{proof}

\begin{definition}
\label{def.subspace.sum}Let $V$ be a vector space. Let $U_{1}$ and $U_{2}$ be
two subspaces of $V$. Then, $U_{1}+U_{2}$ denotes the set%
\[
\left\{  u_{1}+u_{2}\ \mid\ u_{1}\in U_{1}\text{ and }u_{2}\in U_{2}\right\}
.
\]
This set $U_{1}+U_{2}$ is called the \textit{sum} of the subspaces $U_{1}$ and
$U_{2}$.
\end{definition}

\begin{proposition}
\label{prop.subspace.sum}Let $V$ be a vector space. Let $U_{1}$ and $U_{2}$ be
two subspaces of $V$. Then, $U_{1}+U_{2}$ is a subspace of $V$.
\end{proposition}

\begin{proof}
[Proof of Proposition \ref{prop.subspace.sum}.]We have%
\begin{equation}
U_{1}+U_{2}=\left\{  u_{1}+u_{2}\ \mid\ u_{1}\in U_{1}\text{ and }u_{2}\in
U_{2}\right\}  \label{pf.prop.subspace.sum.def}%
\end{equation}
(by the definition of $U_{1}+U_{2}$). Hence, the elements of $U_{1}+U_{2}$ are
vectors of the form $u_{1}+u_{2}$ with $u_{1}\in U_{1}$ and $u_{2}\in U_{2}$;
clearly, these vectors all belong to $V$. Hence, $U_{1}+U_{2}$ is a subset of
$V$. We want to show that $U_{1}+U_{2}$ is a subspace of $V$. Thus, we need to
show that $U_{1}+U_{2}$ contains the zero vector, is closed under addition,
and is closed under scaling. Let us do this:

\textit{Proof that }$U_{1}+U_{2}$\textit{ contains the zero vector:} The set
$U_{1}$ is a subspace of $V$, and thus contains the zero vector. In other
words, $\overrightarrow{0}\in U_{1}$. Similarly, $\overrightarrow{0}\in U_{2}%
$. Now, $\overrightarrow{0}=\overrightarrow{0}+\overrightarrow{0}$. Hence,
$\overrightarrow{0}$ is a vector of the form $u_{1}+u_{2}$ with $u_{1}\in
U_{1}$ and $u_{2}\in U_{2}$ (namely, $u_{1}=\overrightarrow{0}$ and
$u_{2}=\overrightarrow{0}$). In other words,%
\[
\overrightarrow{0}\in\left\{  u_{1}+u_{2}\ \mid\ u_{1}\in U_{1}\text{ and
}u_{2}\in U_{2}\right\}  .
\]
In light of (\ref{pf.prop.subspace.sum.def}), this rewrites as
$\overrightarrow{0}\in U_{1}+U_{2}$. In other words, $U_{1}+U_{2}$ contains
the zero vector (since $\overrightarrow{0}$ is the zero vector of the vector
space $V$).

\textit{Proof that }$U_{1}+U_{2}$ \textit{is closed under addition:} Let $v\in
U_{1}+U_{2}$ and $w\in U_{1}+U_{2}$. We must prove that $v+w\in U_{1}+U_{2}$.

We have $v\in U_{1}+U_{2}=\left\{  u_{1}+u_{2}\ \mid\ u_{1}\in U_{1}\text{ and
}u_{2}\in U_{2}\right\}  $. In other words, $v$ has the form $v=u_{1}+u_{2}$
for some $u_{1}\in U_{1}$ and $u_{2}\in U_{2}$. Denote these $u_{1}$ and
$u_{2}$ by $v_{1}$ and $v_{2}$. Thus, $v_{1}\in U_{1}$ and $v_{2}\in U_{2}$
and $v=v_{1}+v_{2}$.

We have $w\in U_{1}+U_{2}=\left\{  u_{1}+u_{2}\ \mid\ u_{1}\in U_{1}\text{ and
}u_{2}\in U_{2}\right\}  $. In other words, $w$ has the form $w=u_{1}+u_{2}$
for some $u_{1}\in U_{1}$ and $u_{2}\in U_{2}$. Denote these $u_{1}$ and
$u_{2}$ by $w_{1}$ and $w_{2}$. Thus, $w_{1}\in U_{1}$ and $w_{2}\in U_{2}$
and $w=w_{1}+w_{2}$.

Now, the set $U_{1}$ is a subspace of $V$, and thus is closed under addition.
Hence, from $v_{1}\in U_{1}$ and $w_{1}\in U_{1}$, we can conclude that
$v_{1}+w_{1}\in U_{1}$. Similarly, $v_{2}+w_{2}\in U_{2}$.

Adding the equalities $v=v_{1}+v_{2}$ and $w=w_{1}+w_{2}$, we obtain%
\[
v+w=\left(  v_{1}+v_{2}\right)  +\left(  w_{1}+w_{2}\right)  =\left(
v_{1}+w_{1}\right)  +\left(  v_{2}+w_{2}\right)  .
\]
Hence, $v+w$ has the form $v+w=u_{1}+u_{2}$ for some $u_{1}\in U_{1}$ and
$u_{2}\in U_{2}$ (namely, for $u_{1}=v_{1}+w_{1}$ and $u_{2}=v_{2}+w_{2}%
$)\ \ \ \ \footnote{Here, we are using the observations (made above) that
$v_{1}+w_{1}\in U_{1}$ and $v_{2}+w_{2}\in U_{2}$.}. In other words,%
\[
v+w\in\left\{  u_{1}+u_{2}\ \mid\ u_{1}\in U_{1}\text{ and }u_{2}\in
U_{2}\right\}  .
\]
In light of (\ref{pf.prop.subspace.sum.def}), this rewrites as $v+w\in
U_{1}+U_{2}$. This completes the proof of the fact that $U_{1}+U_{2}$ is
closed under addition.

\textit{Proof that }$U_{1}+U_{2}$ \textit{is closed under scaling:} We leave
this proof to the reader. (It is similar to the above proof that $U_{1}+U_{2}$
is closed under addition.)

We now have shown that $U_{1}+U_{2}$ contains the zero vector, is closed under
addition, and is closed under scaling. Hence, $U_{1}+U_{2}$ is a subspace of
$V$. This proves Proposition \ref{prop.subspace.sum}.
\end{proof}

Generalizing Definition \ref{def.subspace.sum}, we can define the sum
$U_{1}+U_{2}+\cdots+U_{n}$ of any (finite) number of subspaces $U_{1}%
,U_{2},\ldots,U_{n}$ of $V$. This, too, will be a subspace of $V$. The proof
of this is analogous to our above proof of Proposition \ref{prop.subspace.sum}.

\begin{proposition}
\label{prop.subspace.dirprod}Let $V_{1}$ and $V_{2}$ be two vector spaces. Let
$U_{1}$ be a subspace of $V_{1}$. Let $U_{2}$ be a subspace of $V_{2}$. Then,
$U_{1}\times U_{2}$ is a subspace of $V_{1}\times V_{2}$. (See Definition
\ref{def.vectspace.product} for the definition of the vector space
$V_{1}\times V_{2}$.)
\end{proposition}

\begin{proof}
[Proof of Proposition \ref{prop.subspace.dirprod}.]Every element of
$U_{1}\times U_{2}$ is a pair $\left(  u_{1},u_{2}\right)  \in U_{1}\times
U_{2}$. All such pairs $\left(  u_{1},u_{2}\right)  $ belong to $V_{1}\times
V_{2}$\ \ \ \ \footnote{\textit{Proof.} Let $\left(  u_{1},u_{2}\right)  \in
U_{1}\times U_{2}$. Then, $u_{1}\in U_{1}$ and $u_{2}\in U_{2}$. Hence,
$u_{1}\in U_{1}\subseteq V_{1}$ and $u_{2}\in U_{2}\subseteq V_{2}$. Thus,
$\left(  u_{1},u_{2}\right)  \in V_{1}\times V_{2}$, qed.}. Hence,
$U_{1}\times U_{2}$ is a subset of $V_{1}\times V_{2}$. We want to show that
$U_{1}\times U_{2}$ is a subspace of $V_{1}\times V_{2}$. Thus, we need to
show that $U_{1}\times U_{2}$ contains the zero vector, is closed under
addition, and is closed under scaling. Let us do this:

\textit{Proof that }$U_{1}\times U_{2}$\textit{ contains the zero vector:} In
the following, we will denote the zero vector of a vector space $W$ by
$\overrightarrow{0}_{W}$. (We are using this notation instead of the usual
notation $\overrightarrow{0}$, because we want to be able to distinguish
between the zero vectors of different vector spaces.)

The set $U_{1}$ is a subspace of $V_{1}$, and thus contains the zero vector.
In other words, $\overrightarrow{0}_{V_{1}}\in U_{1}$. Similarly,
$\overrightarrow{0}_{V_{2}}\in U_{2}$. Now, the definition of
$\overrightarrow{0}_{V_{1}\times V_{2}}$ yields $\overrightarrow{0}%
_{V_{1}\times V_{2}}=\left(  \overrightarrow{0}_{V_{1}},\overrightarrow{0}%
_{V_{2}}\right)  \in U_{1}\times U_{2}$ (since $\overrightarrow{0}_{V_{1}}\in
U_{1}$ and $\overrightarrow{0}_{V_{2}}\in U_{2}$). In other words,
$U_{1}\times U_{2}$ contains the zero vector (of the vector space $V_{1}\times
V_{2}$).

\textit{Proof that }$U_{1}\times U_{2}$ \textit{is closed under addition:} Let
$v\in U_{1}\times U_{2}$ and $w\in U_{1}\times U_{2}$. We must prove that
$v+w\in U_{1}\times U_{2}$.

We have $v\in U_{1}\times U_{2}$. In other words, $v$ is a pair of the form
$v=\left(  u_{1},u_{2}\right)  $ for some $u_{1}\in U_{1}$ and $u_{2}\in
U_{2}$. Denote these $u_{1}$ and $u_{2}$ by $v_{1}$ and $v_{2}$. Thus,
$v_{1}\in U_{1}$ and $v_{2}\in U_{2}$ and $v=\left(  v_{1},v_{2}\right)  $.

We have $w\in U_{1}\times U_{2}$. In other words, $w$ is a pair of the form
$w=\left(  u_{1},u_{2}\right)  $ for some $u_{1}\in U_{1}$ and $u_{2}\in
U_{2}$. Denote these $u_{1}$ and $u_{2}$ by $w_{1}$ and $w_{2}$. Thus,
$w_{1}\in U_{1}$ and $w_{2}\in U_{2}$ and $w=\left(  w_{1},w_{2}\right)  $.

Now, the set $U_{1}$ is a subspace of $V_{1}$, and thus is closed under
addition. Hence, from $v_{1}\in U_{1}$ and $w_{1}\in U_{1}$, we can conclude
that $v_{1}+w_{1}\in U_{1}$. Similarly, $v_{2}+w_{2}\in U_{2}$.

Adding the equalities $v=\left(  v_{1},v_{2}\right)  $ and $w=\left(
w_{1},w_{2}\right)  $, we obtain%
\[
v+w=\left(  v_{1},v_{2}\right)  +\left(  w_{1},w_{2}\right)  =\left(
v_{1}+w_{1},v_{2}+w_{2}\right)  .
\]
Hence, $v+w$ has the form $v+w=\left(  u_{1},u_{2}\right)  $ for some
$u_{1}\in U_{1}$ and $u_{2}\in U_{2}$ (namely, for $u_{1}=v_{1}+w_{1}$ and
$u_{2}=v_{2}+w_{2}$)\ \ \ \ \footnote{Here, we are using the observations
(made above) that $v_{1}+w_{1}\in U_{1}$ and $v_{2}+w_{2}\in U_{2}$.}. In
other words, $v+w\in U_{1}\times U_{2}$. This completes the proof of the fact
that $U_{1}\times U_{2}$ is closed under addition.

\textit{Proof that }$U_{1}\times U_{2}$ \textit{is closed under scaling:} We
leave this proof to the reader. (It is similar to the above proof that
$U_{1}\times U_{2}$ is closed under addition.)

We now have shown that $U_{1}\times U_{2}$ contains the zero vector, is closed
under addition, and is closed under scaling. Hence, $U_{1}\times U_{2}$ is a
subspace of $V_{1}\times V_{2}$. This proves Proposition
\ref{prop.subspace.dirprod}.
\end{proof}

\subsubsection{Matrix spaces}

As we have seen in Example \ref{exam.vectspace.Rnm}, the $n \times m$-matrices
(with real entries, for given $n$ and $m$) form a vector space, called
$\mathbb{R}^{n \times m}$. How do subspaces of this space look like? Let us
see some examples of subspaces of $\mathbb{R}^{2 \times2}$:

\begin{example}
\label{exam.subspace.R22} \textbf{(a)} Let $A$ be the subset
\[
\left\{  \left(
\begin{array}
[c]{cc}%
a & b\\
c & d
\end{array}
\right)  \in\mathbb{R}^{2 \times2} \mid a+b+c+d = 0 \right\}
\]
of $\mathbb{R}^{2\times2}$. Then, $A$ is a subspace of $\mathbb{R}^{2\times2}$.

\textbf{(b)} Let $B$ be the subset
\[
\left\{  \left(
\begin{array}
[c]{cc}%
a & b\\
c & d
\end{array}
\right)  \in\mathbb{R}^{2 \times2} \mid ab+cd = 0 \right\}
\]
of $\mathbb{R}^{2\times2}$. Then, $B$ is \textbf{not} a subspace of
$\mathbb{R}^{2\times2}$.

\textbf{(c)} Let $C$ be the set of all upper-triangular $2 \times2$-matrices.
Then, $C$ is a subspace of $\mathbb{R}^{2 \times2}$.

\textbf{(d)} Let $D$ be the set of all strictly upper-triangular $2 \times
2$-matrices. Then, $D$ is a subspace of $\mathbb{R}^{2 \times2}$.

\textbf{(e)} Let $E$ be the set of all upper-unitriangular $2 \times
2$-matrices. Then, $E$ is \textbf{not} a subspace of $\mathbb{R}^{2 \times2}$.

\textbf{(f)} Let $F$ be the set of all invertible $2 \times2$-matrices. Then,
$F$ is \textbf{not} a subspace of $\mathbb{R}^{2 \times2}$.

\textbf{(g)} Fix a vector $p\in\mathbb{R}^{2}$. Let $G_{p}$ be the set of all
$2\times2$-matrices $X\in\mathbb{R}^{2\times2}$ satisfying $Xp=0_{2\times1}$.
Then, $G_{p}$ is a subspace of $\mathbb{R}^{2\times2}$.
\end{example}

\begin{proof}
[Proof of Example \ref{exam.subspace.R22}.]This proof is similar to that of
Example \ref{exam.subspace.R3}, except that the vectors are now $2\times
2$-matrices instead of being column vectors of size $3$. Again, in order to
prove that some subset $U$ of $\mathbb{R}^{2\times2}$ is a subspace of
$\mathbb{R}^{2\times2}$, we must prove that it satisfies the three conditions
\textbf{(a)}, \textbf{(b)} and \textbf{(c)} of Definition \ref{def.subspace};
but in order to prove that some subset $U$ of $\mathbb{R}^{2\times2}$ is
\textbf{not} a subspace of $\mathbb{R}^{2\times2}$, it suffices to show that
at least one of these three conditions is violated.

Keep in mind that the zero vector of the vector space $\mathbb{R}^{2\times2}$
is the zero matrix $0_{2\times2}$. We shall sometimes keep using the notation
$\overrightarrow{0}$ for it, just because it is shorter.

\textbf{(a)} We want to show that $A$ is a subspace of $\mathbb{R}^{2\times2}%
$. Thus, we need to show that $A$ contains the zero vector, is closed under
addition, and is closed under scaling. Let us do this:

\textit{Proof that }$A$\textit{ contains the zero vector:} Recall that the
zero vector of $\mathbb{R}^{2\times2}$ is $0_{2\times2}=\left(
\begin{array}
[c]{cc}%
0 & 0\\
0 & 0
\end{array}
\right)  $. This vector $0_{2\times2}$ lies in $A$ if and only if $0+0+0+0=0$
(by the definition of $A$). Thus, $0_{2\times2}$ lies in $A$ (since
$0+0+0+0=0$ holds). In other words, $A$ contains the zero vector.

\textit{Proof that }$A$ \textit{is closed under addition:} Let $v\in A$ and
$w\in A$. We must prove that $v+w\in A$.

Write the vector\footnote{It might appear to refer to the elements of $A$ as
vectors (after all, they are $2\times2$-matrices), and to denote them by
lowercase letters. This should remind you once again that vectors (in the
general meaning of this word -- i.e., elements of a vector space) can be
almost anything.} $v\in A\subseteq\mathbb{R}^{2\times2}$ in the form
$v=\left(
\begin{array}
[c]{cc}%
v_{1,1} & v_{1,2}\\
v_{2,1} & v_{2,2}%
\end{array}
\right)  $ (for four real numbers $v_{1,1},v_{1,2},v_{2,1},v_{2,2}%
\in\mathbb{R}$). Since $\left(
\begin{array}
[c]{cc}%
v_{1,1} & v_{1,2}\\
v_{2,1} & v_{2,2}%
\end{array}
\right)  =v\in A$, we have $v_{1,1}+v_{1,2}+v_{2,1}+v_{2,2}=0$.

Write the vector $w\in A\subseteq\mathbb{R}^{2\times2}$ in the form $w=\left(
%
\begin{array}
[c]{cc}%
w_{1,1} & w_{1,2}\\
w_{2,1} & w_{2,2}%
\end{array}
\right)  $ (for four real numbers $w_{1,1},w_{1,2},w_{2,1},w_{2,2}%
\in\mathbb{R}$). Since $\left(
\begin{array}
[c]{cc}%
w_{1,1} & w_{1,2}\\
w_{2,1} & w_{2,2}%
\end{array}
\right)  =w\in A$, we have $w_{1,1}+w_{1,2}+w_{2,1}+w_{2,2}=0$.

From $v=\left(
\begin{array}
[c]{cc}%
v_{1,1} & v_{1,2}\\
v_{2,1} & v_{2,2}%
\end{array}
\right)  $ and $w=\left(
\begin{array}
[c]{cc}%
w_{1,1} & w_{1,2}\\
w_{2,1} & w_{2,2}%
\end{array}
\right)  $, we obtain%
\[
v+w=\left(
\begin{array}
[c]{cc}%
v_{1,1} & v_{1,2}\\
v_{2,1} & v_{2,2}%
\end{array}
\right)  +\left(
\begin{array}
[c]{cc}%
w_{1,1} & w_{1,2}\\
w_{2,1} & w_{2,2}%
\end{array}
\right)  =\left(
\begin{array}
[c]{cc}%
v_{1,1}+w_{1,1} & v_{1,2}+w_{1,2}\\
v_{2,1}+w_{2,1} & v_{2,2}+w_{2,2}%
\end{array}
\right)
\]
(since matrices are added entry by entry). Thus, in order to prove that
$v+w\in A$, we must show that
\[
\left(  v_{1,1}+w_{1,1}\right)  +\left(  v_{1,2}+w_{1,2}\right)  +\left(
v_{2,1}+w_{2,1}\right)  +\left(  v_{2,2}+w_{2,2}\right)  =0
\]
(by the definition of $A$). But showing this is easy: Just notice that%
\begin{align*}
&  \left(  v_{1,1}+w_{1,1}\right)  +\left(  v_{1,2}+w_{1,2}\right)  +\left(
v_{2,1}+w_{2,1}\right)  +\left(  v_{2,2}+w_{2,2}\right) \\
&  =\underbrace{\left(  v_{1,1}+v_{1,2}+v_{2,1}+v_{2,2}\right)  }%
_{=0}+\underbrace{\left(  w_{1,1}+w_{1,2}+w_{2,1}+w_{2,2}\right)  }%
_{=0}=0+0=0.
\end{align*}
Thus, we have proven that $v+w\in A$. This completes the proof of the fact
that $A$ is closed under addition.

\textit{Proof that }$A$ \textit{is closed under scaling:} We leave this proof
to the reader again. (It is similar to the proof of the fact that $A$ is
closed under addition, except that now we must reason about $\lambda v$
instead of $v+w$.)

We now have shown that $A$ contains the zero vector, is closed under addition,
and is closed under scaling. Hence, $A$ is a subspace of $\mathbb{R}%
^{2\times2}$. Example \ref{exam.subspace.R22} \textbf{(a)} is proven.

\textbf{(b)} The vector $\left(
\begin{array}
[c]{cc}%
1 & 0\\
0 & 1
\end{array}
\right)  \in\mathbb{R}^{2\times2}$ belongs to $B$ (since $1\cdot0+0\cdot1=0$).
So does the vector $\left(
\begin{array}
[c]{cc}%
1 & 0\\
1 & 0
\end{array}
\right)  \in\mathbb{R}^{2\times2}$ (since $1\cdot0+1\cdot0=0$). But the sum of
these vectors does not (because it is $\left(
\begin{array}
[c]{cc}%
1 & 0\\
0 & 1
\end{array}
\right)  +\left(
\begin{array}
[c]{cc}%
1 & 0\\
1 & 0
\end{array}
\right)  =\left(
\begin{array}
[c]{cc}%
2 & 0\\
1 & 1
\end{array}
\right)  $, which does not satisfy $2\cdot0+1\cdot1=0$). Hence, $B$ is not
closed under addition. Thus, $B$ is not a subspace of $\mathbb{R}^{2\times2}$.
Example \ref{exam.subspace.R22} \textbf{(b)} is proven.

\textbf{(c)} We want to show that $C$ is a subspace of $\mathbb{R}^{2\times2}%
$. Thus, we need to show that $C$ contains the zero vector, is closed under
addition, and is closed under scaling. Let me show only one of these three steps:

\textit{Proof that }$C$ \textit{is closed under addition:} Let $v\in C$ and
$w\in C$. We must prove that $v+w\in C$.

The matrix $v$ belongs to $C$. In other words, $v$ is upper-triangular (since
$C$ is the set of all upper-triangular $2\times2$-matrices). In other words,%
\begin{equation}
v_{i,j}=0\ \ \ \ \ \ \ \ \ \ \text{whenever }i>j
\label{pf.exam.subspace.R22.c.v}%
\end{equation}
(because this is how \textquotedblleft upper-triangular\textquotedblright\ was
defined). The same argument (but for $w$ instead of $v$) yields%
\begin{equation}
w_{i,j}=0\ \ \ \ \ \ \ \ \ \ \text{whenever }i>j.
\label{pf.exam.subspace.R22.c.w}%
\end{equation}


Now, every $\left(  i,j\right)  \in\left\{  1,2\right\}  \times\left\{
1,2\right\}  $ satisfying $i>j$ must satisfy%
\begin{align*}
\left(  v+w\right)  _{i,j}  &  =\underbrace{v_{i,j}}_{\substack{=0\\\text{(by
(\ref{pf.exam.subspace.R22.c.v}))}}}+\underbrace{w_{i,j}}%
_{\substack{=0\\\text{(by (\ref{pf.exam.subspace.R22.c.w}))}}%
}\ \ \ \ \ \ \ \ \ \ \left(  \text{since matrices are added entry by
entry}\right) \\
&  =0+0=0.
\end{align*}
In other words,
\[
\left(  v+w\right)  _{i,j}=0\ \ \ \ \ \ \ \ \ \ \text{whenever }i>j.
\]
In other words, the matrix $v+w$ is upper-triangular (since this is how
\textquotedblleft upper-triangular\textquotedblright\ was defined). In other
words, $v+w\in C$ (since $C$ is the set of all upper-triangular $2\times
2$-matrices). This completes the proof of the fact that $C$ is closed under addition.

As I have promised, I am omitting the proofs of the facts that $C$ contains
the zero vector and is closed under scaling. (They are similar to the above
proof, and the modifications necessary to obtain them should be obvious by
now.) Thus, Example \ref{exam.subspace.R22} \textbf{(c)} is proven.

\textbf{(d)} The proof of Example \ref{exam.subspace.R22} \textbf{(d)} is
analogous to the above proof of Example \ref{exam.subspace.R22} \textbf{(c)},
except that the word \textquotedblleft upper-triangular\textquotedblright%
\ must now be replaced by \textquotedblleft strictly
upper-triangular\textquotedblright, and that the condition \textquotedblleft%
$i>j$\textquotedblright\ must be replaced by \textquotedblleft$i\geq
j$\textquotedblright. (And, of course, the symbol \textquotedblleft%
$C$\textquotedblright\ must be replaced by \textquotedblleft$D$%
\textquotedblright.)

\textbf{(e)} The vector $\left(
\begin{array}
[c]{cc}%
1 & 0\\
0 & 1
\end{array}
\right)  \in\mathbb{R}^{2\times2}$ belongs to $E$ (since it is an
upper-unitriangular $2\times2$-matrix). But the sum of this vector with itself
does not (since it is $\left(
\begin{array}
[c]{cc}%
1 & 0\\
0 & 1
\end{array}
\right)  +\left(
\begin{array}
[c]{cc}%
1 & 0\\
0 & 1
\end{array}
\right)  =\left(
\begin{array}
[c]{cc}%
2 & 0\\
0 & 2
\end{array}
\right)  $, which is not upper-unitriangular). Thus, $E$ is not closed under
addition. Thus, $E$ is not a subspace of $\mathbb{R}^{2\times2}$. Example
\ref{exam.subspace.R22} \textbf{(e)} is proven.

\textbf{(f)} The vectors $\left(
\begin{array}
[c]{cc}%
1 & 0\\
0 & 1
\end{array}
\right)  \in\mathbb{R}^{2\times2}$ and $\left(
\begin{array}
[c]{cc}%
0 & 1\\
1 & 0
\end{array}
\right)  \in\mathbb{R}^{2\times2}$ both belong to $F$ (since they both are
invertible matrices\footnote{Actually, each of them is its own inverse.}). But
their sum does not (because it is $\left(
\begin{array}
[c]{cc}%
1 & 1\\
1 & 1
\end{array}
\right)  $, which is not invertible). Hence, $F$ is not closed under addition.
Thus, $F$ is not a subspace of $\mathbb{R}^{2\times2}$. Example
\ref{exam.subspace.R22} \textbf{(f)} is proven.

\textbf{(g)} We want to show that $G_{p}$ is a subspace of $\mathbb{R}%
^{2\times2}$. Thus, we need to show that $G_{p}$ contains the zero vector, is
closed under addition, and is closed under scaling.

\textit{Proof that }$G_{p}$\textit{ contains the zero vector:} Recall that the
zero vector of $\mathbb{R}^{2\times2}$ is $0_{2\times2}$. This vector
$0_{2\times2}$ satisfies $0_{2\times2}p=0_{2\times1}$, and thus belongs to
$G_{p}$ (by the definition of $G_{p}$). In other words, $G_{p}$ contains the
zero vector.

\textit{Proof that }$G_{p}$ \textit{is closed under addition:} Let $v\in
G_{p}$ and $w\in G_{p}$. We must prove that $v+w\in G_{p}$.

We have $v\in G_{p}$. In other words, $vp=0_{2\times1}$ (by the definition of
$G_{p}$). (Keep in mind that $v$ is a $2\times2$-matrix.)

We have $w\in G_{p}$. In other words, $wp=0_{2\times1}$ (by the definition of
$G_{p}$).

Now, $\left(  v+w\right)  p=\underbrace{vp}_{=0_{2\times1}}+\underbrace{wp}%
_{=0_{2\times1}}=0_{2\times1}+0_{2\times1}=0_{2\times1}$. In other words,
$v+w\in G_{p}$ (by the definition of $G_{p}$). This completes the proof of the
fact that $G_{p}$ is closed under addition.

\textit{Proof that }$G_{p}$ \textit{is closed under scaling:} This proof is
left to the reader.

We now have shown that $G_{p}$ contains the zero vector, is closed under
addition, and is closed under scaling. Hence, $G_{p}$ is a subspace of
$\mathbb{R}^{2\times2}$. Example \ref{exam.subspace.R22} \textbf{(g)} is proven.
\end{proof}

[...]

\begin{todo}
Polynomials $p$ such that $p\left(  3\right)  =0$.
\end{todo}

\begin{todo}
Functions $f$ such that $f$ is continuous.
\end{todo}

\begin{todo}
Do functions $\left[  -1,1\right]  \rightarrow\mathbb{R}$ form a subspace of
the functions $\mathbb{R}\rightarrow\mathbb{R}$ ? No, since they are not even
a subset.
\end{todo}

\begin{todo}
Do functions $\mathbb{R}\rightarrow\mathbb{R}$ with image in $\left[
-1,1\right]  $ form a subspace of the functions $\mathbb{R}\rightarrow
\mathbb{R}$ ? No (e.g., scaling fails).
\end{todo}

\begin{todo}
Subspaces of $\mathbb{R}^{2}$ and $\mathbb{R}^{3}$ in geometric terms.
\end{todo}

[...]

\subsection{\label{sect.spaces.more-on-subs}%
%TCIMACRO{\TEXTsymbol{<}}%
%BeginExpansion
$<$%
%EndExpansion
DRAFT%
%TCIMACRO{\TEXTsymbol{>} }%
%BeginExpansion
$>$
%EndExpansion
More on subspaces}

If $U$ is a subspace of a vector space $V$, then the sum of any two elements
of $U$ must belong to $U$ (by the definition of a subspace). The same holds
for sums of any (finite) number of elements of $U$:

\begin{proposition}
\label{prop.subspace.closed}Let $V$ be a vector space. Let $U$ be a subspace
of $V$.

\textbf{(a)} If $u_{1},u_{2},\ldots,u_{k}$ are elements of $U$, then
$u_{1}+u_{2}+\cdots+u_{k}\in U$.

\textbf{(b)} If $u_{1},u_{2},\ldots,u_{k}$ are elements of $U$, then every
linear combination of $u_{1},u_{2},\ldots,u_{k}$ also lies in $U$.

\textbf{(c)} Let $u_{1},u_{2},\ldots,u_{k}$ be elements of $U$. Then,
\[
\operatorname{span}\left(  u_{1},u_{2},\ldots,u_{k}\right)  \subseteq U.
\]

\end{proposition}

The claim of Proposition \ref{prop.subspace.closed} \textbf{(b)} is often
expressed as the following short slogan: \textquotedblleft A subspace $U$ of a
vector space $V$ is closed under linear combination\textquotedblright.

\begin{proof}
[Proof of Proposition \ref{prop.subspace.closed}.]\textbf{(a)} Roughly
speaking, this is just a matter of applying the \textquotedblleft closed under
addition\textquotedblright\ axiom several times. But there is a subtlety
involved (the sum of $0$ elements of $U$ is not obtained by addition, but
rather defined as $\overrightarrow{0}$), and I want to illustrate the
principle of mathematical induction once again, so I am going to present the
proof in full detail.

Let $u_{1},u_{2},\ldots,u_{k}$ be elements of $U$. We must show that
$u_{1}+u_{2}+\cdots+u_{k}\in U$.

We shall prove that%
\begin{equation}
u_{1}+u_{2}+\cdots+u_{i}\in U\ \ \ \ \ \ \ \ \ \ \text{for every }i\in\left\{
0,1,\ldots,k\right\}  . \label{pf.prop.subspace.closed.a.1}%
\end{equation}


We will prove (\ref{pf.prop.subspace.closed.a.1}) by induction over $i$. (See
the proof of Proposition \ref{prop.sum.matrices.multi} given above for a brief
explanation of what this principle means).

\begin{enumerate}
\item \textit{Induction base:} For $i=0$, the statement
(\ref{pf.prop.subspace.closed.a.1}) claims that $u_{1}+u_{2}+\cdots+u_{0}\in
U$. In order to make sense of this, we must recall that empty sums of vectors
are defined to mean $\overrightarrow{0}$. Thus,
\[
u_{1}+u_{2}+\cdots+u_{0}=\left(  \text{empty sum of vectors}\right)
=\overrightarrow{0}.
\]
But $U$ is a subspace of $V$, and thus contains $\overrightarrow{0}$ (this is
one of the axioms for a subspace). Thus, $\overrightarrow{0}\in U$, so that
$u_{1}+u_{2}+\cdots+u_{0}=\overrightarrow{0}\in U$. In other words,
(\ref{pf.prop.subspace.closed.a.1}) holds for $i=0$. This completes the
induction base.

\item \textit{Induction step:} Let $j\in\left\{  0,1,\ldots,k-1\right\}  $ be
such that (\ref{pf.prop.subspace.closed.a.1}) holds for $i=j$. (The statement
that (\ref{pf.prop.subspace.closed.a.1}) holds for $i=j$ is called the
\textquotedblleft induction hypothesis\textquotedblright.) We must show that
(\ref{pf.prop.subspace.closed.a.1}) also holds for $i=j+1$.

Since (\ref{pf.prop.subspace.closed.a.1}) holds for $i=j$, we have
$u_{1}+u_{2}+\cdots+u_{j}\in U$. Now,%
\[
u_{1}+u_{2}+\cdots+u_{j+1}=\underbrace{\left(  u_{1}+u_{2}+\cdots
+u_{j}\right)  }_{\in U}+\underbrace{u_{j+1}}_{\in U}.
\]
This is a sum of two vectors in $U$, and thus belongs to $U$ (since $U$ is
closed under addition). In other words, $u_{1}+u_{2}+\cdots+u_{j+1}\in U$.
Thus, (\ref{pf.prop.subspace.closed.a.1}) also holds for $i=j+1$. This
completes the induction step.
\end{enumerate}

Now, the proof of (\ref{pf.prop.subspace.closed.a.1}) is complete (since both
the induction base and the induction step are complete).

Now that (\ref{pf.prop.subspace.closed.a.1}) is proven, we can simply apply
(\ref{pf.prop.subspace.closed.a.1}) to $i=k$, and conclude that $u_{1}%
+u_{2}+\cdots+u_{k}\in U$. This proves Proposition \ref{prop.subspace.closed}
\textbf{(a)}.

\textbf{(b)} Let $u_{1},u_{2},\ldots,u_{k}$ be elements of $U$. Let
$\lambda_{1},\lambda_{2},\ldots,\lambda_{k}$ be real numbers.

The set $U$ is a subspace of $V$, and thus is closed under scaling. Hence, the
vectors $\lambda_{1}u_{1},\lambda_{2}u_{2},\ldots,\lambda_{k}u_{k}$ all belong
to $U$ (since the vectors $u_{1},u_{2},\ldots,u_{k}$ belong to $U$). Thus,
Proposition \ref{prop.subspace.closed} \textbf{(a)} (applied to $\lambda
_{1}u_{1},\lambda_{2}u_{2},\ldots,\lambda_{k}u_{k}$ instead of $u_{1}%
,u_{2},\ldots,u_{k}$) shows that $\lambda_{1}u_{1}+\lambda_{2}u_{2}%
+\cdots+\lambda_{k}u_{k}\in U$.

Now, we have shown that $\lambda_{1}u_{1}+\lambda_{2}u_{2}+\cdots+\lambda
_{k}u_{k}\in U$ whenever $\lambda_{1},\lambda_{2},\ldots,\lambda_{k}$ are real
numbers. In other words: every linear combination of $u_{1},u_{2},\ldots
,u_{k}$ also lies in $U$. This proves Proposition \ref{prop.subspace.closed}
\textbf{(b)}.

\textbf{(c)} Proposition \ref{prop.subspace.closed} \textbf{(b)} shows that
every linear combination of $u_{1},u_{2},\ldots,u_{k}$ lies in $U$. In other
words, the set of all linear combinations of $u_{1},u_{2},\ldots,u_{k}$ is a
subset of $U$. But since $\operatorname{span}\left(  u_{1},u_{2},\ldots
,u_{k}\right)  $ is precisely the set of all linear combinations of
$u_{1},u_{2},\ldots,u_{k}$ (in fact, this is how $\operatorname{span}\left(
u_{1},u_{2},\ldots,u_{k}\right)  $ was defined), this rewrites as follows:
$\operatorname{span}\left(  u_{1},u_{2},\ldots,u_{k}\right)  $ is a subset of
$U$. This proves Proposition \ref{prop.subspace.closed} \textbf{(c)}.
\end{proof}

I owe you a proof of Proposition \ref{prop.span.uniprop}; it can now be done
quite easily:

\begin{proof}
[Proof of Proposition \ref{prop.span.uniprop}.]\textbf{(a)} Let $i\in\left\{
1,2,\ldots,k\right\}  $. We claim that $v_{i}\in\operatorname{span}\left(
v_{1},v_{2},\ldots,v_{k}\right)  $.

[\textit{Proof:} We have%
\begin{align*}
&  \underbrace{0v_{1}}_{=\overrightarrow{0}}+\underbrace{0v_{2}}%
_{=\overrightarrow{0}}+\cdots+\underbrace{0v_{i-1}}_{=\overrightarrow{0}%
}+\underbrace{1v_{i}}_{=v_{i}}+\underbrace{0v_{i+1}}_{=\overrightarrow{0}%
}+\underbrace{0v_{i+2}}_{=\overrightarrow{0}}+\cdots+\underbrace{0v_{k}%
}_{=\overrightarrow{0}}\\
&  =\overrightarrow{0}+\overrightarrow{0}+\cdots+\overrightarrow{0}%
+v_{i}+\overrightarrow{0}+\overrightarrow{0}+\cdots+\overrightarrow{0}=v_{i},
\end{align*}
so that $v_{i}=0v_{1}+0v_{2}+\cdots+0v_{i-1}+1v_{i}+0v_{i+1}+0v_{i+2}%
+\cdots+0v_{k}$. Hence, the vector $v_{i}$ has the form $\lambda_{1}%
v_{1}+\lambda_{2}v_{2}+\cdots+\lambda_{k}v_{k}$ for some $\lambda_{1}%
,\lambda_{2},\ldots,\lambda_{k}\in\mathbb{R}$ (namely, for $\lambda
_{j}=\left\{
\begin{array}
[c]{c}%
1\text{, if }j=i\text{;}\\
0\text{, if }j\neq i
\end{array}
\right.  $). In other words,
\[
v_{i}\in\left\{  \lambda_{1}v_{1}+\lambda_{2}v_{2}+\cdots+\lambda_{k}%
v_{k}\ \mid\ \lambda_{1},\lambda_{2},\ldots,\lambda_{k}\in\mathbb{R}\right\}
.
\]
In view of (\ref{eq.def.subspace.span.explicit}), this rewrites as $v_{i}%
\in\operatorname{span}\left(  v_{1},v_{2},\ldots,v_{k}\right)  $%
.\ \ \ \ \footnote{Here is a more rigorous version of the same proof:
\par
Proposition \ref{prop.sum.vectors.delta} (applied to $p=1$, $q=k$ and $r=i$)
yields%
\[
\sum_{g=1}^{k}\delta_{g,i}v_{g}=v_{i}.
\]
(Here, we are using the letter $g$ instead of $k$ for the summation index, in
order to avoid a clash with the already existing meaning of the letter $k$.)
Thus,%
\[
v_{i}=\sum_{g=1}^{k}\delta_{g,i}v_{g}=\delta_{1,i}v_{1}+\delta_{2,i}%
v_{2}+\cdots+\delta_{k,i}v_{k}.
\]
Hence, the vector $v_{i}$ has the form $\lambda_{1}v_{1}+\lambda_{2}%
v_{2}+\cdots+\lambda_{k}v_{k}$ for some $\lambda_{1},\lambda_{2}%
,\ldots,\lambda_{k}\in\mathbb{R}$ (namely, for $\lambda_{j}=\delta_{j,i}$). In
other words,
\[
v_{i}\in\left\{  \lambda_{1}v_{1}+\lambda_{2}v_{2}+\cdots+\lambda_{k}%
v_{k}\ \mid\ \lambda_{1},\lambda_{2},\ldots,\lambda_{k}\in\mathbb{R}\right\}
.
\]
In view of (\ref{eq.def.subspace.span.explicit}), this rewrites as $v_{i}%
\in\operatorname{span}\left(  v_{1},v_{2},\ldots,v_{k}\right)  $.}]

Now, forget that we fixed $i$. We thus have shown that $v_{i}\in
\operatorname{span}\left(  v_{1},v_{2},\ldots,v_{k}\right)  $ for each
$i\in\left\{  1,2,\ldots,k\right\}  $. In other words, the vectors
$v_{1},v_{2},\ldots,v_{k}$ all belong to $\operatorname{span}\left(
v_{1},v_{2},\ldots,v_{k}\right)  $. This proves Proposition
\ref{prop.span.uniprop} \textbf{(a)}.

\textbf{(b)} Let $U$ be a subspace of $V$ that contains the vectors
$v_{1},v_{2},\ldots,v_{k}$. We must prove that $\operatorname{span}\left(
v_{1},v_{2},\ldots,v_{k}\right)  \subseteq U$. But this follows immediately
from Proposition \ref{prop.subspace.closed} \textbf{(c)} (applied to
$u_{i}=v_{i}$). Thus, Proposition \ref{prop.span.uniprop} \textbf{(b)} is proven.
\end{proof}

[...]

[...]

\subsection{%
%TCIMACRO{\TEXTsymbol{<}}%
%BeginExpansion
$<$%
%EndExpansion
TODO%
%TCIMACRO{\TEXTsymbol{>} }%
%BeginExpansion
$>$
%EndExpansion
More on spans}

\textbf{References for vector spaces:} \cite[\S 5.1]{LaNaSc16} (possibly the
best one), \cite[\S 2.3]{OlvSha06} (focusses on the analysis-related and
applied stuff) and \cite[Two.I.2]{Heffer16}.

\begin{todo}
The terminology \textquotedblleft$\left(  v_{1},v_{2},\ldots,v_{k}\right)  $
spans $V$\textquotedblright\ (or \textquotedblleft$v_{1},v_{2},\ldots,v_{k}$
span $V$\textquotedblright) means $V=\operatorname{span}\left(  v_{1}%
,v_{2},\ldots,v_{k}\right)  $.
\end{todo}

\begin{todo}
$E_{i,j}$ span $\mathbb{R}^{n\times m}$. (Use Proposition \ref{prop.Euv.A=sum}.)
\end{todo}

\begin{todo}
$e_{i} = E_{i,1}$ span $\mathbb{R}^{n}$.
\end{todo}

\begin{todo}
How to check that a given vector is in a given span:

$w\in\operatorname{span}\left(  v_{1},v_{2},\ldots,v_{k}\right)  $

$\Longleftrightarrow$ the equation $w=\lambda_{1}v_{1}+\cdots+\lambda_{k}%
v_{k}$ has at least one solution $\left(  \lambda_{1},\lambda_{2}%
,\ldots,\lambda_{k}\right)  ^{T}\in\mathbb{R}^{k}$.

Can solve this using Gaussian elimination.
\end{todo}

\begin{todo}
How to check that a given span is contained in another given span:

Checking that $\operatorname{span}\left(  \alpha,\beta\right)  \subseteq
\operatorname{span}\left(  \gamma,\delta,\varepsilon\right)  $ is tantamount
to checking that both $\alpha$ and $\beta$ lie in $\operatorname{span}\left(
\gamma,\delta,\varepsilon\right)  $ (by Proposition \ref{prop.subspace.closed}
\textbf{(c)}).
\end{todo}

\begin{todo}
How to check that two given spans are identical?

Check that each is contained in the other.
\end{todo}

\begin{todo}
How to write the subspace carved out by equations (i.e., a kernel) as a span?

Solve the system.

For instance, write $\operatorname*{Ker}\left(
\begin{array}
[c]{ccc}%
1 & 3 & 2\\
3 & 9 & 6\\
2 & 6 & 4
\end{array}
\right)  $ as a span.

Solve the system $\left(
\begin{array}
[c]{ccc}%
1 & 3 & 2\\
3 & 9 & 6\\
2 & 6 & 4
\end{array}
\right)  \left(
\begin{array}
[c]{c}%
x_{1}\\
x_{2}\\
x_{3}%
\end{array}
\right)  =\left(
\begin{array}
[c]{c}%
0\\
0\\
0
\end{array}
\right)  $. Solution: $x=\left(
\begin{array}
[c]{c}%
-3s-2r\\
s\\
r
\end{array}
\right)  $. Rewrite this as $x=r\left(
\begin{array}
[c]{c}%
-2\\
0\\
1
\end{array}
\right)  +s\left(
\begin{array}
[c]{c}%
-3\\
1\\
0
\end{array}
\right)  $ (here I have just moved the free variables $r,s$ outside of the
vector, so that only constant numbers have remained in the vector).

Thus, the elements of $\operatorname*{Ker}\left(
\begin{array}
[c]{ccc}%
1 & 3 & 2\\
3 & 9 & 6\\
2 & 6 & 4
\end{array}
\right)  $ are precisely the vectors of the form $x=r\left(
\begin{array}
[c]{c}%
-2\\
0\\
1
\end{array}
\right)  +s\left(
\begin{array}
[c]{c}%
-3\\
1\\
0
\end{array}
\right)  $. In other words, they are the linear combinations of $\left(
\begin{array}
[c]{c}%
-2\\
0\\
1
\end{array}
\right)  ,\left(
\begin{array}
[c]{c}%
-3\\
1\\
0
\end{array}
\right)  $. In other words, $\operatorname*{Ker}\left(
\begin{array}
[c]{ccc}%
1 & 3 & 2\\
3 & 9 & 6\\
2 & 6 & 4
\end{array}
\right)  =\operatorname{span}\left(  \left(
\begin{array}
[c]{c}%
-2\\
0\\
1
\end{array}
\right)  ,\left(
\begin{array}
[c]{c}%
-3\\
1\\
0
\end{array}
\right)  \right)  $.
\end{todo}

\begin{todo}
Example: $\left(  \left(  1,3,2\right)  ^{T},\left(  0,1,1\right)
^{T}\right)  $ spans $W:=\left\{  \left(  x,y,z\right)  ^{T}\ \mid
\ x+z=y\right\}  \subseteq\mathbb{R}^{3}$.

Prove this in full: Let $\alpha=\left(  1,3,2\right)  ^{T}$ and $\beta=\left(
0,1,1\right)  ^{T}$.

Show $\alpha,\beta\in W$. Conclude that $\operatorname{span}\left(
\alpha,\beta\right)  \subseteq W$ by Proposition \ref{prop.subspace.closed}
\textbf{(c)}.

Now prove converse: Solve the system $\left\{  x+z=y\right.  $ and write the
solution as $\left(  y-z,y,z\right)  ^{T}=y\left(  1,1,0\right)  ^{T}+z\left(
-1,0,1\right)  ^{T}$. So $W=\operatorname{span}\left(  \left(  1,1,0\right)
^{T},\left(  -1,0,1\right)  ^{T}\right)  $. Now, need to check that two spans
are equal. We know how this is done.
\end{todo}

\begin{todo}
Example: $\left(  \left(  1,-1,1\right)  ^{T}\right)  $ spans $X:=\left\{
\left(  x,y,z\right)  ^{T}\ \mid\ x+3y+z=0\text{ and }y+z=0\right\}  $.

Prove this too.
\end{todo}

\begin{todo}
Exercise: what spans $\left\{  \left(  u-v,v-u\right)  ^{T}\right\}
\subseteq\mathbb{R}^{2}$ ? Is there an easier description?
\end{todo}

\begin{todo}
Exercise: what spans $\left\{  \left(  u,v,0\right)  ^{T}\right\}  $ ?

$\left\{  \left(  u,0,0\right)  ^{T}\right\}  $ ?

$\left\{  \left(  u,v,u\right)  ^{T}\right\}  $ ?

$\left\{  \left(  x_{1},x_{2},x_{3}\right)  ^{T}\ \mid\ x_{1}^{2}+x_{2}%
^{2}=0\right\}  $ ?
\end{todo}

\begin{todo}
Redundant vectors in spans can be removed.
\end{todo}

\begin{todo}
hw3: If $\left(  w_{1}, w_{2}, \ldots, w_{k}\right)  $ and $\left(  v_{1},
v_{2}, \ldots, v_{\ell}\right)  $ are two lists of vectors such that $\left\{
w_{1}, w_{2}, \ldots, w_{k}\right\}  = \left\{  v_{1}, v_{2}, \ldots, v_{\ell
}\right\}  $, then $\operatorname{span}\left(  w_{1}, w_{2}, \ldots,
w_{k}\right)  = \operatorname{span}\left(  v_{1}, v_{2}, \ldots, v_{\ell
}\right)  $. In particular, if we reorder or duplicate the elements of a list,
then its span does not change.
\end{todo}

\begin{todo}
Rest of hw3.
\end{todo}

\begin{todo}
The list $\left(  1, x, x^{2}, \ldots, x^{n}\right)  $ spans the vector space
$\mathcal{P}_{n}$ of all polynomial functions $\mathbb{R} \to\mathbb{R}$ of
degree $\leq n$. (This allows the constant-$0$ function, which is understood
to have degree $-\infty$.)
\end{todo}

\begin{todo}
hw3: The subspace $\left\{  \left(  x_{1},x_{2},\ldots,x_{n}\right)  ^{T}%
\in\mathbb{R}^{n}\mid x_{1}+x_{2}+\cdots+x_{n}=0\right\}  $ of $\mathbb{R}%
^{n}$ is spanned by each of the two lists $\left(  e_{1}-e_{n},e_{2}%
-e_{n},\ldots,e_{n-1}-e_{n}\right)  $ and $\left(  e_{1}-e_{2},e_{2}%
-e_{3},\ldots,e_{n-1}-e_{n}\right)  $.
\end{todo}

\begin{definition}
A vector space $V$ is said to be \textit{finite-dimensional} if there is a
(finite) list $\left(  v_{1},v_{2},\ldots,v_{k}\right)  $ spanning $V$.
\end{definition}

\begin{todo}
Examples: $\mathbb{R}^{n}$, $\mathbb{R}^{n\times m}$, $\mathbb{R}$,
$\mathbb{C}$ and all their subspaces are finite-dimensional.

On the other hand, $\mathbb{R}^{\mathbb{R}}$, the set of all polynomial
functions, etc. are not finite-dimensional.
\end{todo}

[...]

\subsection{%
%TCIMACRO{\TEXTsymbol{<}}%
%BeginExpansion
$<$%
%EndExpansion
TODO%
%TCIMACRO{\TEXTsymbol{>} }%
%BeginExpansion
$>$
%EndExpansion
Linear independence}

\textbf{References for vector spaces:} \cite[\S 5.2]{LaNaSc16} (possibly the
best one), \cite[\S 2.3]{OlvSha06} (focusses on the analysis-related and
applied stuff) and \cite[Two.II]{Heffer16}.

\begin{definition}
\label{def.linind}Let $v_{1},v_{2},\ldots,v_{k}$ be vectors in a vector space
$V$. We say that the list $\left(  v_{1},v_{2},\ldots,v_{k}\right)  $ is
\textit{linearly independent} if the only $k$-tuple $\left(  \lambda
_{1},\lambda_{2},\ldots,\lambda_{k}\right)  $ of real numbers satisfying
$\lambda_{1}v_{1}+\lambda_{2}v_{2}+\cdots+\lambda_{k}v_{k}=\overrightarrow{0}$
is $\left(  0,0,\ldots,0\right)  $.

Instead of saying \textquotedblleft the list $\left(  v_{1},v_{2},\ldots
,v_{k}\right)  $ is linearly independent\textquotedblright, we can also say
\textquotedblleft the vectors $v_{1},v_{2},\ldots,v_{k}$ are linearly
independent\textquotedblright. (But keep in mind that linear independence is a
property of the whole list $\left(  v_{1},v_{2},\ldots,v_{k}\right)  $, not of
each single vector in this list. It often happens (for example) that two
vectors $a,b$ are linearly independent, and two further vectors $c,d$ are
linearly independent, but the four vectors $a,b,c,d$ together are not linearly independent.)
\end{definition}

\begin{todo}
Equivalent restatements of linear independence:

\begin{enumerate}
\item The list $\left(  v_{1},v_{2},\ldots,v_{k}\right)  $ is linearly independent.

\item The vectors $v_{1},v_{2},\ldots,v_{k}$ are linearly independent. (This
is just another way of saying things.)

\item The only $k$-tuple $\left(  \lambda_{1},\lambda_{2},\ldots,\lambda
_{k}\right)  $ of real numbers satisfying $\lambda_{1}v_{1}+\lambda_{2}%
v_{2}+\cdots+\lambda_{k}v_{k}=\overrightarrow{0}$ is $\left(  0,0,\ldots
,0\right)  $.

\item If $\lambda_{1},\lambda_{2},\ldots,\lambda_{k}$ are real numbers
satisfying $\lambda_{1}v_{1}+\lambda_{2}v_{2}+\cdots+\lambda_{k}%
v_{k}=\overrightarrow{0}$, then $\lambda_{1}=\lambda_{2}=\cdots=\lambda_{k}=0$.

\item If $\lambda_{1},\lambda_{2},\ldots,\lambda_{k}$ are real numbers,
\textbf{not all zero}, then $\lambda_{1}v_{1}+\lambda_{2}v_{2}+\cdots
+\lambda_{k}v_{k}\neq\overrightarrow{0}$.

\item For each vector $v\in V$, there exists \textbf{at most one} $k$-tuple
$\left(  \lambda_{1},\lambda_{2},\ldots,\lambda_{k}\right)  $ of real numbers
satisfying $\lambda_{1}v_{1}+\lambda_{2}v_{2}+\cdots+\lambda_{k}v_{k}=v$.

\item Each $i\in\left\{  1,2,\ldots,k\right\}  $ satisfies $v_{i}%
\notin\operatorname*{span}\left(  v_{1},v_{2},\ldots,v_{i-1}\right)  $. (See
Proposition below.)

\item Each $i\in\left\{  1,2,\ldots,k\right\}  $ satisfies $v_{i}%
\notin\operatorname*{span}\left(  v_{1},v_{2},\ldots,v_{i-1},v_{i+1}%
,v_{i+2},\ldots,v_{k}\right)  $. (See Proposition below.)
\end{enumerate}
\end{todo}

\begin{todo}
Example: The standard basis vectors $e_{1},e_{2},\ldots,e_{n}$ of
$\mathbb{R}^{n}$ are linearly independent.
\end{todo}

\begin{todo}
Non-example: The vectors $e_{1}-e_{2},e_{2}-e_{3},e_{3}-e_{1}$ in
$\mathbb{R}^{3}$ are \textbf{not} linearly independent. In fact,%
\[
1\left(  e_{1}-e_{2}\right)  +1\left(  e_{2}-e_{3}\right)  +1\left(
e_{3}-e_{1}\right)  =\overrightarrow{0}.
\]

\end{todo}

\begin{todo}
We say \textquotedblleft linearly dependent\textquotedblright\ for
\textquotedblleft not linearly independent\textquotedblright.

Another non-example: The vectors $e_{1},\overrightarrow{0},e_{2}$ in
$\mathbb{R}^{3}$ are linearly dependent. In fact,%
\[
0e_{1}+1\cdot\overrightarrow{0}+0e_{2}=\overrightarrow{0}.
\]
More generally, if $\overrightarrow{0}$ appears in a list of vectors, then
said list must be linearly dependent.
\end{todo}

\begin{todo}
Another non-example: If $\alpha$ and $\beta$ are two vectors, then $\left(
\alpha,\beta,\alpha\right)  $ is linearly dependent. Indeed,%
\[
1\alpha+0\beta+\left(  -1\right)  \alpha=\overrightarrow{0}.
\]
More generally, if a vector in a list appears more than once, then said list
is linearly dependent.
\end{todo}

\begin{todo}
How to check whether a list of vectors in $\mathbb{R}^{n}$ (or $\mathbb{R}%
^{n\times m}$) is linearly independent?

You have to check whether the only solution of $\lambda_{1}v_{1}+\lambda
_{2}v_{2}+\cdots+\lambda_{k}v_{k}=\overrightarrow{0}$ is $\left(
0,0,\ldots,0\right)  $. This can be done by Gaussian elimination.

For example, is the list $\left(  \left(
\begin{array}
[c]{c}%
2\\
1\\
3
\end{array}
\right)  ,\left(
\begin{array}
[c]{c}%
1\\
3\\
6
\end{array}
\right)  ,\left(
\begin{array}
[c]{c}%
2\\
5\\
5
\end{array}
\right)  \right)  $ of vectors in $\mathbb{R}^{3}$ linearly independent?

So we are asking for the solutions of $\lambda_{1}\left(
\begin{array}
[c]{c}%
2\\
1\\
3
\end{array}
\right)  +\lambda_{2}\left(
\begin{array}
[c]{c}%
1\\
3\\
6
\end{array}
\right)  +\lambda_{3}\left(
\begin{array}
[c]{c}%
2\\
5\\
5
\end{array}
\right)  =\overrightarrow{0}$. This rewrites as the system $\left\{
\begin{array}
[c]{c}%
2\lambda_{1}+1\lambda_{2}+2\lambda_{3}=0;\\
1\lambda_{1}+3\lambda_{2}+5\lambda_{3}=0;\\
3\lambda_{1}+6\lambda_{2}+5\lambda_{3}=0
\end{array}
\right.  $. The only solution is $\left(  \lambda_{1},\lambda_{2},\lambda
_{3}\right)  =\left(  0,0,0\right)  $. So, yes, it is linearly independent.
\end{todo}

\begin{todo}
Visual meaning of linear independence in $\mathbb{R}^{2}$ and $\mathbb{R}^{3}$.
\end{todo}

\begin{todo}
If $\left(  v_{1},v_{2},\ldots,v_{k}\right)  $ is linearly independent, then
so is every rearrangement of $\left(  v_{1},v_{2},\ldots,v_{k}\right)  $, and
every sublist of $\left(  v_{1},v_{2},\ldots,v_{k}\right)  $ too.
\end{todo}

\begin{todo}
\textbf{Proposition:} Let $V$ be a vector space. Let $v_{1},v_{2},\ldots
,v_{k}$ be $k$ vectors in $V$. Assume that the list $\left(  v_{1}%
,v_{2},\ldots,v_{k}\right)  $ is linearly dependent. Then:

\textbf{(a)} There exists some $i\in\left\{  1,2,\ldots,k\right\}  $ such that
$v_{i}\in\operatorname*{span}\left(  v_{1},v_{2},\ldots,v_{i-1}\right)  $.
[Notice that when $i=1$, the span $\operatorname*{span}\left(  v_{1}%
,v_{2},\ldots,v_{i-1}\right)  $ has to be interpreted as the \textquotedblleft
empty span\textquotedblright\ $\operatorname*{span}\left(  {}\right)
=\left\{  \overrightarrow{0}\right\}  $.]

\textbf{(b)} This $i$ satisfies $\operatorname*{span}\left(  v_{1}%
,v_{2},\ldots,v_{k}\right)  =\operatorname*{span}\left(  v_{1},v_{2}%
,\ldots,v_{i-1},v_{i+1},\ldots,v_{k}\right)  $.
\end{todo}

\begin{todo}
\textbf{Corollary:} Let $V$ be a vector space. Let $v_{1},v_{2},\ldots,v_{k}$
be $k$ vectors in $V$. Then, the following are equivalent:

\textbf{(a)} The list $\left(  v_{1},v_{2},\ldots,v_{k}\right)  $ is linearly independent.

\textbf{(b)} No $i$ satisfies $v_{i}\in\operatorname*{span}\left(  v_{1}%
,v_{2},\ldots,v_{i-1}\right)  $.

\textbf{(c)} No $i$ satisfies $v_{i}\in\operatorname*{span}\left(  v_{1}%
,v_{2},\ldots,v_{i-1},v_{i+1},\ldots,v_{k}\right)  $.
\end{todo}

\begin{todo}
Thus, if a list spans a subspace $U$ of $V$, then we can keep shrinking this
list (by throwing out redundant elements) until we obtain a linearly
independent list that also spans $U$. For example, the list $\left(
e_{1}-e_{2},e_{1},e_{2}-2e_{1},e_{1}+e_{3},e_{2}+e_{3}\right)  $ spanning
$\mathbb{R}^{3}$ can be shrunk by first throwing out $e_{2}-2e_{1}$ (which is
$\in\operatorname*{span}\left(  e_{1}-e_{2},e_{1}\right)  $) and then throwing
out $e_{2}+e_{3}$ (which is $\in\operatorname*{span}\left(  e_{1}-e_{2}%
,e_{1},e_{1}+e_{3}\right)  $); the result is a linearly independent list
$\left(  e_{1}-e_{2},e_{1},e_{1}+e_{3}\right)  $ that still spans
$\mathbb{R}^{3}$.

How do we find redundant entries in our list? One way to get them is just to
check, for each $i$, whether $v_{i}\in\operatorname*{span}\left(  v_{1}%
,v_{2},\ldots,v_{i-1}\right)  $. Another (faster) way is to pick a
not-all-zeroes solution $\left(  \lambda_{1},\lambda_{2},\ldots,\lambda
_{k}\right)  $ of the equation $\lambda_{1}v_{1}+\lambda_{2}v_{2}%
+\cdots+\lambda_{k}v_{k}=\overrightarrow{0}$, and take the highest $i$ for
which $\lambda_{i}\neq0$; then, the corresponding $v_{i}$ is redundant.

Exercise: Check that the list obtained at the end of this process is indeed
linearly independent.
\end{todo}

\begin{todo}
The \textbf{Steinitz exchange lemma} is the claim that if $\left(  v_{1}%
,v_{2},\ldots,v_{k}\right)  $ is a list of vectors in a vector space $V$ such
that $V=\operatorname*{span}\left(  v_{1},v_{2},\ldots,v_{k}\right)  $, and if
$\left(  w_{1},w_{2},\ldots,w_{l}\right)  $ is a linearly independent list of
vectors in the same vector space $V$, then

\textbf{(a)} we have $k\geq\ell$ (that is, the spanning list is at least as
long as the linearly independent list), and

\textbf{(b)} we can obtain a basis of $V$ by filling up the list $\left(
w_{1},w_{2},\ldots,w_{\ell}\right)  $ with some vectors from $\left(
v_{1},v_{2},\ldots,v_{k}\right)  $. (\textquotedblleft Filling
up\textquotedblright\ means that we pick some vectors from the list $\left(
v_{1},v_{2},\ldots,v_{k}\right)  $ and append them to the list $\left(
w_{1},w_{2},\ldots,w_{\ell}\right)  $. For example, filling up the list
$\left(  w_{1},w_{2},\ldots,w_{\ell}\right)  $ with some vectors from $\left(
v_{1},v_{2},\ldots,v_{k}\right)  $ can result in the list $\left(  w_{1}%
,w_{2},\ldots,w_{\ell},v_{2},v_{4},v_{9}\right)  $. But \textquotedblleft
some\textquotedblright\ can also be none; thus, we might end up with $\left(
w_{1},w_{2},\ldots,w_{\ell}\right)  $.)

The Wikipedia has a proof of the Steinitz exchange lemma, and it appears to be
among the most readable:
\url{https://en.wikipedia.org/wiki/Steinitz_exchange_lemma#Proof} . (Note that
the Wikipedia talks of sets instead of lists, but you can just read "list" for
"set" there.)

Another source is Theorem 5.2.9 in L/N/S. Notice that L/N/S are trying to
avoid talking about empty lists; this forces them to take some slightly clumsy
precautions (for example, the "v\_1 is nonzero" condition in Lemma 5.2.7 is
just there because they don't want to allow j to be 1). If you feel at home
with empty lists, you can simplify their proof of Theorem 5.2.9 as well (Step
1 becomes just a particular case of Step k for k = 1). That said, I still
prefer the proof on Wikipedia.

Another proof is in Olver/Shakiban, using Gaussian elimination.
\end{todo}

[...]

\subsection{%
%TCIMACRO{\TEXTsymbol{<}}%
%BeginExpansion
$<$%
%EndExpansion
TODO%
%TCIMACRO{\TEXTsymbol{>} }%
%BeginExpansion
$>$
%EndExpansion
Bases and dimension}

\textbf{References for vector spaces:} \cite[\S 5.3-5.4]{LaNaSc16} (possibly
the best one), \cite[\S 2.4]{OlvSha06} (focusses on the analysis-related and
applied stuff) and \cite[Two.III]{Heffer16}.

\begin{definition}
Let $v_{1},v_{2},\ldots,v_{k}$ be vectors in a vector space $V$. We say that
the list $\left(  v_{1},v_{2},\ldots,v_{k}\right)  $ is a \textit{basis} of
$V$ if and only if this list spans $V$ (that is, it satisfies
$V=\operatorname*{span}\left(  v_{1},v_{2},\ldots,v_{k}\right)  $) and is
linearly independent.
\end{definition}

\begin{theorem}
Let $V$ be a finite-dimensional vector space. Then, any two bases of $V$ have
the same size.
\end{theorem}

\begin{todo}
Prove this theorem using the Steinitz exchange lemma.
\end{todo}

\begin{todo}
If $\left(  v_{1},v_{2},\ldots,v_{k}\right)  $ is a linearly independent list
in a vector space $V$, then $\left(  v_{1},v_{2},\ldots,v_{k}\right)  $ is a
basis of the subspace $\operatorname*{span}\left(  v_{1},v_{2},\ldots
,v_{k}\right)  $.
\end{todo}

\begin{todo}
Each spanning list can be shrunk to a basis. (This was done above, just
without saying the word \textquotedblleft basis\textquotedblright.)
\end{todo}

\begin{todo}
........
\end{todo}

[...]

[to be continued]

\begin{thebibliography}{9999999999}                                                                                       %


\bibitem[Artin10]{Artin10}Michael Artin, \textit{Algebra}, 2nd edition,
Pearson 2010.

\bibitem[BarSch73]{BarSch73}Hans Schneider, George Phillip Barker,
\textit{Matrices and Linear Algebra}, 2nd edition, Dover 1973.

\bibitem[Camero08]{Camero08}Peter J. Cameron, \textit{Notes on Linear
Algebra}, version 5 Sep 2008.\newline\url{http://www.maths.qmul.ac.uk/~pjc/notes/linalg.pdf}

\bibitem[Chen08]{Chen08}William Chen, \textit{Linear Algebra}, 2011.\newline\url{https://rutherglen.science.mq.edu.au/wchen/lnlafolder/lnla.html}

\bibitem[deBoor]{deBoor}Carl de Boor, \textit{An empty exercise}.
\url{ftp://ftp.cs.wisc.edu/Approx/empty.pdf} .

\bibitem[Drucker12]{Drucker12}D. Drucker, \textit{Annotated Bibliography of
Linear Algebra Books}, February 2012.\newline\url{http://www.math.wayne.edu/~drucker/linalgrefsW12.pdf}

\bibitem[DumFoo04]{DumFoo04}David S. Dummit, Richard M. Foote,
\textit{Abstract Algebra}, 3rd edition 2004.

\bibitem[Goodma15]{Goodma15}Frederick M. Goodman, \textit{Algebra: Abstract
and Concrete}, edition 2.6.\newline\url{http://homepage.divms.uiowa.edu/~goodman/algebrabook.dir/download.htm}

\bibitem[Grcar10]{Grcar10}Joseph F. Grcar, \textit{How ordinary elimination
became Gaussian elimination}, Historia Mathematica, Volume 38, Issue 2, May
2011, pp. 163--218.\newline\url{http://www.sciencedirect.com/science/article/pii/S0315086010000376}

\bibitem[Gill12]{Gill}%
\href{http://cseweb.ucsd.edu/~gill/CILASite/Resources/LinearAlgebra.pdf}{Stanley
Gill Williamson, \textit{Matrix Canonical Forms: notational skills and proof
techniques}, 15 July 2015.}

\bibitem[Grinbe16]{detnotes}Darij Grinberg, \textit{Notes on the combinatorial
fundamentals of algebra}, 10 January 2019.\newline\url{http://www.cip.ifi.lmu.de/~grinberg/primes2015/sols.pdf}

\bibitem[OlvSha06]{OlvSha06}Peter J. Olver, Chehrzad Shakiban, \textit{Applied
Linear Algebra}, Prentice Hall, 2006.\newline See also
\url{http://www.math.umn.edu/~olver/ala.html} for corrections.

\bibitem[Heffer16]{Heffer16}Jim Hefferon, \textit{Linear Algebra},
2016.\newline\url{http://joshua.smcvt.edu/linearalgebra/} (Scroll down to
\textquotedblleft The next edition\textquotedblright\ to download the newest version.)

\bibitem[Kowals16]{Kowals16}Emmanuel Kowalski, \textit{Linear Algebra},
version 15 Sep 2016.\newline\url{https://people.math.ethz.ch/~kowalski/script-la.pdf}

\bibitem[LaNaSc16]{LaNaSc16}Isaiah Lankham, Bruno Nachtergaele, Anne
Schilling, \textit{Linear Algebra As an Introduction to Abstract Mathematics},
2016.\newline\url{https://www.math.ucdavis.edu/~anne/linear_algebra/mat67_course_notes.pdf}

\bibitem[LeLeMe16]{LeLeMe16}Eric Lehman, F. Thomson Leighton, Albert R. Meyer,
\textit{Mathematics for Computer Science}, revised Wednesday 6th June, 2018,
\newline\url{https://courses.csail.mit.edu/6.042/spring18/mcs.pdf} .

\bibitem[m.se709196]{m.se709196}Daniela Diaz and others, \textit{Definition of
General Associativity for binary operations}, math.stackexchange question
\#709196 .\newline\url{http://math.stackexchange.com/q/709196}

\bibitem[Stanle12]{Stanley-EC1}Richard P. Stanley, \textit{Enumerative
Combinatorics, Volume 1}, 2nd edition, CUP 2012.\newline See
\url{http://math.mit.edu/~rstan/ec/ec1/} for a preliminary version.

\bibitem[Treil15]{Treil15}Sergei Treil, \textit{Linear Algebra Done Wrong},
2017.\newline\url{https://www.math.brown.edu/~treil/papers/LADW/LADW.html}

\bibitem[Wildon16]{Wildon16}Mark Wildon, \textit{MT182 Matrix Algebra},
version 14 May 2016.\newline\url{http://www.ma.rhul.ac.uk/~uvah099/Maths/MatrixAlgebra15/MT1822015Notes.pdf}

\bibitem[Zuker14]{Zuker14}M. Zuker, \textit{Binary Operations \& General
Associativity},\newline\url{http://snark.math.rpi.edu/Teaching/MATH-4010/Binary_Ops.pdf}
\end{thebibliography}


\end{document}