update

bib
2025-10-30 15:47:54 +01:00 · 2025-10-30 14:51:12 +01:00 · 2025-10-30 13:20:34 +01:00 · 2025-10-29 22:31:42 +01:00 · 2025-10-29 17:21:03 +01:00 · 2025-10-28 13:26:00 +01:00
10 changed files with 4 additions and 1185 deletions
@@ -333,4 +333,3 @@ TSWLatexianTemp*
 # End of https://www.toptal.com/developers/gitignore/api/latex,visualstudiocode
 bib
 *.*-SAVE-ERROR
 *.ist
@@ -1,5 +0,0 @@
 $latex = 'latex  %O  --shell-escape %S';
 $pdflatex = 'pdflatex  %O  --shell-escape %S';
 $pdf_mode = 1;
 $clean_ext = "lol nav snm loa bbl* glo ist";
 $bibtex_use = 2;
@@ -1,94 +0,0 @@
@article{shannon1948mathematical,
  title={A mathematical theory of communication},
  author={Shannon, Claude E},
  journal={The Bell system technical journal},
  volume={27},
  number={3},
  pages={379--423},
  year={1948},
  publisher={Nokia Bell Labs}
 }
@misc{ enwiki:shannon-source-coding,
  author = "{Wikipedia contributors}",
  title = "Shannon's source coding theorem --- {Wikipedia}{,} The Free Encyclopedia",
  year = "2025",
  url = "https://en.wikipedia.org/w/index.php?title=Shannon%27s_source_coding_theorem&oldid=1301398440",
  note = "[Online; accessed 25-November-2025]"
 }
@misc{ enwiki:shannon-fano,
  author = "{Wikipedia contributors}",
  title = "Shannon–Fano coding --- {Wikipedia}{,} The Free Encyclopedia",
  year = "2025",
  url = "https://en.wikipedia.org/w/index.php?title=Shannon%E2%80%93Fano_coding&oldid=1315776380",
  note = "[Online; accessed 26-November-2025]"
 }
@misc{ enwiki:huffman-code,
  author = "{Wikipedia contributors}",
  title = "Huffman coding --- {Wikipedia}{,} The Free Encyclopedia",
  year = "2025",
  url = "https://en.wikipedia.org/w/index.php?title=Huffman_coding&oldid=1321991625",
  note = "[Online; accessed 26-November-2025]"
 }
@misc{ enwiki:lzw,
  author = "{Wikipedia contributors}",
  title = "Lempel–Ziv–Welch --- {Wikipedia}{,} The Free Encyclopedia",
  year = "2025",
  url = "https://en.wikipedia.org/w/index.php?title=Lempel%E2%80%93Ziv%E2%80%93Welch&oldid=1307959679",
  note = "[Online; accessed 26-November-2025]"
 }
@misc{ enwiki:arithmetic-code,
  author = "{Wikipedia contributors}",
  title = "Arithmetic coding --- {Wikipedia}{,} The Free Encyclopedia",
  year = "2025",
  url = "https://en.wikipedia.org/w/index.php?title=Arithmetic_coding&oldid=1320999535",
  note = "[Online; accessed 26-November-2025]"
 }
@misc{ enwiki:kraft-mcmillan,
  author = "{Wikipedia contributors}",
  title = "Kraft–McMillan inequality --- {Wikipedia}{,} The Free Encyclopedia",
  year = "2025",
  url = "https://en.wikipedia.org/w/index.php?title=Kraft%E2%80%93McMillan_inequality&oldid=1313803157",
  note = "[Online; accessed 26-November-2025]"
 }
@misc{ enwiki:partition,
  author = "{Wikipedia contributors}",
  title = "Partition problem --- {Wikipedia}{,} The Free Encyclopedia",
  year = "2025",
  url = "https://en.wikipedia.org/w/index.php?title=Partition_problem&oldid=1320732818",
  note = "[Online; accessed 30-November-2025]"
 }
@misc{ dewiki:shannon-fano,
 author = "Wikipedia",
 title = "Shannon-Fano-Kodierung --- Wikipedia{,} die freie Enzyklopädie",
 year = "2024",
 url = "https://de.wikipedia.org/w/index.php?title=Shannon-Fano-Kodierung&oldid=246624798",
 note = "[Online; Stand 26. November 2025]"
 }
@misc{ dewiki:huffman-code,
 author = "Wikipedia",
 title = "Huffman-Kodierung --- Wikipedia{,} die freie Enzyklopädie",
 year = "2025",
 url = "https://de.wikipedia.org/w/index.php?title=Huffman-Kodierung&oldid=254369306",
 note = "[Online; Stand 26. November 2025]"
 }
@misc{ dewiki:lzw,
 author = "Wikipedia",
 title = "Lempel-Ziv-Welch-Algorithmus --- Wikipedia{,} die freie Enzyklopädie",
 year = "2025",
 url = "https://de.wikipedia.org/w/index.php?title=Lempel-Ziv-Welch-Algorithmus&oldid=251943809",
 note = "[Online; Stand 26. November 2025]"
 }
@misc{ dewiki:kraft-mcmillan,
 author = "Wikipedia",
 title = "Kraft-Ungleichung --- Wikipedia{,} die freie Enzyklopädie",
 year = "2018",
 url = "https://de.wikipedia.org/w/index.php?title=Kraft-Ungleichung&oldid=172862410",
 note = "[Online; Stand 26. November 2025]"
 }
@misc{ dewiki:partition,
 author = "Wikipedia",
 title = "Partitionsproblem --- Wikipedia{,} die freie Enzyklopädie",
 year = "2025",
 url = "https://de.wikipedia.org/w/index.php?title=Partitionsproblem&oldid=255787013",
 note = "[Online; Stand 26. November 2025]"
 }
@@ -1,332 +0,0 @@
 \documentclass{article}
 %%% basic layouting
 \usepackage[utf8x]{inputenc}
 \usepackage[margin=1in]{geometry} % Adjust margins
 \usepackage{caption}
 \usepackage{hyperref}
 \PassOptionsToPackage{hyphens}{url} % allow breaking urls
 \usepackage{float}
 \usepackage{wrapfig}
 \usepackage{subcaption}
 \usepackage{parskip} % dont indent after paragraphs, figures
 \usepackage{xcolor}
 %%% algorithms
 \usepackage{algorithm}
 \usepackage{algpseudocodex}
 % graphs and plots
 \usepackage{tikz}
 \usepackage{pgfplots}
 \usetikzlibrary{positioning}
 \usetikzlibrary{trees}
 %\usetikzlibrary{graphs, graphdrawing}
 %%% math
 \usepackage{amsmath}
 %%% citations
 \usepackage[style=ieee, backend=biber, maxnames=1, minnames=1]{biblatex}
 %\usepackage{csquotes} % Recommended for biblatex
 \addbibresource{compression.bib}
 \title{Compression}
 \date{\today}
 \begin{document}
 \maketitle
 \section{Introduction}
 As the volume of data grows exponentially around the world, compression is only gaining in importance to all disciplines.
 Not only does it enable the storage of large amounts of information needed for research in scientific domains
 like DNA sequencing and analysis, it also plays a vital role in keeping stored data accessible by
 facilitating cataloging, search and retrieval.
 The concept of entropy introduced in the previous entry is closely related to the design of efficient codes for compression.
 In coding theory, the events of an information source are to be encoded in a manner that minimizes the bits needed to store
 the information provided by the source.
 The process of encoding can thus be described by a function $C$ transforming from a source alphabet $X$ to a code alphabet $Y$.
 Symbols in the alphabets are denominated $x_i$ and $y_j$ respectively, and have underlying probabilities $p_{i}$.
 % TODO fix use of alphabet / symbol / code word: alphabet is usually binary -> code word is 010101
 \begin{equation}
    C: X \rightarrow Y \qquad X=\{x_1,x_2,...x_n\} \qquad Y=\{y_1,y_2,...y_m\}
    \label{eq:formal-code}
 \end{equation}
 The understanding of entropy as the expected information $E(I)$ of a message provides an intuition that,
 given a source with a given entropy (in bits), any coding can not have a lower average word length $l_j$ (in bits)
 than this entropy without losing information.
 \begin{equation}
    H = E(I) = - \sum_i p_i \log_2(p_i) \quad \leq \quad E(L) = \sum_i p_j l_j
    \label{eq:entropy-information}
 \end{equation}
 This is the content of Shannons's source coding theorem,
 introduced in \citeyear{shannon1948mathematical}.
 In his paper, \citeauthor{shannon1948mathematical} proposed two principal ideas to minimize the average length of a code.
 The first is to use short codes for symbols with higher probability.
 This is an intuitive approach as more frequent symbols have a higher impact on average code length.
 The second idea is to encode events that frequently occur together at the same time, artificially increasing
 the size of the code alphabet $Y$ to allow for greater flexibility in code design.\cite{enwiki:shannon-source-coding}
 Codes can have several properties. A code where all codewords have equal lengths is called a \textit{block code}.
 While easy to construct, they are not well suited for our goal of minimizing average word length
 as specified in \autoref{eq:entropy-information} because the source alphabet is generally not equally distributed
 in a way that $p_i = \frac{1}{n}$.
 In order to send (or store, for that matter) multiple code words in succession, a code $Y$ has to be uniquely decodable.
 When receiving 0010 in succesion using the nonsingular code $Y_2$ from \autoref{tab:code-properties},
 it is not clear to the recipient which source symbols make up the intended message.
 For the specified sequence, there are a total of three possibilities to decode the received code:
 $s_0 s_3 s_0$, $s_0 s_0 s_1$ or $s_2 s_1$ could all be the intended message, making the code useless.
 \begin{table}[H]
 \centering
 \begin{tabular}{c l l l}
    Source Code $X$ & Prefix Code $Y_0$ & Suffix Code $Y_1$ & Nonsingular Code $Y_2$ \\
    \hline
    $s_0$ & 0       & 0     & 0   \\
    $s_1$ & 10      & 01    & 10  \\
    $s_2$ & 110     & 011   & 00  \\
    $s_3$ & 1110    & 0111  & 01  \\
 \end{tabular}
 \caption{Examples of different properties of codes}
 \label{tab:code-properties}
 \end{table}
 Another interesting property of a code that is specifically important for transmission but less so for storage, is
 being prefix-free.
 A prefix code (which is said to be prefix-free) can be decoded by the receiver of the symbol as soon as it is received
 because no code word $y_j$ is the prefix of another valid code word.
 As shown in \autoref{tab:code-properties} $Y_0$ is a prefix code, in this case more specifically called a \textit{comma code}
 because each code word is separated by a trailing 0 from the next code word.
 $Y_1$ in contrast is called a \textit{capital code} (capitalizes the beginning of each word) and is not a prefix code.
 In the case of the capital code in fact every word other than the longest possible code word is a prefix of the longer words
 lower in the table. As a result, the receiver cannot instantaneously decode each word but rather has to wait for the leading 0
 of the next codeword.
 Further, a code is said to be \textit{efficient} if it has the smallest possible average word length, i.e. matches
 the entropy of the source alphabet.
 \section{Kraft-McMillan inequality}
 The Kraft-McMillan inequality gives a necessary and sufficient condition for the existence of a prefix code.
 In the form shown in \autoref{eq:kraft-mcmillan} it is intuitive to understand given a code tree.
 Because prefix codes require code words to only be situated on the leaves of a code tree,
 for every code word $i$ using an alphabet of size $r$, it uses up exactly $r^{-l_i}$ of the available code words.
 The sum over all of them can thus never be larger than one else
 the code will not be uniquely decodable \cite{enwiki:kraft-mcmillan}.
 \begin{equation}
    \sum_l r^{-l_i} \leq 1
    \label{eq:kraft-mcmillan}
 \end{equation}
 \section{Shannon-Fano}
 Shannon-Fano coding is one of the earliest methods for constructing prefix codes.
 It is a top-down method that divides symbols into equal groups based on their probabilities,
 recursively partitioning them to assign shorter codewords to more frequent events.
 \begin{algorithm}
 \caption{Shannon-Fano compression}
 \label{alg:shannon-fano}
 \begin{algorithmic}
    \Procedure{ShannonFano}{symbols, probabilities}
        \If{length(symbols) $= 1$}
            \State \Return codeword for single symbol
        \EndIf
        \State $\text{current\_sum} \gets 0$
        \State $\text{split\_index} \gets 0$
        \For{$i \gets 1$ \textbf{to} length(symbols)}
            \If{$|\text{current\_sum} + \text{probabilities}[i] - 0.5| < |\text{current\_sum} - 0.5|$}
                \State $\text{current\_sum} \gets \text{current\_sum} + \text{probabilities}[i]$
                \State $\text{split\_index} \gets i$
            \EndIf
        \EndFor
        \State $\text{left\_group} \gets \text{symbols}[1 : \text{split\_index}]$
        \State $\text{right\_group} \gets \text{symbols}[\text{split\_index} + 1 : \text{length(symbols)}]$
        \State Assign prefix ``0'' to codes from ShannonFano($\text{left\_group}, \ldots$)
        \State Assign prefix ``1'' to codes from ShannonFano($\text{right\_group}, \ldots$)
    \EndProcedure
 \end{algorithmic}
 \end{algorithm}
 While Shannon-Fano coding guarantees the generation of a prefix-free code with an average word length close to the entropy,
 it is not guaranteed to be optimal. In practice, it often generates codewords that are only slightly longer than necessary.
 Weaknesses of the algorithm also include the non-trivial partitioning phase \cite{enwiki:partition}, which can in practice however be solved relatively efficiently.
 Due to the aforementioned limitations, neither of the two historically slightly ambiguous Shannon-Fano algorithms are almost never used,
 in favor of the Huffman coding as described in the next section.
 \section{Huffman Coding}
 \label{sec:huffman}
 Huffman coding is an optimal prefix coding algorithm that minimizes the expected codeword length
 for a given set of symbol probabilities.  Developed by David Huffman in 1952, it guarantees optimality
 by constructing a binary tree where the most frequent symbols are assigned the shortest codewords.
 Huffman coding achieves the theoretical limit of entropy for discrete memoryless sources,
 making it one of the most important compression techniques in information theory.
 Unlike Shannon-Fano, which uses a top-down approach, Huffman coding employs a bottom-up strategy.
 The algorithm builds the code tree by iteratively combining the two symbols with the lowest probabilities
 into a new internal node.  This greedy approach ensures that the resulting tree minimizes the weighted path length,
 where the weight of each symbol is its probability.
 \begin{algorithm}
 \caption{Huffman coding algorithm}
 \label{alg:huffman}
 \begin{algorithmic}
    \Procedure{Huffman}{symbols, probabilities}
        \State Create a leaf node for each symbol and add it to a priority queue
        \While{priority queue contains more than one node}
            \State Extract two nodes with minimum frequency: $\text{left}$ and $\text{right}$
            \State Create a new internal node with frequency $\text{freq(left)} + \text{freq(right)}$
            \State Set $\text{left}$ as the left child and $\text{right}$ as the right child
            \State Add the new internal node to the priority queue
        \EndWhile
        \State $\text{root} \gets$ remaining node in priority queue
        \State Traverse tree and assign codewords: ``0'' for left edges, ``1'' for right edges
        \State \Return codewords
    \EndProcedure
 \end{algorithmic}
 \end{algorithm}
 The optimality of Huffman coding can be proven by exchange arguments.
 The key insight is that if two codewords have the maximum length in an optimal code, they must correspond to the two least frequent symbols.
 Moreover, these two symbols can be combined into a single meta-symbol without affecting optimality,
 which leads to a recursive structure that guarantees Huffman's method produces an optimal code.
 The average codeword length $L_{\text{Huffman}}$ produced by Huffman coding satisfies the following bounds:
 \begin{equation}
    H(X) \leq L_{\text{Huffman}} < H(X) + 1
    \label{eq:huffman-bounds}
 \end{equation}
 where $H(X)$ is the entropy of the source.  This means Huffman coding is guaranteed to be within one bit
 of the theoretical optimum.  In practice, when symbol probabilities are powers of $\frac{1}{2}$,
 Huffman coding achieves perfect compression and $L_{\text{Huffman}} = H(X)$.
 The computational complexity of Huffman coding is $O(n \log n)$, where $n$ is the number of distinct symbols.
 A priority queue implementation using a binary heap achieves this bound, making Huffman coding
 efficient even for large alphabets.  Its widespread use in compression formats such as DEFLATE, JPEG, and MP3
 testifies to its practical importance.
 However, Huffman coding has limitations. First, it requires knowledge of the probability distribution
 of symbols before encoding, necessitating a preprocessing pass or transmission of frequency tables.
 Second, it assigns an integer number of bits to each symbol, which can be suboptimal
 when symbol probabilities do not align well with powers of two.
 Symbol-by-symbol coding imposes a constraint that is often unneeded since codes will usually be packed in long sequences,
 leaving room for further optimization as provided by Arithmetic Coding.
 \section{Arithmetic Coding}
 Arithmetic coding is a modern compression technique that encodes an entire message as a single interval
 within the range $[0, 1)$, as opposed to symbol-by-symbol coding used by Huffman.
 By iteratively refining this interval based on the probabilities of the symbols in the message,
 arithmetic coding can achieve compression rates that approach the entropy of the source.
 Its ability to handle non-integer bit lengths makes it particularly powerful
 for applications requiring high compression efficiency.
 In the basic form, a message is first written in the base of the alphabet with a leading '$0.$': $ \text{ABBCAB} = 0.011201_3$,
 in this case yielding a ternary number as the alphabet is $ |\{A,B,C\}| = 3 $.
 This number can then be encoded to the target base (usually 2) with sufficient precision to yield back the original number, resulting in $0.0010110001_2$.
 The decoder only gets the rational number $q$ and the length $n$ of the original message.
 The encoding can then be easily reversed by changing base and rounding to $n$ digits.
 In general, arithmetic coding can produce near-optimal output for any given source probability distribution.
 This is achieved by adjusting the intervals that are interpreted as a given source symbol.
 Given the following source probabilities of $p_A = \frac{6}{8}, p_B = p_C = \frac{1}{8}$ the intervals would be adjusted to
 $ A= [0,\frac{6}{8}), B=(\frac{6}{8}, \frac{7}{8}), C=(\frac{7}{8},1]$.
 Instead of transforming the base of the number and rounding to appropriate precision, the encoder recursively refines the interval and in the end chooses a number inside that interval.
 \begin{enumerate}
    \item \textbf{Symbol:A} $A=[0, \frac{6}{8})$
    \item $ A= [0,(\frac{6}{8})^2), B=((\frac{6}{8})^2, \frac{7}{8} \cdot \frac{6}{8}), C=(\frac{7}{8} \cdot \frac{6}{8},1 \cdot \frac{6}{8}]$.
    \item \textbf{Symbol:B} $B=((\frac{6}{8})^2, \frac{7}{8} \cdot \frac{6}{8}) = (\frac{36}{64}, \frac{42}{64})$
 \end{enumerate}
 Depending on implementation, the source message can also be encoded in base $n+1$, reserving room for a special \verb|END-OF-DATA| symbol that the decoder
 will look for and consequently stop reading from the input $q$.
 \section{LZW Algorithm}
 The Lempel-Ziv-Welch (LZW) algorithm is a dictionary-based compression method that dynamically builds a dictionary
 of recurring patterns in the data as compression proceeds. Unlike entropy-based methods such as Huffman or arithmetic coding,
 LZW does not require prior knowledge of symbol probabilities, making it highly adaptable and efficient
 for a wide range of applications, including image and text compression.
 The algorithm was developed by Abraham Lempel and Jacob Ziv, with refinements by Terry Welch in 1984.
 The fundamental insight of LZW is that many data sources contain repeating patterns that can be exploited
 by replacing longer sequences with shorter codes. Rather than assigning variable-length codes to individual symbols
 based on their frequency, LZW identifies recurring substrings and assigns them fixed-length codes.
 As the algorithm processes the data, it dynamically constructs a dictionary that maps these patterns to codes,
 without requiring the dictionary to be transmitted with the compressed data.
 \begin{algorithm}
 \caption{LZW compression algorithm}
 \label{alg:lzw}
 \begin{algorithmic}
    \Procedure{LZWCompress}{data}
        \State Initialize dictionary with all single characters
        \State $\text{code} \gets$ next available code (typically 256 for byte alphabet)
        \State $w \gets$ first symbol from data
        \State $\text{output} \gets [\,]$
        \For{each symbol $c$ in remaining data}
            \If{$w + c$ exists in dictionary}
                \State $w \gets w + c$
            \Else
                \State append $\text{code}(w)$ to output
                \If{code $<$ max\_code}
                    \State Add $w + c$ to dictionary with code $\text{code}$
                    \State $\text{code} \gets \text{code} + 1$
                \EndIf
                \State $w \gets c$
            \EndIf
        \EndFor
        \State append $\text{code}(w)$ to output
        \State \Return output
    \EndProcedure
 \end{algorithmic}
 \end{algorithm}
 The decompression process is equally elegant. The decompressor initializes an identical dictionary
 and reconstructs the original data by decoding the transmitted codes.  Crucially, the decompressor
 can reconstruct the dictionary entries on-the-fly as it processes the compressed data,
 recovering the exact sequence of dictionary updates that occurred during compression.
 This property is what allows the dictionary to remain implicit rather than explicitly transmitted.
 \begin{algorithm}
 \caption{LZW decompression algorithm}
 \label{alg:lzw-decompress}
 \begin{algorithmic}
    \Procedure{LZWDecompress}{codes}
        \State Initialize dictionary with all single characters
        \State $\text{code} \gets$ next available code
        \State $w \gets \text{decode}(\text{codes}[0])$
        \State $\text{output} \gets w$
        \For{each code $c$ in $\text{codes}[1:]$}
            \If{$c$ exists in dictionary}
                \State $k \gets \text{decode}(c)$
            \Else
                \State $k \gets w + w[0]$ \quad \{handle special case\}
            \EndIf
            \State append $k$ to output
            \State Add $w + k[0]$ to dictionary with code $\text{code}$
            \State $\text{code} \gets \text{code} + 1$
            \State $w \gets k$
        \EndFor
        \State \Return output
    \EndProcedure
 \end{algorithmic}
 \end{algorithm}
 LZW's advantages make it particularly valuable for certain applications. First, it requires no statistical modeling
 of the input data, making it applicable to diverse data types without prior analysis.
 Second, the dictionary is built incrementally and implicitly, eliminating transmission overhead.
 Third, it can achieve significant compression on data with repeating patterns, such as text, images, and structured data.
 Fourth, the algorithm is relatively simple to implement and computationally efficient, with time complexity $O(n)$
 where $n$ is the length of the input.
 However, LZW has notable limitations. Its compression effectiveness is highly dependent on the structure and repetitiveness
 of the input data. On truly random data with no repeating patterns, LZW can even increase the file size.
 Additionally, the fixed size of the dictionary (typically 12 or 16 bits, allowing $2^12=4096$ or $2^16=65536$ entries)
 limits its ability to adapt to arbitrarily large vocabularies of patterns.
 When the dictionary becomes full, most implementations stop adding new entries, potentially reducing compression efficiency.
 LZW has seen widespread practical deployment in compression standards and applications.
 The GIF image format uses LZW compression, as does the TIFF image format in some variants.
 The relationship between dictionary-based methods like LZW and entropy-based methods like Huffman
 is complementary rather than competitive. LZW excels at capturing structure and repetition,
 while entropy-based methods optimize symbol encoding based on probability distributions.
 This has led to hybrid approaches that combine both techniques, such as the Deflate algorithm,
 which uses LZSS (a variant of LZ77) followed by Huffman coding of the output to achieve better compression ratios.
 \printbibliography
 \end{document}
@@ -1,49 +0,0 @@
@misc{ enwiki:signal-to-noise,
  author = "{Wikipedia contributors}",
  title = "Signal-to-noise ratio --- {Wikipedia}{,} The Free Encyclopedia",
  year = "2026",
  url = "https://en.wikipedia.org/w/index.php?title=Signal-to-noise_ratio&oldid=1334458479",
  note = "[Online; accessed 16-February-2026]"
 }
  @misc{ enwiki:convolutional,
    author = "{Wikipedia contributors}",
    title = "Convolutional code --- {Wikipedia}{,} The Free Encyclopedia",
    year = "2026",
    url = "https://en.wikipedia.org/w/index.php?title=Convolutional_code&oldid=1353119167",
    note = "[Online; accessed 6-July-2026]"
  }
@misc{ enwiki:gilbert-varshamov,
  author = "{Wikipedia contributors}",
  title = "Gilbert–Varshamov bound --- {Wikipedia}{,} The Free Encyclopedia",
  year = "2025",
  url = "https://en.wikipedia.org/w/index.php?title=Gilbert%E2%80%93Varshamov_bound&oldid=1329330572",
  note = "[Online; accessed 7-July-2026]"
 }
@misc{ enwiki:hamming-distance,
  author = "{Wikipedia contributors}",
  title = "Hamming distance --- {Wikipedia}{,} The Free Encyclopedia",
  year = "2025",
  url = "https://en.wikipedia.org/w/index.php?title=Hamming_distance&oldid=1320319980",
  note = "[Online; accessed 7-July-2026]"
 }
@misc{ enwiki:cyclic,
  author = "{Wikipedia contributors}",
  title = "Cyclic code --- {Wikipedia}{,} The Free Encyclopedia",
  year = "2026",
  url = "https://en.wikipedia.org/w/index.php?title=Cyclic_code&oldid=1362628486",
  note = "[Online; accessed 7-July-2026]"
 }
@misc{ enwiki:crc,
  author = "{Wikipedia contributors}",
  title = "Cyclic redundancy check --- {Wikipedia}{,} The Free Encyclopedia",
  year = "2026",
  url = "https://en.wikipedia.org/w/index.php?title=Cyclic_redundancy_check&oldid=1360762341",
  note = "[Online; accessed 7-July-2026]"
 }
@misc{ enwiki:reed-solomon,
  author = "{Wikipedia contributors}",
  title = "Reed–Solomon error correction --- {Wikipedia}{,} The Free Encyclopedia",
  year = "2026",
  url = "https://en.wikipedia.org/w/index.php?title=Reed%E2%80%93Solomon_error_correction&oldid=1360527569",
  note = "[Online; accessed 7-July-2026]"
 }
@@ -1,322 +0,0 @@
 \documentclass{article}
 \usepackage[utf8x]{inputenc}
 \usepackage[margin=1in]{geometry} % Adjust margins
 \usepackage{caption}
 \usepackage{wrapfig}
 \usepackage{subcaption}
 \usepackage{parskip} % dont indent after paragraphs, figures
 \usepackage{xcolor}
 %\usepackage{csquotes} % Recommended for biblatex
 \usepackage{tikz}
 \usepackage{pgfplots}
 \usepackage{svg}
 \usepackage{multirow}
 \usetikzlibrary{positioning}
 \usepackage{float}
 \usepackage{amsmath}
 \PassOptionsToPackage{hyphens}{url}
 \usepackage{hyperref} % allows urls to follow line breaks of text
 \usepackage{glossaries}
 \usepackage[style=ieee, backend=biber, maxnames=1, minnames=1]{biblatex}
 \addbibresource{correction.bib}
 \newcommand{\red}[1]{\textcolor{red}{#1}}
 \newacronym{snr}{SNR}{signal-to-noise ratio}
 \newacronym{crc}{CRC}{Cyclic Redundancy Check}
 \title{Error Correction Codes}
 \date{\today}
 \begin{document}
 \maketitle
 \section{Introduction}
 When messages are transmitted over real media, errors are inevitably introduced.
 Examples range from mundane sources of errors like scratches on CDs and capacitive coupling of conductors
 to geomagnetic storms.
 Though the source of errors are manifold, they are ubiquitous and need to be accounted for if we want any
 real chance of reading a correct message at the end.
 On a basic level, we can understand that a channel with a lower \textit{\acrfull{snr}},
 i.e. more errors per intended information will require more effort to retain the message information.
 \[ \mathrm{SNR} = \frac{P_{signal}}{P_{noise}} \]
 In an analog system, one might attempt to simply increase transmission power $P_{signal}$
 as is done for example in professional audio equipment.
 This would however require the modification of the transmission channel itself, which is only feasible to a certain degree.
 In a digital system, we could understand \acrshort{snr} as the probability of a bit being flipped in transit.
 For example, if a binary message is sent electronically, with a 1 being represented by a voltage of one volt
 and a 0 being represented by zero volts,
 the receiver is likely to observe some intermediate value such as .82 volts.
 Using a nearest-neighbor criterion, the receiver might consider anything over .5 volts to be the binary signal 1
 and anything less than .5 volts to be the binary signal 0.
 If the noise level is small on average, most transmissions will be interpreted correctly.
 However, there generally remains a chance that an intended 1 will be received as .47 volts and hence interpreted incorrectly as a 0.
 \cite{enwiki:signal-to-noise}
 In this digital system, we can improve communication reliability by using a coding scheme that is tolerant of errors.
 As a naive approach, we will simply transmit our message multiple times.
 This is called a \textit{repetition code}, where the receiver will perform a majority vote over the received bits,
 resulting in error detection for 2 bits used, and error correction for 3+ bits used.
 \begin{figure}[H]
 \begin{minipage}{.5\textwidth}
    \begin{tabular}{c|ccc}
        received & 0 & \textcolor{red}{1} & 2 \\
        \hline
        read & 0 & ERR &  1 \\
    \end{tabular}
    \begin{tabular}{c|cccccccc}
        received & 0 & \textcolor{red}{1} & \textcolor{red}{2} & 3 \\
        \hline
        read & 0 & 0 & 1 & 1  \\
    \end{tabular}
    \begin{tabular}{c|cccccccc}
        received & 0 & \textcolor{red}{1} & \textcolor{red}{2} & \textcolor{red}{3} & 4 \\
        \hline
        read & 0 & 0 & ERR & 1 & 1 \\
    \end{tabular}
 \caption{Error detection and detection for 2,3 and 4 bits used per symbol in a repetition code}
 \label{tab:detection-correction}
 \end{minipage}
 \begin{minipage}{.5\textwidth}
 \begin{tikzpicture}
  \begin{axis}[
    domain=0:1,
    samples=100,
    axis lines=middle,
    xlabel={$p$},
    ylabel={Channel Capacity $C$},
    xmin=0, xmax=1,
    ymin=0, ymax=1.1,
    xtick={0,0.25,0.5,0.75,1},
    grid=both,
    width=8cm,
    height=6cm,
    every axis x label/.style={at={(current axis.right of origin)}, anchor=west},
    every axis y label/.style={at={(current axis.above origin)}, anchor=south},
  ]
    \addplot[thick, blue] {1-(-x * log2(x) - (1-x) * log2(1-x))};
  \end{axis}
 \end{tikzpicture}
 \caption{Capacity of a binary channel with crossover probability $p$}
 \label{fig:graph-errorrate}
 \end{minipage}
 \end{figure}
 Formally, a code can be understood as a function $C : \mathcal{X} \rightarrow \Sigma^*$ mapping data $\mathcal{X}$ to a string
 (code word) over an alphabet set $\Sigma$.
 With a code word $C(x)$, we can relate the length of data word to code word, resulting in the measure of information rate:
 \[R = \frac{\log_q(|C|)}{n}\]
 Given the size $q= |\Sigma|$ of an alphabet, size of a code $|C|$ and length of the code words $n$,
 it relates the information to bits transmitted.
 Because any error correction requires redundancy, i.e. adding bits to pure information, the \textit{information rate} has to decrease
 with increasing error correction.
 As a result, $R \leq 1$ ($<$ if error correction is used) and the redundancy is $1-R$.
 \subsection{Mathematical Bounds}
 In general, the amount of errors a code can detect or correct
 is determined by the Hamming distance $h$ defined as the number of positions in which neighboring strings (code words) differ.
 "Karolin" and "Kerstin" differ in 3 letters and thus have a Hamming distance of $h=3$, just as the binary example.
 Given a Hamming distance of $h$, $(h-1)$ errors can be detected and to correct $r$ errors a minimum distance of $h\geq 2r+1$ is required.
 This is analogous to the repetition code already shown in \autoref{tab:detection-correction},
 as the length of our repetition code is directly equivalent to the hamming distance.
 The Hamming distance of a code is then defined as the minimum pairwise distance between any two code words.
 From it, minimal detection and correction capabilities follow according to the above rules.
 \cite{enwiki:hamming-distance}
 Given a $q$-ary code with length $n$ and minimum Hamming distance $d$, the Gilbert-Varshamov bound provides a bound for the size of
 the resulting code, i.e. the number of available code words given the conditions.
 Let $\mathcal{A}_q(n,d)$ be the maximum possible size of said code, then the Gilbert-Varshamov bound consists of
 $q^n$ as the number of total possible code words in a $q$-ary code of length n, divided by
 $\sum_{j=0}^{d-1} \binom{n}{j} (q-1)^j$, the size of a ball created around a code word by the required Hamming distance.
 \cite{enwiki:gilbert-varshamov}
 The ball is the geometric interpretation of the space created around each used code word by requiring a minimal hamming distance,
 i.e. requiring that no code word lie closer within the vector space.
 % TODO kraft mcmillan
 \begin{figure}[h]
  \begin{minipage}{.3\textwidth}
    \begin{tabular}{c|c}
      karolin & 0111 \\
      k\red{e}r\red{st}in & 0\red{000}
    \end{tabular} 
  \end{minipage}
  \begin{minipage}{.3\textwidth}
    \[
    \mathcal{A}_q(n,d) \geq \frac{q^n}{\sum_{j=0}^{d-1} \binom{n}{j} (q-1)^j}
    \]
  \end{minipage}
 \end{figure}
 \section{Block Codes}
 Block codes are \textit{memoryless}, meaning that each block is encoded independently using a static dictionary.
 Importantly, they can be described by a tuple $(n,k,d)$ taking in $k$ bits of information outputting $n$ bits of code word
 while maintaining minimum distance $d$,
 implying that the code words produced are of fixed length.
 \subsection{Cyclic Redundancy Check}
 A \acrfull{crc} is a method of detecting (correcting) errors by interpreting the information to be sent as a polynomial.
 They are particularly suited for the type of burst error common in storage media such as DVDs.
 Typically an $n$-bit \acrshort{crc} can detect any error burst of length $n$,
 with a chance of also detecting longer error bursts of approximately $1-2^{-n}$.
 Specification of a \acrshort{crc} code requires definition of a so-called \textit{generator polynomial}.
 It is used as the divisor in a polynomial division taking the message as the dividend.
 The remainder of $n$ bits is then appended to the message, thus requiring a generator polynomial of degree $n$ for the calculation.
 Because the division is calculated in a finite (also known as galois) field, so the operation can be performed
 bitwise-parallel, reducing to a simple bitwise \verb|XOR| in the binary case.
 One of the simplest error-detection systems besides a repetition code, the parity bit, is in fact a 1-bit \acrshort{crc}.
 Using the generator polynomial $g=x+1$ of degree 1 results in the well-known pattern of extending the code words to achieve
 an even number of 1s.
 \cite{enwiki:crc}
 \begin{figure}[H]
 \begin{minipage}{0.7\textwidth}
  To calculate the \acrshort{crc}-1, the message $100$ is first extended by $n$ 0s to $1000$.
  It is then divided by the generator polynomial $g=x+1 \equiv (11_2)$, leaving a remainder of $1$.
  Thus, the code word that should be transmitted is $1001$.
  As a result, any 1-bit flip will be detected, allowing the receiver to request retransmission of the message.
 \end{minipage}
 \hspace{1cm}
 \begin{minipage}{0.2\textwidth}
 \begin{verbatim}
  1000 : 11 = 111
 :11
 =010
  :11
  =010
   :11
    =1
 \end{verbatim}
 \end{minipage}
 \end{figure}
 \subsection{Interleaving}
 Another mitigation for burst errors is interleaving.
 When strong burst errors occur for short intervals in a channel that is otherwise on average low distortion,
 the idea is to spread out code words over time, i.e. interleave them in a pattern known to both transmitter and receiver.
 \[
 \begin{array}{cccc}
  (x_{11} & x_{12} & x_{13} & x_{14}) \\
  (x_{21} & x_{22} & x_{23} & x_{24}) \\
  (x_{31} & x_{32} & x_{33} & x_{34}) \\
  (x_{41} & x_{42} & x_{43} & x_{44}) \\
 \end{array}
 \rightarrow
 x_{11} x_{21}x_{31}x_{41}x_{12}x_{22}x_{32}x_{42}x_{13}x_{23}x_{33}x_{43}x_{14}x_{24}x_{34}x_{44}
 \]
 In the above example, we interleave 4 code words $x_1$ through $x_4$ of length 4 by writing them row-wise in a table.
 They are then read and transmitted columnwise, cycling bit by bit through each code word, leaving each of them less vulnerable to a short
 burst error.
 The receiver has to keep a buffer of 4 code words to de-interleave the transmission and then apply correction as usual.
 \subsection{Reed-Solomon}
 % TODO cyclic code
 Similar to \acrshort{crc}, a Reed-Solomon code also interprets the message as a polynomial.
 This polynomial of degree $k-1$ is uniquely identified by any $k$ evaluation points,
 employing the same property as Shamir's secret sharing.
 By transmitting $n>k$ points, a Reed-Solomon code can detect $t=n-k$ errors or locate and correct up to $\lfloor t/2 \rfloor$ errors.
 Formally, the message will be $(a_1,a_2,...,a_k)$ coefficients of a polynomial
 \[
 f(x) = a_1 + a_2 x + a_3 x^2+...+a_k x^{k-1} = \sum_{i=1}^{k} a_i x^{i-1}
 \]
 \cite{enwiki:reed-solomon}
 \clearpage
 \section{Convolutional Codes}
 A convolutional code generates its parity symbols by using a sliding input window of a boolean polynomial function.
 A visualisation is provided in \autoref{fig:convolutional}, where bit inputs are processed by a sliding window of size 3,
 processed by 3 different functions resulting in a tuple $(C_1,C_2,C_3)$ being sent over the channel.
 The rate of such a convolutional code is simply determined by the number of input bits to output bits, in this case $\frac{1}{3}$
 as our encoder will output 3 bits for every new input bit being shifted into the window.
 \begin{figure}[H]
 \begin{minipage}{0.4\textwidth}
    \centering
    \includesvg[height=5cm]{figures/convolutional.drawio.svg}
 \end{minipage}
 \hfill
 \begin{minipage}{0.4\textwidth}
  \begin{tabular}{cc|ccc|c}
    $S_0S_1$ & In & $C_1$ & $C_2$ & $C_3$ & $S_0'S_1'$ \\
    \hline
    \multirow{2}{*}{00} & 0 & 0 & 0 & 0 & 00 \\
                        & 1 & 1 & 0 & 1 & 10 \\
    \hline
    \multirow{2}{*}{10} & 0 & 1 & 1 & 0 & 01 \\
                        & 1 & 0 & 1 & 0 & 11 \\
    \hline
    \multirow{2}{*}{01} & 0 & 1 & 1 & 0 & 00 \\
                        & 1 & 0 & 1 & 0 & 10 \\
    \hline
    \multirow{2}{*}{11} & 0 & 0 & 0 & 1 & 01\\
                        & 1 & 1 & 0 & 0 & 11\\
  \end{tabular}
 \end{minipage}
    \caption{Visualisation and transitions of a convolutional encoder}
    \label{fig:convolutional}
 \end{figure}
 Knowing the underlying function allows us to first create a table that later translates to a \textit{trellis},
 a data structure later used in decoding using the Viterbi algorithm.
 Specifically, we can understand this window of 3 bits to be 2 bits of state $S_0S_1$
 and 1 bit of input, that is the leftmost bit in the window in \autoref{fig:convolutional}.
 \subsection{Viterbi algorithm}
 The viterbi algorithm operates on a \textit{trellis}, that is a variant of a state machine that shows the transitions
 from $S_0S_1$ to $S_0'S_1'$ like in \autoref{fig:trellis}.
 \begin{figure}[H]
 \begin{tikzpicture}
    % Draw the nodes
    \node (00) at (0,3) [draw,circle] {00};
    \node (10) at (0,2) [draw,circle] {10};
    \node (01) at (0,1) [draw,circle] {01};
    \node (11) at (0,0) [draw,circle] {11};
    \node (00') at (3,3) [draw,circle] {00};
    \node (10') at (3,2) [draw,circle] {10};
    \node (01') at (3,1) [draw,circle] {01};
    \node (11') at (3,0) [draw,circle] {11};
    % Draw arrows between nodes
    \draw[->] (00) -- (00') node[midway, above] {0} ;
    \draw[->] (00) -- (10') node[pos=0.2, below] {1} ;
    \draw[->] (10) -- (01') node[pos=0.2, above] {0} ;
    \draw[->] (10) -- (11') node[midway, above] {1} ;
    \draw[->] (01) -- (00') node[midway, above] {0} ;
    \draw[->] (01) -- (10') node[pos=0.2, below] {1} ;
    \draw[->] (11) -- (01') node[pos=0.2, above] {0} ;
    \draw[->] (11) -- (11') node[midway, below] {1} ;
 \end{tikzpicture}
 \caption{Trellis for the viterbi algorithm}
 \label{fig:trellis}
 \end{figure}
 For a given received sequence, we can then duplicate this trellis for each block of (in this example 3) bits
 to decide which bits were most likely sent by minimizing the hamming distance through the graph.
 As a result, we will decode a minimum error input message by retracing the path through the trellis.
 \section{Conclusion}
 Numerous different forward error correction methods exist that optimise for different scenarios and vary in complexity.
 In practice, they find applications in data storage and transmission, often in combination of each other,
 with the most interesting implementations being used for extreme scenarios such as satellite communication.
 \clearpage
 \printbibliography
 \end{document}
@@ -1,82 +0,0 @@
@book{luenberger,
  title={Information science},
  author={Luenberger, David G},
  year={2012},
  publisher={Princeton University Press}
 }
@misc{ enwiki:maryofscots,
  author = "{Wikipedia contributors}",
  title = "Mary, Queen of Scots --- {Wikipedia}{,} The Free Encyclopedia",
  year = "2026",
  url = "https://en.wikipedia.org/w/index.php?title=Mary,_Queen_of_Scots&oldid=1333198012",
  note = "[Online; accessed 22-January-2026]"
 }
@misc{ enwiki:kerckhoff,
  author = "{Wikipedia contributors}",
  title = "Kerckhoffs's principle --- {Wikipedia}{,} The Free Encyclopedia",
  year = "2025",
  url = "https://en.wikipedia.org/w/index.php?title=Kerckhoffs%27s_principle&oldid=1320402404",
  note = "[Online; accessed 2-February-2026]"
 }
@misc{ enwiki:confusion-diffusion,
  author = "{Wikipedia contributors}",
  title = "Confusion and diffusion --- {Wikipedia}{,} The Free Encyclopedia",
  year = "2025",
  url = "https://en.wikipedia.org/w/index.php?title=Confusion_and_diffusion&oldid=1307746165",
  note = "[Online; accessed 3-February-2026]"
 }
@ARTICLE{diffiehellman,
  author={Diffie, W. and Hellman, M.},
  journal={IEEE Transactions on Information Theory},
  title={New directions in cryptography},
  year={1976},
  volume={22},
  number={6},
  pages={644-654},
  keywords={Cryptography;Receivers;Authentication;Eavesdropping;Costs;Business;Public key cryptography},
  doi={10.1109/TIT.1976.1055638}}
@article{rsa,
  title={A method for obtaining digital signatures and public-key cryptosystems},
  author={Rivest, Ronald L and Shamir, Adi and Adleman, Leonard},
  journal={Communications of the ACM},
  volume={21},
  number={2},
  pages={120--126},
  year={1978},
  publisher={ACM New York, NY, USA}
 }
@misc{ enwiki:ciphersuite,
  author = "{Wikipedia contributors}",
  title = "Cipher suite --- {Wikipedia}{,} The Free Encyclopedia",
  year = "2026",
  url = "https://en.wikipedia.org/w/index.php?title=Cipher_suite&oldid=1342766306",
  note = "[Online; accessed 10-April-2026]"
 }
@misc{ enwiki:mitm,
  author = "{Wikipedia contributors}",
  title = "Man-in-the-middle attack --- {Wikipedia}{,} The Free Encyclopedia",
  year = "2026",
  url = "https://en.wikipedia.org/w/index.php?title=Man-in-the-middle_attack&oldid=1347824570",
  note = "[Online; accessed 10-April-2026]"
 }
@misc{ enwiki:galoismode,
  author = "{Wikipedia contributors}",
  title = "Galois/Counter Mode --- {Wikipedia}{,} The Free Encyclopedia",
  year = "2026",
  url = "https://en.wikipedia.org/w/index.php?title=Galois/Counter_Mode&oldid=1352962810",
  note = "[Online; accessed 10-May-2026]"
 }
@book{rijndael,
  title={The design of Rijndael},
  author={Daemen, Joan and Rijmen, Vincent},
  volume={2},
  year={2002},
  publisher={Springer}
 }
@misc{ enwiki:aes,
  author = "{Wikipedia contributors}",
  title = "Advanced Encryption Standard --- {Wikipedia}{,} The Free Encyclopedia",
  year = "2026",
  url = "https://en.wikipedia.org/w/index.php?title=Advanced_Encryption_Standard&oldid=1358692329",
  note = "[Online; accessed 28-June-2026]"
 }
@@ -1,284 +0,0 @@
 \documentclass{article}
 \usepackage[utf8x]{inputenc}
 \usepackage[margin=1in]{geometry} % Adjust margins
 \usepackage{caption}
 \usepackage{wrapfig}
 \usepackage{subcaption}
 \usepackage{parskip} % dont indent after paragraphs, figures
 \usepackage{xcolor}
 %\usepackage{csquotes} % Recommended for biblatex
 \usepackage{tikz}
 \usepackage{pgfplots}
 \usetikzlibrary{positioning}
 \usepackage{float}
 \usepackage{amsmath}
 \PassOptionsToPackage{hyphens}{url}
 \usepackage{hyperref} % allows urls to follow line breaks of text
 \usepackage[style=ieee, backend=biber, maxnames=1, minnames=1]{biblatex}
 \addbibresource{crypto.bib}
 \usepackage{glossaries}
 \newcommand{\ub}[2]{\underbrace{\text{#1}}_{\text{#2}}}
 \newcommand{\secret}[1]{\textcolor{red}{#1}}
 \newcommand{\public}[1]{\textcolor{blue}{#1}}
 \makeglossaries
 \newacronym{DES}{DES}{Data Encryption Standard}
 \newacronym{AES}{AES}{Advanced Encryption Standard}
 \newacronym{RSA}{RSA}{Rivest–Shamir–Adleman}
 \newacronym{mitm}{MITM}{Man-in-the-middle attack}
 \newacronym{CA}{CA}{Certificate Authority}
 \title{Cryptography}
 \date{\today}
 \begin{document}
 \maketitle
 \section{Introduction}
 Cryptography is ubiquitous in our modern world.
 While the origins of cryptography date back thousands of years, evidence of its use in ancient times is sparse.
 \cite{luenberger}
 Historically, most of its use seemed to be reserved for political and military leaders, e.g. notably Mary Queen of Scots,
 who while in prison, plotted to kill Queen Elizabeth using encrypted letters \cite{enwiki:maryofscots}.
 Much later, with the widespread adoption of the internet, the need for several cryptographical functions arose.
 Due to its intended use as a trusted research network (ARPANET),
 almost none of the original protocols were 'secure' in any sense of the word.
 One of the most notable examples still today is SMTP, the \textit{Simple Mail Transfer Protocol}, used to send email to servers.
 In its original implementation, it allowed attackers to intercept emails in transit to read and modify them
 and even spoof the sender address to impersonate others.
 SMTP today is secured using a combination of mitigations for these attacks, such as STARTTLS, SPF, DKIM and DMARC,
 emphasizing the need for securely designed protocols.
 \subsection{Security}
 Common goals associated with security include the \textit{CIA triad}, consisting of
 \begin{itemize}
    \item Confidentiality: Prevent unauthorized reading
    \item Integrity: Prevent unauthorized modification
    \item Availability: Prevent denial of service
 \end{itemize}
 With further goals including Authenticity and Non-repudiation. Cryptography can help with all of the aforementioned goals
 except availability.
 This can be achieved using several different applications of cryptography:
 \begin{itemize}
    \item Encryption provides confidentiality by only saving / transmitting an encrypted message.
    \item Hash functions ensure data has not been altered.
    \item Digital signatures confirm a message was indeed sent by who we expect it to be, preventing man-in-the-middle attacks
    where the message is simply swapped out before reaching its destination, as well as providing proof a message was sent (Non-repudiation).
    \item Certificates confirm the sender's identity.
 \end{itemize}
 Importantly, Kerckhoff's principle \cite{enwiki:kerckhoff} is what allows us to go into detail on the following algorithms.
 Embraced by researchers today, it holds that the security of a cryptosystem should only rely on the secrecy of the key,
 allowing and encouraging the publication of cryptographic algorithms. \newline
 It is closely related to Shannon's maxim, stating that
 "one ought to design systems under the assumption that the enemy will immediately gain full familiarity with them".
 This is opposed to \textit{security through obscurity}, which doesnt allow for verification of the cryptographic
 algorithm through a scientific process in the public domain.
 \subsection{Hash Functions}
 A general hash function $h(m)$ is a function that takes a message $m$ of arbitrary and produces an output $h$ called \textit{hash}
 of fixed length. However, not every mathematical function can be considered a hash function.
 The main applications of hash functions include integrity checking and hash maps for efficient data retrieval.
 Depending on the applications, different properties determine the usefulness of a function.
 An obvious desired property is efficiency - every application benefits from faster computing times.
 Also central to all applications of hash functions is a property called \textit{collision resistance}, where there should be no
 efficient way, i.e. no better way than brute force to find $m_1 \neq m_2$ so that $h(m_1) = h(m_2)$.
 Again, for encryption the importance is clear. If a password is stored in hashed form to obfuscate the clear text,
 no security is gained if it is easy for an attacker to find a password that produces the same hash and thus passes the challenge.
 A similar notion holds true for data retrieval. If it is too easy to find collisions, e.g. similar inputs produce similar outputs,
 there will be an uneven distribution in the target domain and thus little to no efficiency gain.
 Another desired property, specifically for encryption is what is usually used synonymously with a hash function: a \textit{one-way function}.
 Given $h(m)$, there should be no method more efficient than brute force to find a matching $m$. \newline
 As alluded to earlier, hash functions are readily used for integrity checking.
 By generating a fixed-size hash value for a given input, they allow users to verify that data has not been altered,
 whether intentionally or accidentally.
 For example, when downloading a file, comparing its hash with a published checksum ensures the file's integrity.
 They are also often used in combination with public key cryptography, allowing the sender to sign with his private key
 to prove not only integrity but authenticity.
 \subsection{Encryption}
 Even though the properties of hash functions are similar to encryption, the fact that the input message is reduced to a fixed size hash
 also means that inevitably information is lost by every hash function.
 Fundamentally, encryption has the goal of only allowing authorized parties to read a message.
 This is achieved by encoding the \textit{plaintext} into a \textit{ciphertext} and then transmitting/storing that ciphertext
 separately from the necessary key to decrypt it.
 Early encryptions intuitively demonstrate two concepts that can be employed to encode a message:
 \textit{substitution} and \textit{transposition}.
 \paragraph{Substitution} is used by
 the simple Caesar cipher, often achieved by rotating two disks against each other, each with the alphabet written out on them.
 \autoref{tab:caesar} shows a simple caesar cipher where the cipher alphabet is simply shifted by $+3$ positions from the plaintext alphabet.
 In the process of encoding, A is therefore replaced (substituted) with D, B with E, and so on.
 Upon reception of the message, the same process is done in reverse, i.e. shifted by $-3$.
 \begin{table}[h]
 \resizebox{\textwidth}{!}{
 \begin{tabular}{c|c|c|c|c|c|c|c|c|c|c|c|c|c|c|c|c|c|c|c|c|c|c|c|c|c}
    A&B&C&D&E&F&G&H&I&J&K&L&M&N&O&P&Q&R&S&T&U&V&W&X&Y&Z \\
    \hline
    D&E&F&G&H&I&J&K&L&M&N&O&P&Q&R&S&T&U&V&W&X&Y&Z&A&B&C
 \end{tabular}
 }
 \caption{A simple substitution cipher demonstrated by a 3-letter shift.}
 \label{tab:caesar}
 \end{table}
 This simple encryption is easy to break however for several reasons.
 Caesar ciphers in general only offer 26 different keys as further shifts only wrap around to e.g. $29 \mod 26 = 3$, with a shift of 26
 being equal to the cleartext. \newline
 Furter, by shifting every letter by the same amount,
 properties of the source language such as word spacing and letter frequencies are retained in the ciphertext,
 leaving it vulnerable to simple statistical attacks.
 \paragraph{Transposition} is the process of reordering the plaintext to obtain a ciphertext.
 Here, the key can be understood as instructions on how to re-order the ciphertext to obtain the original message.
 The \textit{scytale} is one of the earliest implementations of a transposition cipher.
 The implementation involves a rope or band of text as well as a stick of known circumference as the key.
 The band is wrapped in a spiral around the stick and the message written across the spiral.
 When the stick is removed and the band unwrapped, the letters on the band appear scrambled by a fixed offset determined
 by the stick's circumference.
 \paragraph{Diffusion} is one of two properties of a secure cipher introduced by Shannon in 1945.
 It is closely related to the \textit{collision resistance} of hash functions
 and means that for a 1-bit change of the plaintext, about half the bits of the ciphertext should change.
 The purpose of diffusion is to hide the statistical statistical relationship between plaintext and ciphertext
 exhibited by simple encryption methods.
 % Block ciphers achieve this by "diffusing" the information about the plaintext's structure across the rows and columns of the cipher.
 \paragraph{Confusion} similarly aims to obscure the connection of ciphertext and key,
 requiring each bit of the ciphertext to depend on multiple parts of the key.
 Confusion and diffusion are often mistaken for substitution and transposition, as block ciphers achieve
 confusion through substitution boxes (S-box) and
 diffusion through permutation boxes (P-box).
 \cite{enwiki:confusion-diffusion}
 \section{Symmetric Encryption}
 \subsection{DES}\label{sec:des}
 The \acrfull{DES} is a symmetric (or private-key) cipher developed in the 1970s at IBM as an archetypal block cipher.
 It takes in a block of 64 bits and transforms it to a ciphertext using a key of equal length.
 Despite suspicions of backdoors engineered into the algorithm due to the involvement of the NSA in the development of \acrshort{DES},
 only minor weaknesses were found.
 It was approved as a federal standard in the USA in 1976 and only retired due to its short key length,
 for which the NSA however was directly responsible as well.
 Nevertheless, it sparked public and scientific interest in the research of encryption algorithms, producing a large body of publications.
 Extended versions of \acrshort{DES} such as Triple-DES (or 3DES) are still in use in embedded applications (chipcards).
 In \acrshort{DES}, the message is encrypted in 16 rounds, as well as an initial and final permutation (IP and FP),
 which are inverses. IP and FP were included to facilitate hardware loading of blocks.
 After splitting the 64-Bit message into two . 2-Bit blocks left (L) and right (R), the two halves are processed in the
 16-round feistel network in a criss-cross pattern as follows:
 \begin{itemize}
    \item derive a 48-Bit \textit{round key} from the original 64-Bit \textit{main key} using a fixed \textit{key schedule}.
    \item apply the \textit{Feistel function} to the R-block, which also applies the \textit{round key}.
    \item XOR the right block to the left block.
 \end{itemize}
 Modern encryptions are commonly based on feistel networks as they make it easy to ensure bijection,
 thus providing the necessary criterion for a cipher.
 Further, due to the decryption process using the same algorithm with a reversed \textit{key schedule}
 as well as the iterative nature of a feistel network,
 they naturally lend themselves to hardware implementations.
 \subsection{AES} %TODO
 The \acrfull{AES} superseded \acrshort{DES} in 2001 after an official selection process.
 Standardized under \verb|ISO 18033-3|, \acrshort{AES} supports key lengths of 128, 192 and 256 bits,
 the underlying algorithm supports any key length that is a multiple of 32.
 Developed as Rijndael, contrary to its predecessor it does not employ a Feistel network,
 though it does use the familiar pattern of substitution and permutation boxes.
 The key size used also specifies the number of rounds, as a round key is derived for each transformation round,
 resulting in 10 (14) rounds for 128-bit (256-bit) keys.
 First, the round keys are derived and an initial \verb|AddRoundKey| is performed, before iterating over the following steps with
 a 4x4 byte array termed \textit{the state}:
 \begin{enumerate}
    \item \verb|SubBytes| substitutes 8-bit blocks according to a lookup table
    \item \verb|ShiftRows| transposes the last 3 rows of the state
    \item \verb|MixColumns| linearly mixes the columns
    \item \verb|AddRoundKey| adds the round key to the state
 \end{enumerate}
 Then, a final round is performed that only consists of SubBytes, ShiftRows, AddRoundKey.
 Though minor theoretical attacks exist, such as reducing the key search space by a factor of 4,
 the algorithm can be considered secure even to quantum-computing supported attacks only halving the key space,
 as simply increasing the key length is a viable mitigation.
 \cite{rijndael,enwiki:aes}
 \section{Asymmetric Encryption}
 Symmetric encryption however historically suffered from a key exchange problem;
 because the same key is used for encryption and decryption, a secure channel is required to agree on a common key.
 This chicken-and-egg problem can be solved in two major ways, both typically relying on the mathematical theory of
 \textbf{galois fields}, employing either \textit{modular arithmetic} or \textit{elliptic curves}.
 %NOTE section on galois fields/ modulo/elliptic curve?
 \paragraph{The Difie-Hellman Key Exchange} is an algorithm allowing the communication parties to establish a shared secret using
 properties of the discrete logarithm.
 The algorithm is implemented by the following steps, with \public{public values} in blue and \secret{secret values} in red.
 \begin{enumerate}
    \item First, Alice and Bob publicly agree on prime numbers \public{$p$} and \public{$g$}.
    \item Alice (and Bob, each on their own) choose their secret keys \secret{$a$} (\secret{$b$})
    \item They then exchange
        $\public{A} = \public{g}^{\secret{a}} \mod \public{p}$ and $\public{B} = \public{g}^{\secret{b}} \mod \public{p}$, respectively.
    \item Both arrive at the same key $\secret{s}$ by computing $\secret{s} = \public{B}^{\secret{a}} \mod \public{p} =
        \public{A}^{\secret{b}} \mod \public{p}  $.
 \end{enumerate}
 Both methods however are still vulnerable to a \acrfull{mitm}, thus also requiring a trusted \acrfull{CA} for authentication. \cite{enwiki:mitm}
 \subsection{RSA}\label{sec:rsa}
 \acrfull{RSA} is the first asymmetric (or public-key) cryptographic algorithm and can thus be used for encryption and digital signing.
 It was named after its eponymous inventors in \citeyear{rsa} after trying to disprove the existence of \textit{trapdoor functions},
 a concept introduced by \citeauthor{diffiehellman} in their appropriately named pivotal paper \citetitle{diffiehellman} \cite{diffiehellman}.
 The algorithm they came up with relies on modular arithmetic, which remains the most popular class of asymmetric cryptography.
 \begin{enumerate}
    \item Choose randomly and stochastically independet primes $\secret{p,q}$ of similar size so that
        \newline$0.1 < | \log_2 p - \log_2 q | < 30 $.
    \item Calculate $ \public{N}= \secret{p} \cdot \secret{q} $
    \item Compute Euler's totient function of $ \secret{\varphi (N)} = (\secret{p}-1) \cdot (\secret{q}-1)$ which is kept secret.
    \item Choose an integer $\public{e}$ so that $ 1 < \public{e} < \secret{\varphi (N)} $ and $\gcd(\public{e}, \secret{\varphi(N)}) =1$,
        i.e. $\public{e}$ and $\secret{\varphi(N)}$ are coprime.
        The most common choice here is $ e= 2^(16) +1 = 65537 $, as $\public{e}$ is released as part of the public key.
    \item Calculate $ \secret{d}  \equiv \public{e}^{-1}$, i.e. $\secret{d}\public{e} = 1\quad | \mod \secret{\varphi(N)}$
        (modular multiplicative inverse).
 \end{enumerate}
 The \public{public key} consists of the modulus $N$ and the public exponent $e$.
 The \secret{private key} consists of the private exponent $d$, as a result of which $p,q,\varphi(N)$ have to be kept secret as well.
 In the encryption process, the ciphertext $c$ is obtained from the message $m$ by calculating $c \equiv m^e \mod N$.
 After transmission, the message can then be recovered using the the private key exponent $d$ as follows:
 \[ c^d \equiv (m^e)^d = m^{e*d} \equiv m^1 | \mod N \]
 \clearpage
 \section{Conclusion}
 Trust on the web with untrusted channels fundamentally remains an unsolved issue,
 though depending on the threat model, everyday communications can be considered relatively secure from non-APT actors.
 A typical cipher suite employed by TLS to achieve this security could look like the following:
 $$\ub{ECDHE}{Key exchange}-\ub{ECDSA}{authentication}-\ub{AES128}{encryption}-\ub{GCM}{Cipher operation mode}-\ub{SHA256}{hashing} $$
 \begin{itemize}
    \item \textbf{ECDHE} Elliptic Curve Diffie Hellman Exchange
    \item \textbf{ECDSA} Elliptic Curve Digital Signing Algorithm
    \item \textbf{AES128} 128-Bit \acrfull{AES} symmetric encryption
    \item \textbf{GCM} Galois Counter Mode
    \item \textbf{SHA256} Secure Hash Algorithm
 \end{itemize}
 \cite{enwiki:ciphersuite,enwiki:galoismode}
 %\printglossary[type=\acronymtype]
 %\printglossary
 \printbibliography
 \end{document}
@@ -21,11 +21,12 @@
 \title{Entropy as a measure of information}
 \author{Erik Neller}
 \date{\today}
 \begin{document}
 \maketitle
-\section{Introduction}
+\section{What is entropy?}
 Across disciplines, entropy is a measure of uncertainty or randomness.
 Originating in classical thermodynamics,
 over time it has been applied in different sciences such as chemistry and information theory.
@@ -347,22 +348,13 @@ The capacity of the binary symmetric channel is given by:
 where $H_2(p) = -p \log_2(p) - (1-p)\log_2(1-p)$ is the binary entropy function.
 As $p$ increases, uncertainty grows and channel capacity declines.
 When $p = 0.5$, output bits are completely random and no information can be transmitted ($C = 0$).
-As illustrated in \autoref{fig:graph-entropy}, an error rate over $p > 0.5$ is equivalent to $ 1-p < 0.5$,
+As already shown in \autoref{fig:graph-entropy}, an error rate over $p > 0.5$ is equivalent to $ 1-p < 0.5$,
 though not relevant in practice.
 Shannon’s theorem is not constructive as it does not provide an explicit method for constructing such efficient codes,
 but it guarantees their existence.
 In practice, structured codes such as Hamming and Reed–Solomon codes are employed to approach channel capacity.
 \section{Conclusion}
 Entropy provides a fundamental measure of uncertainty and information,
 bridging concepts from thermodynamics to modern communication theory.
 Beyond the provided examples, the concept of entropy has far-reaching applications in diverse fields:
 from cryptography, where it quantifies randomness and security, 
 to statistical physics, where it characterizes disorder in complex systems,
 to biology, connecting molecular information and population diversity.
 \printbibliography
-\end{document}
+\end{document}
@@ -1,4 +0,0 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!-- Do not edit this file with editors other than draw.io -->
 <!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
 <svg xmlns="http://www.w3.org/2000/svg" style="background: transparent; background-color: transparent; color-scheme: light dark;" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" width="390px" height="341px" viewBox="0 0 390 341" id="ge-svg-OR950KacOfshYnHW8CG3" content="&lt;mxfile host=&quot;app.diagrams.net&quot; scale=&quot;1&quot; border=&quot;0&quot;&gt;&#xA;  &lt;diagram name=&quot;Page-1&quot; id=&quot;-0qN2aSnk7K1o-c1WUeR&quot;&gt;&#xA;    &lt;mxGraphModel dx=&quot;1161&quot; dy=&quot;813&quot; grid=&quot;1&quot; gridSize=&quot;10&quot; guides=&quot;1&quot; tooltips=&quot;1&quot; connect=&quot;1&quot; arrows=&quot;1&quot; fold=&quot;1&quot; page=&quot;1&quot; pageScale=&quot;1&quot; pageWidth=&quot;850&quot; pageHeight=&quot;1100&quot; math=&quot;0&quot; shadow=&quot;0&quot;&gt;&#xA;      &lt;root&gt;&#xA;        &lt;mxCell id=&quot;0&quot; /&gt;&#xA;        &lt;mxCell id=&quot;1&quot; parent=&quot;0&quot; /&gt;&#xA;        &lt;mxCell id=&quot;BxGZwdInxDfRHZgYTuuF-1&quot; parent=&quot;1&quot; style=&quot;whiteSpace=wrap;html=1;aspect=fixed;&quot; value=&quot;&quot; vertex=&quot;1&quot;&gt;&#xA;          &lt;mxGeometry height=&quot;80&quot; width=&quot;80&quot; x=&quot;270&quot; y=&quot;330&quot; as=&quot;geometry&quot; /&gt;&#xA;        &lt;/mxCell&gt;&#xA;        &lt;mxCell id=&quot;BxGZwdInxDfRHZgYTuuF-2&quot; parent=&quot;1&quot; style=&quot;whiteSpace=wrap;html=1;aspect=fixed;&quot; value=&quot;&quot; vertex=&quot;1&quot;&gt;&#xA;          &lt;mxGeometry height=&quot;80&quot; width=&quot;80&quot; x=&quot;350&quot; y=&quot;330&quot; as=&quot;geometry&quot; /&gt;&#xA;        &lt;/mxCell&gt;&#xA;        &lt;mxCell id=&quot;BxGZwdInxDfRHZgYTuuF-3&quot; parent=&quot;1&quot; style=&quot;whiteSpace=wrap;html=1;aspect=fixed;&quot; value=&quot;&quot; vertex=&quot;1&quot;&gt;&#xA;          &lt;mxGeometry height=&quot;80&quot; width=&quot;80&quot; x=&quot;430&quot; y=&quot;330&quot; as=&quot;geometry&quot; /&gt;&#xA;        &lt;/mxCell&gt;&#xA;        &lt;mxCell id=&quot;BxGZwdInxDfRHZgYTuuF-4&quot; edge=&quot;1&quot; parent=&quot;1&quot; style=&quot;shape=flexArrow;endArrow=classic;html=1;rounded=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;&quot; value=&quot;&quot;&gt;&#xA;          &lt;mxGeometry height=&quot;50&quot; relative=&quot;1&quot; width=&quot;50&quot; as=&quot;geometry&quot;&gt;&#xA;            &lt;mxPoint x=&quot;200&quot; y=&quot;370&quot; as=&quot;sourcePoint&quot; /&gt;&#xA;            &lt;mxPoint x=&quot;261&quot; y=&quot;370&quot; as=&quot;targetPoint&quot; /&gt;&#xA;          &lt;/mxGeometry&gt;&#xA;        &lt;/mxCell&gt;&#xA;        &lt;mxCell id=&quot;BxGZwdInxDfRHZgYTuuF-6&quot; edge=&quot;1&quot; parent=&quot;1&quot; style=&quot;shape=flexArrow;endArrow=classic;html=1;rounded=0;exitX=1;exitY=0.5;exitDx=0;exitDy=0;&quot; value=&quot;&quot;&gt;&#xA;          &lt;mxGeometry height=&quot;50&quot; relative=&quot;1&quot; width=&quot;50&quot; as=&quot;geometry&quot;&gt;&#xA;            &lt;mxPoint x=&quot;518&quot; y=&quot;370&quot; as=&quot;sourcePoint&quot; /&gt;&#xA;            &lt;mxPoint x=&quot;588&quot; y=&quot;370&quot; as=&quot;targetPoint&quot; /&gt;&#xA;          &lt;/mxGeometry&gt;&#xA;        &lt;/mxCell&gt;&#xA;        &lt;mxCell id=&quot;BxGZwdInxDfRHZgYTuuF-7&quot; parent=&quot;1&quot; style=&quot;ellipse;whiteSpace=wrap;html=1;aspect=fixed;&quot; value=&quot;&amp;lt;font style=&amp;quot;font-size: 25px;&amp;quot;&amp;gt;+&amp;lt;/font&amp;gt;&quot; vertex=&quot;1&quot;&gt;&#xA;          &lt;mxGeometry height=&quot;60&quot; width=&quot;60&quot; x=&quot;440&quot; y=&quot;240&quot; as=&quot;geometry&quot; /&gt;&#xA;        &lt;/mxCell&gt;&#xA;        &lt;mxCell id=&quot;BxGZwdInxDfRHZgYTuuF-8&quot; parent=&quot;1&quot; style=&quot;ellipse;whiteSpace=wrap;html=1;aspect=fixed;&quot; value=&quot;&amp;lt;font style=&amp;quot;font-size: 25px;&amp;quot;&amp;gt;+&amp;lt;/font&amp;gt;&quot; vertex=&quot;1&quot;&gt;&#xA;          &lt;mxGeometry height=&quot;60&quot; width=&quot;60&quot; x=&quot;440&quot; y=&quot;440&quot; as=&quot;geometry&quot; /&gt;&#xA;        &lt;/mxCell&gt;&#xA;        &lt;mxCell id=&quot;BxGZwdInxDfRHZgYTuuF-9&quot; parent=&quot;1&quot; style=&quot;ellipse;whiteSpace=wrap;html=1;aspect=fixed;&quot; value=&quot;&amp;lt;font style=&amp;quot;font-size: 25px;&amp;quot;&amp;gt;+&amp;lt;/font&amp;gt;&quot; vertex=&quot;1&quot;&gt;&#xA;          &lt;mxGeometry height=&quot;60&quot; width=&quot;60&quot; x=&quot;440&quot; y=&quot;520&quot; as=&quot;geometry&quot; /&gt;&#xA;        &lt;/mxCell&gt;&#xA;        &lt;mxCell id=&quot;BxGZwdInxDfRHZgYTuuF-11&quot; edge=&quot;1&quot; parent=&quot;1&quot; source=&quot;BxGZwdInxDfRHZgYTuuF-7&quot; style=&quot;endArrow=none;html=1;rounded=0;curved=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;exitX=0.5;exitY=1;exitDx=0;exitDy=0;&quot; target=&quot;BxGZwdInxDfRHZgYTuuF-3&quot; value=&quot;&quot;&gt;&#xA;          &lt;mxGeometry height=&quot;50&quot; relative=&quot;1&quot; width=&quot;50&quot; as=&quot;geometry&quot;&gt;&#xA;     Line truncated
Author	SHA1	Message	Date
eneller	f8013bcee5	update	2025-10-30 15:47:54 +01:00
eneller	358274979f	update	2025-10-30 14:51:12 +01:00
eneller	3eb0f229fd	bib	2025-10-30 13:20:34 +01:00
eneller	90aa504539	entropy sources	2025-10-29 22:31:42 +01:00
eneller	d1b45ee1ec	begin entropy examples	2025-10-29 17:21:03 +01:00
eneller	b1989ee151	general entropy	2025-10-28 13:26:00 +01:00
eneller	4fc4346d62	Initial Commit	2025-10-27 10:42:33 +01:00