\documentclass[12pt]{article}
\usepackage[a4paper,top=20mm,bottom=20mm,left=20mm,right=20mm]{geometry}
\usepackage{url}
\usepackage{alltt}
\usepackage{xspace}
\usepackage{times}
\usepackage{listings}
\usepackage{bbm}
\usepackage{verbatim}
\usepackage{prognames}
\usepackage{optionman}
\usepackage{skaff}
\newcommand{\VMERSTAT}[0]{\Programname{vmerstat}\xspace}
\newcommand{\VMERSEARCH}[0]{\Programname{vmersearch}\xspace}
\newcommand{\VMERDIST}[0]{\Programname{vmerdistrib}\xspace}
\newcommand{\Kmin}[0]{k_{\min}}
\newcommand{\Kmax}[0]{k_{\max}}
\newcommand{\Kstep}[0]{k_{step}}
\title{Counting, Indexing, and Searching $k$-mers\\
a manual}
\author{Stefan Kurtz}
\author{\begin{tabular}{c}
         \emph{Stefan Kurtz}\\
         Center for Bioinformatics,\\
         University of Hamburg
        \end{tabular}}

\begin{document}
\maketitle
\section{\VMERSTAT}

The program \VMERSTAT is called as follows:
\par
\noindent\VMERSTAT [\emph{options}] \emph{mkvtreeidx0} \emph{mkvtreeidx1} \dots
\par
where \emph{mkvtreeidx0}, \emph{mkvtreeidx1}, \dots are names of 
indices computed by \MKV using the output options \Showoption{suf},
\Showoption{lcp}, and \Showoption{tis}. The following options are available
in \VMERSTAT:

\begin{Justshowoptions}
\Option{mersize}{$k$}{
Specify the size $k$ of the mers. That is, the program generates all
substrings of length $k$ of the given input sequences, given as
\MKV-indices. If this option is missing, then the default value 
for $k$ is 20.
}

\Option{minocc}{$c$}{
Specify the minimum occurrence number for which to output the $k$-mer
sequences. That is, a $k$-mer is output, if it
occurs at least $c$ times in the union of all sequences from the
\MKV-indices. When combined with option \Showoption{indexname},
this option specifies the $k$-mers stored
in the index generated by \VMERSTAT. That is, a $k$-mer is put into the
\VMERSTAT-index, if it occurs at least $c$ times in the union of 
all sequences from the \MKV-indices.
}

\Option{maxocc}{$c$}{
Specify the maximum occurrence number for which to output the $k$-mer
sequences. That is, a $k$-mer is output, if it
occurs not more than $c$ times in the union of all sequences from the
\MKV-indices. When combined with option \Showoption{indexname},
this option specifies the $k$-mers stored
in the index generated by \VMERSTAT. That is, a $k$-mer is put into the
\VMERSTAT-index, if it occurs not more than $c$ times in the union of all 
sequences from the \MKV-indices.
}

\Option{pl}{$\Showoptionalarg{prefixlength}$}{
Specify the prefix length to construct a bucket boundary table
for the generated \VMERSTAT-index. This additional table speeds up the 
search in the \VMERSTAT-index. This option only works for an alphabet
of size 4, i.e.\ for the DNA alphabet.
The argument \Showoptionarg{prefixlength} is
optional. Hence it is denoted in square brackets.
If the argument is omitted, then the value
for \Showoptionarg{prefixlength} is automatically determined. More precisely,
it is \(\left\lfloor\log_{4}\frac{n}{4}\right\rfloor\),
where \(n\) is the total number of $k$-mers in the \VMERSTAT-index.
}

\Option{indexname}{\Showoptionarg{idxname}}{
Specify the name of the index storing the $k$-mers specified according to
the options \Showoption{s}. The \VMERSTAT-index is stored in
file \Showoptionarg{idxname}\Filenamesuffix{.mer}. If option 
\Showoption{pl} is used, then additionally the bucket boundary table is 
stored in a file named \Showoptionarg{idxname}\Filenamesuffix{.mbd}.
}

\Option{counts}{}{
Specify that besides the mer-index a file
\Showoptionarg{idxname}\Filenamesuffix{.mct}
is generated storing the
counts of the mers represented by the mer-index. This option
can only be used together with option \Showoption{indexname}.
This option is required if the program \VMERSEARCH needs to 
report the mer-counts.
}

\Option{tmppref}{\Showoptionarg{tmpfilenameprefix}}{
Specify the prefix of the name of the temporary files which are
used to store intermediate results when handling more than
one index. In the default case, the prefix is 
\texttt{/tmp/Vmerstat}. The temporary files are removed by the program,
as soon as they are no longer required. In case the program execution files
(e.g.\ if not enough memory is available), the user has to remove these
files.}

\Option{v}{~~~}{%
Be verbose, that is, give reports about the different steps as well as the
resource requirements of the computation. This option is recommended.
}

\Option{version}{}{
Show the version of the program. Also report the compilation date and 
the compilation options.
}

\Helpoption

\end{Justshowoptions}
The following conditions must be satisfied:
\begin{enumerate}
\item
The number of \MKV-indices given as arguments to \VMERSTAT is between
1 and 128. If you want to handle more than 128 indices, please contact
me.
\item
For the 32-bit version of the program, the following limits are important:
\begin{itemize}
\item
the total length of the sequence in each \MKV-index must not exceed 
536870911.
\item
the maximum occurrence count handled by the program is 268435455.
\end{itemize}
For the 64-bit version of the program there are virtually no
such limits.
\item
Option \Showoption{pl} requires to also use option \Showoption{indexname}.
\item
Option \Showoption{indexname} requires to also use one of the options
options \Showoption{minocc} and \Showoption{maxocc}.
\end{enumerate}
Note that the program ignores all $k$-mers not entirely consisting of
wildcard characters (i.e. not \texttt{a}, \texttt{c}, \texttt{c}, and 
\texttt{g} in case of the DNA alphabet).

\section{\VMERDIST}

The program \VMERDIST is called as follows:
\par
\noindent\VMERDIST [\emph{options}] \emph{mkvtreeidx0} \emph{mkvtreeidx1} \dots
\par
where \emph{mkvtreeidx0}, \emph{mkvtreeidx1}, \dots are names of 
indices computed by \MKV using the output options \Showoption{suf},
\Showoption{lcp}, and \Showoption{tis}. The following options are available
in \VMERSTAT:

\begin{Justshowoptions}
\Option{mersizes}{$k_{1}~k_{2}\ldots k_{q}$}{
Specify mer sizes $1\leq k_{1}<k_{2}<\cdots<k_{q}$ with $q\geq 1$.}

\Option{minmersize}{$\Kmin$}{
Specify the minimum size of the mers which are counted. 
That is, the program counts the number of unique and nonunique 
mers of length at least $\Kmin$. This option is mandatory if option 
\Showoption{mersizes} is not used.}

\Option{maxmersize}{$\Kmax$}{
Specify the maximum size of the mers which are counted. 
That is, the program counts the number of unique and nonunique 
mers of length at most $\Kmax$. This option is mandatory if option 
\Showoption{mersizes} is not used.}

\Option{step}{$\Kstep$}{
Specify the step width according to which the mer counts are output.
That is, for all $k\in[\Kmin,\Kmin+\Kstep,\Kmin+2\Kstep,\ldots,\Kmax]$
the $k$-mer counts are ouput. If this option is not used, then
$\Kstep$ is 1. This option cannot be used together with option
\Showoption{mersizes}.}

\Option{output}{(\Showoptionkey{unique}$\mid$\Showoptionkey{nonunique}$\mid$\Showoptionkey{nonuniquemulti}$\mid$\Showoptionkey{relative}$\mid$\Showoptionkey{total})}{
Specify what to output by giving at least one of the four keywords
\begin{center}
$\Showoptionkey{unique}$,
$\Showoptionkey{nonunique}$,
$\Showoptionkey{nonuniquemulti}$,
$\Showoptionkey{relative}$, and 
$\Showoptionkey{total}$.
\end{center}
\begin{itemize}
\item
Using the keyword $\Showoptionkey{unique}$, shows the number of unique
$k$-mers for each $k$ between $\Kmin$ and $\Kmax$.
\item
Using the keyword $\Showoptionkey{nonunique}$, shows the number of non-unique
$k$-mers for each $k$ between $\Kmin$ and $\Kmax$. Only the event that
a $k$-mer is not unique is counted.
\item
Using the keyword $\Showoptionkey{nonuniquemulti}$, shows the number of 
non-unique $k$-mers for each $k$ between $\Kmin$ and $\Kmax$. 
Each $k$-mer is counted as the number of times it occurs in the indexed 
sequences.
\item
Using the keyword $\Showoptionkey{total}$, shows the number of all
$k$-mers for each $k$ between $\Kmin$ and $\Kmax$. The distribution
is shown twice, once counting each non-unique $k$-mers as one event,
and once counting each non-unique $k$-mer as the number of times it occurs in
the indexed sequences.
\item
The keyword $\Showoptionkey{relative}$ can be combined with the keywords
$\Showoptionkey{unique}$, $\Showoptionkey{nonunique}$, and 
$\Showoptionkey{nonuniquemulti}$. Its use modifies the output such that 
besides the number also the fraction of unique/non-unique $k$-mers relative 
to all $k$-mers is shown.
\end{itemize}
}

\end{Justshowoptions}

\section{\VMERSEARCH}
The program \VMERSEARCH is called as follows:
\par
\noindent\VMERSEARCH [\emph{options}] \emph{vmerstat-indexname} \emph{queryfile0}
\emph{queryfile1} \dots
\par
where \emph{vmerstat-indexname} is an index generated by \VMERSTAT, and
\emph{queryfile0}, \emph{queryfile1}, etc.\ are queryfiles which are to be
matches against the given index. The following options are available

\begin{Justshowoptions}
\Option{strand}{(\Showoptionkey{f}$\mid$\Showoptionkey{p}$\mid$\Showoptionkey{fp})}{
Specify the strand to be searched. The keyword \Showoptionkey{f} means to
search on the forward strand, i.e.\ each mer is searched in forward
direction. The keyword \Showoptionkey{p} means to
search on the reverse complemented strand, i.e.\ the reverse complement
of the given mer is searched. The keyword \Showoptionkey{fp} means a 
combination of \Showoptionkey{f} and \Showoptionkey{p}.
}

\Option{output}{(\Showoptionkey{qseqnum}$\mid$\Showoptionkey{qpos}$\mid$\Showoptionkey{counts}$\mid$\Showoptionkey{sequence}}{
Specify what to output by giving at least one of the four keywords
\begin{center}
$\Showoptionkey{qseqnum}$,
$\Showoptionkey{qpos}$,
$\Showoptionkey{counts}$,
$\Showoptionkey{sequence}$.
\end{center}
\begin{itemize}
\item
Using the keyword $\Showoptionkey{qseqnum}$, shows the sequence number
of the query sequence, the matching mer comes from.
\item
Using the keyword $\Showoptionkey{qpos}$ shows the relative position of
the matching mer. The symbol \texttt{+} in from the position signifies
a match on the forward strand, while the symbol \texttt{-} signifies
a match on the reverse strand.
\item
Using the keyword $\Showoptionkey{counts}$ shows the counts of the
mer, i.e.\ the number of times, the mer occurs in the indexed
sequences.
\item
Using the keyword $\Showoptionkey{sequence}$ shows the sequence content
of the mer.
\end{itemize}
For each matching mer, the mentioned values are output on a single line
in the order the four keywords are given. Two consecutve values 
are separated by white spaces.
}

\Option{v}{~~~}{%
Be verbose, that is, give reports about the different steps as well as the
resource requirements of the computation. This option is recommended.
}

\Option{version}{}{
Show the version of the program. Also report the compilation date and 
the compilation options.
}

\Helpoption

\end{Justshowoptions}

\section{Examples}

Suppose we have a collection of two files \texttt{read1.fna} and
\texttt{read2.fna}. In the first step, we index 
both files separately using the program \MKV:

\EXECUTE{mkvtree -dna -pl -tis -suf -lcp -v -db read1.fna -indexname read1}

\EXECUTE{mkvtree -dna -pl -tis -suf -lcp -v -db read2.fna -indexname read2}

We get the two indices \texttt{read1} and \texttt{read2}.

These are used in the following call to the program \VMERDIST:

\EXECUTE{vmerdistrib -output unique nonunique -minmersize 10 -maxmersize 20 read1 read2}

This shows the counts of $k$-mers for $k\in[10,20]$. The first part of the
output reports counts of unique $k$-mers, while the second is for
non-unique $k$-mers. For example, there 223755 unique $10$-mers 
and 135526 non-unique $10$-mers. If we add the keyword 
\Showoptionkey{relative}, then we additionally obtain the fraction
of counts relative to the total number of $k$-mers:

\EXECUTE{vmerdistrib -output unique relative -minmersize 10 -maxmersize 20 read1 read2}

For example, we see that $62.3=\frac{223755}{223755+135526}\cdot 100$
percent of all 10-mers are unique. To restrict to specific mer sizes, for
example 10, 13, and 17, we can use option \Showoption{mersizes}:

\EXECUTE{vmerdistrib -output unique nonunique -mersizes 10 13 17 read1 read2}

While \VMERDIST can compute distributions for a range of 
$k$-mers, \VMERSTAT runs for a fixed mer-size, as in the following example:

\EXECUTE{vmerstat -mersize 19 -minocc 40 read1 read2}

The output, as explained at the beginning of the output, shows the
distribution of occurrences of 19-mers in the indexes
\texttt{read1} and \texttt{read2}. The 19-mers occurring more 
than 40 times are reported with their string content.
We now add options \Showoption{indexname} and \Showoption{counts} to 
generate a 19-mer index called \texttt{reads}. The index contains
information to show the counts.

\EXECUTE{vmerstat -mersize 19 -minocc 4 -indexname reads -counts -pl read1 read2}

This generates the 19-mer index file \texttt{reads.mer} and an additional
table with bucket boundaries stored in file \texttt{reads.mbd}.

The program \VMERSEARCH now uses the index \texttt{reads} and
matches all 19-mers of the input sequence \texttt{U89959} against it:

\EXECUTE{vmersearch -output qseqnum  qpos counts sequence reads U89959.fna | head -25}

Each line of the output not beginning with the symbol \texttt{\symbol{35}}
consist of fpur columns: The first column shows the ordinal number of the
sequence in the query file containing the match match. The second number
is the offset in the sequence (counting from 0) whose number is given.
The number is prefixed by the symbol \texttt{+} if the given mer
matches on the forward strand.
The number is prefixed by the symbol \texttt{-} if the given mer
matches on the reverse complemeneted strand. The third column shows
the occurrence count for the mer in the index. There are separated 
counts for the matches on the forward and the reverse complemented strand.
The fourth column shows the mer in forward direction. 

\section{TODO-list}
\begin{enumerate}
\item
the current version of \VMERSTAT and \VMERSEARCH is optimized for short
mer-sizes (i.e.\ up to length 32 for the DNA alphabet). Longer mer-sizes 
lead to large space overheads, and should thus be handled in a different way. 
This has to be implemented.
\item
for more than one query file, the program \VMERSEARCH should add the number
of the query file to the output.
\item
add function for selecting statistically relevant mers.
\item
process each query sequence, before inputting the next query sequence.
\begin{comment}
\item
check if all indexes have an alphabet of the same size
\item
add linear scan over mers to construct bucket boundaries.
This is more efficient than binary partitioning in case the 
bucket boundaries become small.
\item
change option version such that it does not refer to option Vmatch.
\item
preprocess query sequences by sorting them according to some
prefixlength. Then for the query-substrings beginning with the
same prefix, find the prefix of the given length once and 
reuse this information several times.
\end{comment}
\end{enumerate}

\end{document}
