pax_global_header00006660000000000000000000000064152413646640014525gustar00rootroot0000000000000052 comment=e27bc439cf110ac8cca89fcecacda993d2c9df70 jltsiren-gbwtgraph-e27bc43/000077500000000000000000000000001524136466400157555ustar00rootroot00000000000000jltsiren-gbwtgraph-e27bc43/.gitignore000066400000000000000000000005171524136466400177500ustar00rootroot00000000000000# Dependencies, objects, and libraries *.d *.o *.a # Build directories bin lib obj build # Tools gfa2gbwt # Tests test_algorithms test_cached_gbwtgraph test_gbwtgraph test_gbz test_gfa test_index test_minimizer test_naive_graph test_path_cover test_subgraph test_utils # Mac .DS_Store # emacs *~ # vim \#* \.#* # VSCode .vscode jltsiren-gbwtgraph-e27bc43/.gitmodules000066400000000000000000000000001524136466400201200ustar00rootroot00000000000000jltsiren-gbwtgraph-e27bc43/LICENSE000066400000000000000000000021331524136466400167610ustar00rootroot00000000000000Copyright (c) 2019, 2020, 2021, 2022, 2023, 2024, 2025, 2026 Jouni Siren and other authors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. jltsiren-gbwtgraph-e27bc43/Makefile000066400000000000000000000064601524136466400174230ustar00rootroot00000000000000SDSL_DIR=../sdsl-lite include $(SDSL_DIR)/Make.helper BUILD_BIN=bin BUILD_LIB=lib BUILD_OBJ=obj SOURCE_DIR=src # Multithreading with OpenMP. PARALLEL_FLAGS=-fopenmp -pthread # Directories for dependencies. INCLUDES=-Iinclude -I$(INC_DIR) LIBS=-L$(LIB_DIR) -lgbwt -lhandlegraph -lsdsl # Use pkg-config to find system dependencies. ifeq ($(shell pkg-config --exists libcrypto libzstd && echo 1), 1) $(info Found libcrypto and libzstd.) INCLUDES += $(shell pkg-config --cflags libcrypto libzstd) LIBS += $(shell pkg-config --libs libcrypto libzstd) else $(error Could not find libcrypto or libzstd. Please update PKG_CONFIG_PATH) endif # Apple Clang does not support OpenMP directly, so we need special handling. ifeq ($(shell uname -s), Darwin) # The compiler complains about -fopenmp instead of missing input. ifeq ($(strip $(shell $(MY_CXX) -fopenmp /dev/null -o/dev/null 2>&1 | grep fopenmp | wc -l)), 1) $(info The compiler is Apple Clang that needs libomp for OpenMP support.) # The compiler only needs to do the preprocessing. PARALLEL_FLAGS=-Xpreprocessor -fopenmp -pthread # Find libomp installed by Homebrew or MacPorts. ifeq ($(shell if [ -e $(HOMEBREW_PREFIX)/include/omp.h ]; then echo 1; else echo 0; fi), 1) $(info Found libomp installed by Homebrew and linked to $(HOMEBREW_PREFIX).) INCLUDES += -I$(HOMEBREW_PREFIX)/include LIBS += -L$(HOMEBREW_PREFIX)/lib else ifeq ($(shell if [ -d $(HOMEBREW_PREFIX)/opt/libomp/include ]; then echo 1; else echo 0; fi), 1) $(info Found a keg-only libomp installed by Homebrew at $(HOMEBREW_PREFIX)/opt/libomp.) INCLUDES += -I$(HOMEBREW_PREFIX)/opt/libomp/include LIBS += -L$(HOMEBREW_PREFIX)/opt/libomp/lib else ifeq ($(shell if [ -d /opt/local/lib/libomp ]; then echo 1; else echo 0; fi), 1) $(info Found libomp installed by MacPorts at /opt/local.) INCLUDES += -I/opt/local/include/libomp LIBS += -L/opt/local/lib/libomp else $(error Could not find libomp. Please install it using Homebrew or MacPorts) endif # We also need to link it. LIBS += -lomp endif endif CXX_FLAGS=$(MY_CXX_FLAGS) $(PARALLEL_FLAGS) $(MY_CXX_OPT_FLAGS) $(INCLUDES) HEADERS=$(wildcard include/gbwtgraph/*.h) LIBOBJS=$(addprefix $(BUILD_OBJ)/,algorithms.o cached_gbwtgraph.o gbwtgraph.o gbz.o gfa.o index.o internal.o minimizer.o naive_graph.o path_cover.o subgraph.o utils.o) LIBRARY=$(BUILD_LIB)/libgbwtgraph.a PROGRAMS=$(addprefix $(BUILD_BIN)/,canonical_gfa chunk_gbz gfa2gbwt gbz_extract gbz_find gbz_stats kmer_freq subgraph_query) OBSOLETE=gfa2gbwt .PHONY: all clean directories test all: directories $(LIBRARY) $(PROGRAMS) directories: $(BUILD_BIN) $(BUILD_LIB) $(BUILD_OBJ) $(BUILD_BIN): mkdir -p $@ $(BUILD_LIB): mkdir -p $@ $(BUILD_OBJ): mkdir -p $@ $(BUILD_OBJ)/%.o:$(SOURCE_DIR)/%.cpp $(HEADERS) $(MY_CXX) $(CPPFLAGS) $(CXXFLAGS) $(CXX_FLAGS) -c -o $@ $< $(LIBRARY):$(LIBOBJS) ar rcs $@ $(LIBOBJS) $(BUILD_BIN)/%:$(BUILD_OBJ)/%.o $(LIBRARY) $(MY_CXX) $(LDFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(CXX_FLAGS) -o $@ $< $(LIBRARY) $(LIBS) test:$(LIBRARY) cd tests && $(MAKE) test clean: rm -rf $(BUILD_BIN) $(BUILD_LIB) $(BUILD_OBJ) rm -f *.o *.a $(OBSOLETE) cd tests && $(MAKE) clean jltsiren-gbwtgraph-e27bc43/README.md000066400000000000000000000224771524136466400172500ustar00rootroot00000000000000# GBWTGraph GBWTGraph is a [handle graph](https://github.com/vgteam/libhandlegraph) based on the [GBWT](https://github.com/jltsiren/gbwt). Its data model is based on the graph as an alignment of haplotypes. The `gfa2gbwt` tool can be used for converting between a subset of [GFA1](https://github.com/GFA-spec/GFA-spec/blob/master/GFA1.md), the GBZ file format, and GBWTGraph. See [the wiki](https://github.com/jltsiren/gbwtgraph/wiki) for further documentation. There is also a partial [Rust implementation](https://github.com/jltsiren/gbwt-rs) of the GBWTGraph. ## Overview The GBWTGraph represents the graph induced by the haplotypes stored in a GBWT index. It uses the GBWT index for graph topology and stores the node sequences in plain form for fast extraction. Construction extracts the sequences from another graph implementing `handlegraph::HandleGraph` or from `gbwtgraph::SequenceSource`. GBWTGraph supports the following libhandlegraph interfaces: * `HandleGraph` * `PathHandleGraph` * `NamedNodeBackTranslation` * `SerializableHandleGraph` (the GBWT index must be serialized/deserialized separately) Compared to other handle graph implementations, sequence access is very fast, while graph navigation may be slower. There are also some additional operations: * `get_sequence_view()` provides direct access to node sequences without decompression, reverse complementation, or memory allocation. * `follow_paths()` is an analogue of `follow_edges()` using GBWT search states instead of handles. It only follows edges if the resulting path is supported by the haplotypes in the index. * `simple_sds_serialize()` and `simple_sds_load()` offer a more space-efficient serialization alternative. Accessing and decompressing GBWT node records is somewhat slow. Algorithms that repeatedly access the edges in a small subgraph may create a `CachedGBWT` cache using `get_cache()` and pass it explicitly to the relevant queries. Alternatively, they can create a `CachedGBWTGraph` overlay graph that uses a cache automatically. Both types of caches store all accessed records, so a new cache should be created for each subgraph. GBWTGraph also supports an experimental `SegmentHandleGraph` interface with GFA-like semantics. Each GFA segment with a string name maps to a range of node ids, and GFA links correspond to edges that connect the ends of segments. This interface is currently only available in graphs built using `SequenceSource`. The package also includes: * [GBZ file format](https://github.com/jltsiren/gbwtgraph/blob/master/SERIALIZATION.md). * GBWT / GBWTGraph construction from a subset of GFA1, and GFA extraction from a GBWTGraph. * GBWT construction from a greedy path cover: * Artificial paths that try to cover all length-k contexts equally, either in the entire graph or only in components that do not already contain paths. * Concatenations of local length-k haplotypes sampled according to their true frequencies. * Tagging graphs with [pggname](https://github.com/jltsiren/pggname) names using `GraphName`: * Each GBWTGraph has a stable name based on hashing the canonical GFA representation. * Subgraph relationships and coordinate translations can be stored in GBZ tags, GFA headers, and GAF headers. * Support for paralellizing GBWT construction over weakly connected components of the graph: * `gbwt_construction_jobs()` for determining the construction jobs. * `assign_paths()` and `insert_paths()` for passing reference paths to the new GBWT. * `MetadataBuilder` for generating GBWT metadata. * Tools for extracting per-chromosome graphs and merging them: * Chunk graphs by weakly connected components and extract chunks by contig names using `chunk_graph()`. * Merging non-overlapping subgraphs with the GBZ merge constructor. * A generic kmer index and a generic minimizer index for indexing the haplotypes in the GBWTGraph. ## Construction from GFA The `gfa2gbwt` tool can be used for building GBWTGraph from GFA1, for extracting GFA from the graph, and for converting between plain and compressed representations of GBWTGraph. The tool interprets the GFA file in the following way: * Overlaps, containments, and tags are ignored. * Segments and links are inferred from the paths; the corresponding S-lines and L-lines must still exist in the file. * W-lines and/or Z-lines are the primary representation of haplotype paths. * If there are P-lines in the file, GBWT metadata can be parsed by providing regexes that map submatches to metadata fields. In the plain representation, the GBWT index and the GBWTGraph are stored in separate `.gbwt` and `.gg` files. The compressed representation uses a single `.gbz` file, with the graph stored more space-efficiently than the in-memory representation. ## Sequence extraction Sequences corresponding to paths can be extracted from a GBZ graph using the `gbz_extract` tool. The tool extracts sequences using multiple threads and writes them to the standard output. Each sequence is terminated by an endline character (`'\n`), making this tool suitable for multi-string BWT construction using tools such as [grlBWT](https://github.com/ddiazdom/grlBWT). The extracted sequences are written in the same order the paths appear in the GBWT. By default, the line number in the output (which becomes the sequence identifier in the multi-string BWT) is the same as the path identifier in the GBWT. If option `-b` / `--both-orientations` is used, each sequence is followed by the reverse complement of the same sequence. This can be used for building an FMD-index. When reverse complements are included, the line number is the same as the GBWT sequence identifier. Instead of extracting all sequences, it is possible to limit the extraction to the weakly connected component(s) corresponding to a contig name (e.g. `chr19`). This can be done using option `-c` / `--contig`. When this option is used, the sequences will maintain the same relative order as the corresponding paths in the GBWT. The number of extraction threads can be changed using option `-t` / `--threads`. In addition to the extraction threads, there will be a main thread responsible for writing the sequences to the output. The extraction process is often I/O bound, and it is unlikely that more than a few threads will be active at the same time. ## Dependencies All dependencies should be installed before compiling GBWTGraph. By default, libhandlegraph installs to system directories, while GBWT and SDSL install to the user's home directory. Dependencies not installed in system directories should use the same install prefix as SDSL. ### System dependencies * [libcrypto](https://docs.openssl.org/3.0/man7/crypto/) from [OpenSSL](https://www.openssl.org/) for computing [graph names](https://github.com/jltsiren/pggname). * [libzstd](https://github.com/facebook/zstd) for compressing serialized `RecordArray`s and `StringArray`s. These dependencies are found using `pkg-config`. Update `PKG_CONFIG_PATH` as necessary. ### GitHub repositories * [GoogleTest](https://github.com/google/googletest) for the tests. * [libhandlegraph](https://github.com/vgteam/libhandlegraph) for the handle graph interface. * [GBWT](https://github.com/jltsiren/gbwt) for the backend. * [SDSL](https://github.com/vgteam/sdsl-lite) (vgteam fork) for low-level data structures. These dependencies should be installed separately (the latest master should always work). Because libhandlegraph and SDSL are header-based libraries, having multiple versions of them in the same project may cause issues. Hence all submodules of the main project should use the same copies of these libraries. ## Compiling GBWTGraph This library is designed to take the compiler options from the [vgteam fork](https://github.com/vgteam/sdsl-lite) of the Succinct Data Structures Library 2.0 (SDSL). It currently requires a recent C++ compiler supporting C++17 and OpenMP. GCC is recommended, as the multithreaded `std::sort` from libstdc++ parallel mode speeds up some algorithms. On Apple systems, GBWTGraph can be built with Apple Clang, but libomp must be installed via Macports or Homebrew. GBWTGraph is frequently tested in the following environments: * Intel Linux (Ubuntu) with GCC. * Intel macOS with GCC and Apple Clang. * ARM macOS with Apple Clang. Before compiling, you must set `SDSL_DIR` in the makefile to your SDSL main directory. The default value is `../sdsl-lite`, which is usually appropriate. The makefile will read `$SDSL_DIR/Make.helper` to determine compilers and compiler options. After that, `make` will compile the library, while `install.sh` will compile and install the headers and the library to your home directory. Another install directory can be specified with `install.sh prefix`. ## Citing GBWTGraph Jouni Sirén, Jean Monlong, Xian Chang, Adam M. Novak, Jordan M. Eizenga, Charles Markello, Jonas A. Sibbesen, Glenn Hickey, Pi-Chuan Chang, Andrew Carroll, Namrata Gupta, Stacey Gabriel, Thomas W. Blackwell, Aakrosh Ratan, Kent D. Taylor, Stephen S. Rich, Jerome I. Rotter, David Haussler, Erik Garrison, and Benedict Paten: **Pangenomics enables genotyping of known structural variants in 5202 diverse genomes**. Science 374(6574):abg8871, 2021. DOI: [10.1126/science.abg8871](https://doi.org/10.1126/science.abg8871) ## Citing GBZ File Format Jouni Sirén and Benedict Paten: **GBZ file format for pangenome graphs**. Bioinformatics 38(22):5012-5018, 2022. DOI: [10.1093/bioinformatics/btac656](https://doi.org/10.1093/bioinformatics/btac656) jltsiren-gbwtgraph-e27bc43/SERIALIZATION.md000066400000000000000000000157301524136466400203220ustar00rootroot00000000000000# GBZ file format GBZ version 3, GBWTGraph version 4. Updated 2026-08-13. Based on Simple-SDS version 0.4.0, GBWT version 6, and Metadata version 2. |**GBZ version**|**GBWT version**|**GBWTGraph version**| |:-------------:|:--------------:|:-------------------:| |3 |6 |4 | |2 |5 |4 | |1 |5 |3 | ## Basics This document specifies the GBZ file format for storing GFA with many paths. It includes a portable Simple-SDS serialization format for GBWTGraph as an alternative to the old SDSL-based format. The format builds upon the data structures described in: * * The notation uses Simple-SDS conventions. Array indexes start at 0. Rank queries count the number of occurrences strictly before the query position, as in SDSL. A rank query past the end of the sequence counts the total number of occurrences in the sequence. Select queries use 0-based indexing (SDSL uses 1-based indexing). A past-the-end select query should be understood as the length of the sequence, though this is usually clarified later. ## GBZ **GBZ** is a space-efficient binary format for a subset of [GFA1](https://github.com/GFA-spec/GFA-spec/blob/master/GFA1.md) with many paths. It ignores optional GFA fields and assumes that there are no overlaps or containments. The format builds upon the Simple-SDS formats for serializing the GBWT and the GBWTGraph. GBZ file format: 1. GBZ header 2. Tags 3. GBWT 4. GBWTGraph **Tags** are stored in the same format as in the GBWT. Key `source` indicates the implementation that generated the file. The original implementation corresponds to value `jltsiren/gbwtgraph`. ### GBZ header **GBZ header** is a 16-byte (2-element) structure with the following fields: 1. `tag`: Value `0x205A4247` as a 32-bit integer (corresponds to string `GBZ `). 2. `version`: File format version as a 32-bit integer. 3. `flags`: Binary flags as an element. The first two fields are 32-bit unsigned little-endian integers for consistency with the other headers. Flags are currently unused. ### GBWT / GFA paths If the GBWT does not contain metadata with path names, each original path in the GBWT corresponds to a P-line in GFA. The integer identifier of each original path is used as the `PathName` field. Otherwise the original paths for sample `_gbwt_ref` are assumed to be reference paths that correspond to GFA P-lines. Each reference path must have a different contig identifier in the GBWT path name. The corresponding contig name is used as the `PathName` field. Paths for all other samples are assumed to be haplotypes that correspond to GFA W-lines. GBWT path name fields map to GFA W-line fields in the following way: * The sample name corresponding to `sample` maps to `SampleId`. * The contig name corresponding to `contig` maps to `SeqId`. * `phase` maps to `HapIndex`. * `fragment` maps to `SeqStart` (`SeqEnd` can be derived from `fragment` and the path itself). If the GBWT metadata does not contain sample (contig) names, the integer identifier of the sample (contig) is used instead. **Note:** The reference paths mentioned here correspond to generic named paths in VG terminology. VG promotes some samples to reference status by setting GBWT tag `reference_samples` with a space-separated list of sample names (e.g. `GRCh38 CHM13`) as the value. In GFA format, it encodes the same information in header tag `RS` (e.g. `RS:Z:GRCh38 CHM13`). ## GBWTGraph **GBWTGraph** represents a bidirected sequence graph induced by a set of paths. It uses a GBWT index for graph topology and for storing the paths. The GBWT must be bidirectional. Serialization format for GBWTGraph: 1. GBWTGraph header 2. Sequences 3. Node-to-segment translation ### GBWTGraph header **GBWTGraph header** is a 24-byte (3-element) structure with the following fields: 1. `tag`: Value `0x6B3764AF` as a 32-bit integer. 2. `version`: File format version as a 32-bit integer. 3. `nodes`: Number of nodes in the graph as an element. 4. `flags`: Binary flags as an element. The first two fields are 32-bit unsigned little-endian integers for compatibility with the SDSL-based serialization format. Simple-SDS serialization format requires file format version `3` or later. Field `nodes` counts the number of original nodes **present** in the graph. A node is present if the local alphabet size in the corresponding GBWT nodes is nonzero. As the GBWT stores a subgraph induced by the paths, this is equivalent to requiring that the original node is visited by a path. The following flags are supported: * `0x0001`: The translation structure is present. * `0x0002`: Simple-SDS format. Other flag bits must not be set. The translation flag must be set if and only if the translation structure is nonempty. If the Simple-SDS bit is not set, the serialized data is in the SDSL format. ### Sequences The **sequences** structure attaches a string label to every original node present in the graph. Serialization format for sequences: 1. `sequences`: Node labels as a compressed string array. The label of original node `v` is string `v - floor(offset / 2) - 1` in the string array, where `offset` is the alphabet offset in the GBWT. This usually means that the nodes map to consecutive identifiers starting from `0`. **Note:** If a node is not present in the graph, the corresponding string should be empty. While this is not a strict requirement, it is a good practice that makes the serialization deterministic. **Note:** `sequences` was stored as a string array until GBZ version 1 and GBWTGraph version 3. ### Node-to-segment translation The GBWT uses nodes with integer identifiers, while the [GFA specification](https://github.com/GFA-spec/GFA-spec/blob/master/GFA1.md) uses **segments** with string names. GBWTGraph needs to store a **translation** between node identifiers and string names if the names cannot be interpreted as integers. Because some applications expect nodes with short sequences, the translation also supports mapping segments to ranges of nodes. Serialization format for translation: 1. `segments`: Segment names as a string array. 2. `mapping`: First node identifier mapping to each segment as a sparse vector. If the translation is not in use, both structures must be empty. Otherwise the original nodes must have consecutive identifiers starting from `1`. The segment with the `i`th string as its name then corresponds to the concatenation of nodes from `mapping.select(i)` (inclusive) to `mapping.select(i + 1)` (exclusive). **Note:** Because the translation is a core part of the GBWTGraph, it is not serialized as an optional structure. **Note:** For the last segment, the end of the node interval is the length of `mapping`. **Note:** Some segments may consist of nodes that are not present in the graph. As with sequences, such segments should have empty names to make the serialization deterministic. jltsiren-gbwtgraph-e27bc43/include/000077500000000000000000000000001524136466400174005ustar00rootroot00000000000000jltsiren-gbwtgraph-e27bc43/include/gbwtgraph/000077500000000000000000000000001524136466400213655ustar00rootroot00000000000000jltsiren-gbwtgraph-e27bc43/include/gbwtgraph/algorithms.h000066400000000000000000000202211524136466400237040ustar00rootroot00000000000000#ifndef GBWTGRAPH_ALGORITHMS_H #define GBWTGRAPH_ALGORITHMS_H #include #include #include #include #include /* algorithms.h: Various graph algorithms. */ namespace gbwtgraph { //------------------------------------------------------------------------------ /* Return the weakly connected components in the graph. The components are sorted by the minimum node id, and the node ids in each component are also sorted. */ std::vector> weakly_connected_components(const HandleGraph& graph); /* Determine whether the given component is acyclic in a nice way. In such graphs, when we start from nodes with indegree 0 in forward orientation, we reach each node in a single orientation and find no cycles. Return the head nodes when the component passes the tests or an empty vector otherwise. Ignores node ids that are not present in the graph. */ std::vector is_nice_and_acyclic(const HandleGraph& graph, const std::vector& component); /* Return a topological order of handles in the subgraph induced by the given node ids, or an empty vector if no such order exists. NOTE: Node ids that do not exist in the graph are ignored. In particular, some nodes of the original graph may be missing from the corresponding GBWTGraph if no path passes through them. If the subgraph is small, it may be a good idea to use CachedGBWTGraph instead of GBWTGraph. */ std::vector topological_order(const HandleGraph& graph, const std::unordered_set& subgraph); //------------------------------------------------------------------------------ // Parameters for `chunk_graph()`. struct ChunkParameters { // If not empty, output only the chunk(s) containing this contig name. std::string contig_name; // Print progress information during chunking. bool verbose = false; }; /* Partition the graph into chunks based on the weakly connected components. Returns a GBZ graph and a contig name for each chunk, ordered by minimum node id in each chunk. Uses `ConstructionJobs::contig_names()` to determine the contig name for each chunk. If `params.contig_name` is set, returns only the chunk(s) containing a path with the given contig name, or an empty vector if no such paths exist. The contig names reported in the result may be different from the given name. */ std::pair, std::vector> chunk_graph(const GBZ& gbz, const ChunkParameters& params); //------------------------------------------------------------------------------ /* Return the longest common subsequence of the two paths (node sequences), weighted by the sequence length of each node. The result is a list of pairs of path offsets. The paths are not required to be valid in the graph, but they must consist of valid GBWT node identifiers. This is a variant of Myers' O(nd) algorithm. */ std::vector> path_lcs( const GBWTGraph& graph, const gbwt::vector_type& a, const gbwt::vector_type& b); //------------------------------------------------------------------------------ struct ConstructionJobs { // Number of nodes in each job. std::vector nodes_per_job; // Weakly conneted components as sorted lists of node ids. std::vector> weakly_connected_components; // Mapping from node ids to component ids. std::unordered_map node_to_component; // Mapping from component ids to job ids. std::unordered_map component_to_job; // Returns the number of construction jobs. size_t size() const { return this->nodes_per_job.size(); } // Returns the number of components. size_t components() const { return this->weakly_connected_components.size(); } // Returns the size of the given job in nodes. size_t job_size(size_t job_id) const { return this->nodes_per_job[job_id]; } // Maps a node identifier to a component identifier, or `components()` if // there is no such component. size_t component(nid_t node_id) const { auto iter = this->node_to_component.find(node_id); return (iter == this->node_to_component.end() ? this->components() : iter->second); } // Maps a node identifier to a job identifier, or `size()` if there is no such job. size_t job(nid_t node_id) const { auto iter = this->node_to_component.find(node_id); if(iter == this->node_to_component.end()) { return this->size(); } auto component_iter = this->component_to_job.find(iter->second); return (component_iter == this->component_to_job.end() ? this->size() : component_iter->second); } // Maps a component identifier to a job identifier, or `size()` if there is no such job. size_t job_for_component(size_t component_id) const { auto iter = this->component_to_job.find(component_id); return (iter == this->component_to_job.end() ? this->size() : iter->second); } // Returns the list of components assigned to each job. std::vector> components_per_job() const; // Returns a list of contig names for the components based on the given graph. // // This prioritizes reference paths, then generic paths, and finally uses // `component_i` if no other name can be found. std::vector contig_names(const PathHandleGraph& graph) const; // Returns a list of contig names for the components based on the given graph. // // As above, but does not consider paths that do not pass the filter. std::vector contig_names( const PathHandleGraph& graph, const std::function& filter) const; // Clears the jobs and tries to free the memory. void clear(); }; /* Partition the graph into weakly connected components and combine the components into GBWT construction job. Because the jobs do not overlap, partial GBWTs can be built in parallel and merged with the fast algorithm. At the moment, there is only one strategy for determining the jobs: * Sort the components by minimum node id and combine consecutive components as long as their total size in nodes does not exceed `size_bound`. TODO: Add different strategies for combining jobs. */ ConstructionJobs gbwt_construction_jobs(const HandleGraph& graph, size_t size_bound); /* Assigns reference and generic paths to the given construction jobs. Returns the list of path handles assigned to each job. If a path filter is provided, only the paths that pass the filter are assigned to jobs. If a metadata builder is provided, this also copies the metadata from the original graph. As a best practice, the metadata builder should be empty when calling this function. That means the copied paths will be before any newly generated paths in the resulting GBWT for that job. */ std::vector> assign_paths( const PathHandleGraph& graph, const ConstructionJobs& jobs, MetadataBuilder* metadata, const std::function* path_filter ); /* Inserts the selected paths from the graph into the GBWT builder. This is intended to be used with the output of `assign_paths`. */ void insert_paths( const PathHandleGraph& graph, const std::vector& paths, gbwt::GBWTBuilder& builder, size_t job_id, bool show_progress ); //------------------------------------------------------------------------------ struct TopLevelChain { // Handle of the chain in the snarl decomposition. handlegraph::net_handle_t chain; // Handle of the first node visit in the chain. handle_t handle; // Offset of the chain in the children of the root snarl. size_t offset; }; /* Partition the top-level chains in the snarl decomposition between GBWT construction jobs. This assumes that the top-level chains correspond to the weakly connected components in the graph. TODO: Is it possible to test this, as it needs a SnarlDecomposition? */ std::vector> partition_chains(const handlegraph::SnarlDecomposition& snarls, const HandleGraph& graph, const ConstructionJobs& jobs); //------------------------------------------------------------------------------ } // namespace gbwtgraph #endif // GBWTGRAPH_ALGORITHMS_H jltsiren-gbwtgraph-e27bc43/include/gbwtgraph/cached_gbwtgraph.h000066400000000000000000000166621524136466400250250ustar00rootroot00000000000000#ifndef GBWTGRAPH_CACHED_GBWTGRAPH_H #define GBWTGRAPH_CACHED_GBWTGRAPH_H #include #include /* cached_gbwtgraph.h: A cached variant of GBWTGraph. */ namespace gbwtgraph { //------------------------------------------------------------------------------ /* A variant of GBWTGraph intended for algorithms that repeatedly access the edges of a small subgraph. Provides an easy way of using the cached GBWTGraph interface in HandleGraph algorithms. NOTE: The cache is not thread-safe. Use a separate CachedGBWTGraph for each thread. NOTE: For performance reasons, this implementations replicates much of GBWTGraph functionality instead of calling it through virtual functions. */ class CachedGBWTGraph : public HandleGraph { public: CachedGBWTGraph(); CachedGBWTGraph(const CachedGBWTGraph& source); CachedGBWTGraph(CachedGBWTGraph&& source) noexcept; virtual ~CachedGBWTGraph(); explicit CachedGBWTGraph(const GBWTGraph& graph); void swap(CachedGBWTGraph& another) noexcept; CachedGBWTGraph& operator=(const CachedGBWTGraph& source); CachedGBWTGraph& operator=(CachedGBWTGraph&& source) noexcept; const GBWTGraph* graph; gbwt::CachedGBWT cache; //------------------------------------------------------------------------------ /* Standard HandleGraph interface. */ public: // Method to check if a node exists by ID. virtual bool has_node(nid_t node_id) const; // Look up the handle for the node with the given ID in the given orientation. virtual handle_t get_handle(const nid_t& node_id, bool is_reverse = false) const; // Get the ID from a handle. virtual nid_t get_id(const handle_t& handle) const; // Get the orientation of a handle. virtual bool get_is_reverse(const handle_t& handle) const; // Invert the orientation of a handle (potentially without getting its ID). virtual handle_t flip(const handle_t& handle) const; // Get the length of a node. virtual size_t get_length(const handle_t& handle) const; // Get the sequence of a node, presented in the handle's local forward // orientation. virtual std::string get_sequence(const handle_t& handle) const; // Returns one base of a handle's sequence, in the orientation of the // handle. virtual char get_base(const handle_t& handle, size_t index) const; // Returns a substring of a handle's sequence, in the orientation of the // handle. If the indicated substring would extend beyond the end of the // handle's sequence, the return value is truncated to the sequence's end. virtual std::string get_subsequence(const handle_t& handle, size_t index, size_t size) const; // Return the number of nodes in the graph. virtual size_t get_node_count() const; // Return the smallest ID in the graph, or some smaller number if the // smallest ID is unavailable. Return value is unspecified if the graph is empty. virtual nid_t min_node_id() const; // Return the largest ID in the graph, or some larger number if the // largest ID is unavailable. Return value is unspecified if the graph is empty. virtual nid_t max_node_id() const; protected: // Loop over all the handles to next/previous (right/left) nodes. Passes // them to a callback which returns false to stop iterating and true to // continue. Returns true if we finished and false if we stopped early. virtual bool follow_edges_impl(const handle_t& handle, bool go_left, const std::function& iteratee) const; // Loop over all the nodes in the graph in their local forward // orientations, in their internal stored order. Stop if the iteratee // returns false. Can be told to run in parallel, in which case stopping // after a false return value is on a best-effort basis and iteration // order is not defined. Returns true if we finished and false if we // stopped early. virtual bool for_each_handle_impl(const std::function& iteratee, bool parallel = false) const; public: /* More efficient reimplementations. */ // Get the number of edges on the right (go_left = false) or left (go_left // = true) side of the given handle. virtual size_t get_degree(const handle_t& handle, bool go_left) const; // Returns true if there is an edge that allows traversal from the left // handle to the right handle. virtual bool has_edge(const handle_t& left, const handle_t& right) const; //------------------------------------------------------------------------------ /* GBWTGraph specific interface. */ public: // Convert gbwt::node_type to handle_t. static handle_t node_to_handle(gbwt::node_type node) { return GBWTGraph::node_to_handle(node); } // Convert handle_t to gbwt::node_type. static gbwt::node_type handle_to_node(const handle_t& handle) { return GBWTGraph::handle_to_node(handle); } // Get node sequence as a pointer and length. std::string_view get_sequence_view(const handle_t& handle) const { return this->graph->get_sequence_view(handle); } // Determine if the node sequence starts with the given character. bool starts_with(const handle_t& handle, char c) const { return this->graph->starts_with(handle, c); } // Determine if the node sequence ends with the given character. bool ends_with(const handle_t& handle, char c) const { return this->graph->ends_with(handle, c); } // Convert handle_t to gbwt::SearchState. // Note that the state may be empty if the handle does not correspond to a real node. gbwt::SearchState get_state(const handle_t& handle) const { return this->cache.find(handle_to_node(handle)); } // Convert handle_t to gbwt::BidirectionalState. // Note that the state may be empty if the handle does not correspond to a real node. gbwt::BidirectionalState get_bd_state(const handle_t& handle) const { return this->cache.bdFind(handle_to_node(handle)); } // Get the search state corresponding to the vector of handles. gbwt::SearchState find(const std::vector& path) const { return this->graph->find(this->cache, path); } // Get the bidirectional search state corresponding to the vector of handles. gbwt::BidirectionalState bd_find(const std::vector& path) const { return this->graph->bd_find(this->cache, path); } // Visit all successor states of this state and call iteratee for the state. // Stop and return false if the iteratee returns false. // Note that this does not visit empty successor states. bool follow_paths(gbwt::SearchState state, const std::function& iteratee) const { return this->graph->follow_paths(this->cache, state, iteratee); } // Visit all predecessor/successor states of this state and call iteratee for the state. // Stop and return false if the iteratee returns false. // Note that this does not visit empty predecessor/successor states. // Each state corresponds to a path. Going backward extends the path left, while going // extends it right. When going backward, the state is for the reverse orientation. bool follow_paths(gbwt::BidirectionalState state, bool backward, const std::function& iteratee) const { return this->graph->follow_paths(this->cache, state, backward, iteratee); } //------------------------------------------------------------------------------ private: void copy(const CachedGBWTGraph& source); }; //------------------------------------------------------------------------------ } // namespace gbwtgraph #endif // GBWTGRAPH_CACHED_GBWTGRAPH_H jltsiren-gbwtgraph-e27bc43/include/gbwtgraph/gbwtgraph.h000066400000000000000000000763331524136466400235370ustar00rootroot00000000000000#ifndef GBWTGRAPH_GBWTGRAPH_H #define GBWTGRAPH_GBWTGRAPH_H #include #include #include #include #include #include /* gbwtgraph.h: GBWT-based Handle Graph. */ namespace gbwtgraph { //------------------------------------------------------------------------------ /* A HandleGraph implementation that uses GBWT for graph topology and extracts sequences from another HandleGraph. Main features: * Faster sequence access but slower graph navigation than in XG. * `follow_paths()`: A version of `follow_edges()` that only uses edges consistent with the indexed haplotypes. * Exposes haplotype, reference, and generic paths. * Translation from (intervals of) node ids to GFA segment names. The `PathHandleGraph` interface requires GBWT metadata with sample/contig/path names. Information about the named paths is cached using multiple OpenMP threads when the graph is created or loaded. Graph file format versions: 4 Simple-SDS serialization compresses sequences with Zstandard. Compatible with versions 1 to 3. 3 The compressed version uses Simple-SDS serialization. Non-compressed (SDSL) file format is compatible with versions 1 and 2. 2 Translation between GFA segment names and (intervals of) node ids. Optional compressed serialization format. Compatible with version 1. 1 The initial version. */ class GBWTGraph : public PathHandleGraph, public SerializableHandleGraph, public NamedNodeBackTranslation { public: GBWTGraph(); // Call (deserialize() and set_gbwt()) or simple_sds_load() before using the graph. GBWTGraph(const GBWTGraph& source); GBWTGraph(GBWTGraph&& source) noexcept; virtual ~GBWTGraph(); // Build the graph from a GBWT index and a NaiveGraph, which provides sequences and possibly a translation. // Some parts of the construction are multithreaded. GBWTGraph(const gbwt::GBWT& gbwt_index, const NaiveGraph& graph); // Build the graph from another `HandleGraph` and an optional named segment space over it. // Some parts of the construction are multithreaded. GBWTGraph(const gbwt::GBWT& gbwt_index, const HandleGraph& graph, const NamedNodeBackTranslation* segment_space); // Returns a GBWTGraph for the subgraph defined by the given GBWT index. // Updates the given GBWT index to have the same reference samples as this graph, // if they exist in the metadata. // The returned graph will not have a node-to-segment translation. // This is faster than using the graph as a HandleGraph in the constructor. GBWTGraph subgraph(gbwt::GBWT& gbwt_index) const; // Makes some sanity checks on the internal consistency of the structure. // Requires that the GBWT index has been set. // This is called internally during deserialization an is also used in tests. // Throws sdsl::simple_sds::InvalidData if the checks fail. void sanity_checks(); void swap(GBWTGraph& another) noexcept; GBWTGraph& operator=(const GBWTGraph& source); GBWTGraph& operator=(GBWTGraph&& source) noexcept; struct Header { std::uint32_t tag, version; std::uint64_t nodes; std::uint64_t flags; constexpr static std::uint32_t TAG = 0x6B3764AF; constexpr static std::uint32_t VERSION = Version::GRAPH_VERSION; constexpr static std::uint64_t FLAG_MASK = 0x0003; constexpr static std::uint64_t FLAG_TRANSLATION = 0x0001; constexpr static std::uint64_t FLAG_SIMPLE_SDS = 0x0002; // Symbolic names for versions that may be relevant when examining files, // even when the version is current or obsolete. constexpr static std::uint32_t ZSTD_VERSION = 4; // Flag masks for old compatible versions. constexpr static std::uint32_t SIMPLE_SDS_VERSION = 3; constexpr static std::uint64_t SIMPLE_SDS_FLAG_MASK = 0x0003; constexpr static std::uint32_t TRANS_VERSION = 2; constexpr static std::uint64_t TRANS_FLAG_MASK = 0x0001; constexpr static std::uint32_t OLD_VERSION = 1; constexpr static std::uint64_t OLD_FLAG_MASK = 0x0000; constexpr static std::uint32_t MIN_VERSION = OLD_VERSION; // The oldest version we can read. constexpr static std::uint32_t MIN_SERIALIZE_VERSION = SIMPLE_SDS_VERSION; // The oldest version we can write. constexpr static std::uint32_t DEFAULT_VERSION = ZSTD_VERSION; // The version we write by default. Header(); // Throws `sdsl::simple_sds::InvalidData` if the header is invalid. void check() const; void set_version() { this->version = VERSION; } void set(std::uint64_t flag) { this->flags |= flag; } void unset(std::uint64_t flag) { this->flags &= ~flag; } bool get(std::uint64_t flag) const { return (this->flags & flag); } bool operator==(const Header& another) const; bool operator!=(const Header& another) const { return !(this->operator==(another)); } }; const gbwt::GBWT* index; Header header; gbwt::StringArray sequences; sdsl::bit_vector real_nodes; // Segment to node translation. Node `v` maps to segment `node_to_segment.predecessor(v)->first`. gbwt::StringArray segments; sdsl::sd_vector<> node_to_segment; // Cached named path information. std::vector named_paths; std::unordered_map name_to_path; // To offset in `named_paths`. std::unordered_map id_to_path; // To offset in `named_paths`. sample_name_set reference_samples; // Parsed from tags in the GBWT. // Path handles are either indexes into named_paths, or, if larger than // named_paths, are an offset of the size of named_paths plus a path number in // our metadata object. This syntactically allows for aliasing: cached paths // in named_paths also ultimately refer to path numbers in the metadata. So, // when iterating, we need to remember to skip path numbers in the metadata // that are also cached named paths. constexpr static size_t CHUNK_SIZE = 1024; // For parallel for_each_handle(). // TODO: This should be 0, as in GFA W-lines. constexpr static size_t NO_PHASE = std::numeric_limits::max(); const static std::string EXTENSION; // ".gg" //------------------------------------------------------------------------------ /* Standard HandleGraph interface. */ public: // Method to check if a node exists by ID. virtual bool has_node(nid_t node_id) const; // Look up the handle for the node with the given ID in the given orientation. virtual handle_t get_handle(const nid_t& node_id, bool is_reverse = false) const; // Get the ID from a handle. virtual nid_t get_id(const handle_t& handle) const; // Get the orientation of a handle. virtual bool get_is_reverse(const handle_t& handle) const; // Invert the orientation of a handle (potentially without getting its ID). virtual handle_t flip(const handle_t& handle) const; // Get the length of a node. virtual size_t get_length(const handle_t& handle) const; // Get the sequence of a node, presented in the handle's local forward // orientation. virtual std::string get_sequence(const handle_t& handle) const; // Returns one base of a handle's sequence, in the orientation of the // handle. virtual char get_base(const handle_t& handle, size_t index) const; // Returns a substring of a handle's sequence, in the orientation of the // handle. If the indicated substring would extend beyond the end of the // handle's sequence, the return value is truncated to the sequence's end. virtual std::string get_subsequence(const handle_t& handle, size_t index, size_t size) const; // Return the number of nodes in the graph. virtual size_t get_node_count() const; // Return the smallest ID in the graph, or some smaller number if the // smallest ID is unavailable. Return value is unspecified if the graph is empty. virtual nid_t min_node_id() const; // Return the largest ID in the graph, or some larger number if the // largest ID is unavailable. Return value is unspecified if the graph is empty. virtual nid_t max_node_id() const; protected: // Loop over all the handles to next/previous (right/left) nodes. Passes // them to a callback which returns false to stop iterating and true to // continue. Returns true if we finished and false if we stopped early. virtual bool follow_edges_impl(const handle_t& handle, bool go_left, const std::function& iteratee) const; // Loop over all the nodes in the graph in their local forward // orientations, in their internal stored order. Stop if the iteratee // returns false. Can be told to run in parallel, in which case stopping // after a false return value is on a best-effort basis and iteration // order is not defined. Returns true if we finished and false if we // stopped early. virtual bool for_each_handle_impl(const std::function& iteratee, bool parallel = false) const; public: /* More efficient reimplementations. */ // Get the number of edges on the right (go_left = false) or left (go_left // = true) side of the given handle. virtual size_t get_degree(const handle_t& handle, bool go_left) const; // Returns true if there is an edge that allows traversal from the left // handle to the right handle. virtual bool has_edge(const handle_t& left, const handle_t& right) const; //------------------------------------------------------------------------------ /* PathHandleGraph interface. */ public: /// Returns the number of paths stored in the graph virtual size_t get_path_count() const; /// Determine if a path name exists and is legal to get a path handle for. virtual bool has_path(const std::string& path_name) const; /// Look up the path handle for the given path name. /// The path with that name must exist. virtual path_handle_t get_path_handle(const std::string& path_name) const; /// Look up the name of a path from a handle to it virtual std::string get_path_name(const path_handle_t& path_handle) const; /// Look up whether a path is circular virtual bool get_is_circular(const path_handle_t& path_handle) const; /// Returns the number of node steps in the path virtual size_t get_step_count(const path_handle_t& path_handle) const; /// Returns the number of node steps on a handle virtual size_t get_step_count(const handle_t& handle) const; /// Get a node handle (node ID and orientation) from a handle to an step on a path virtual handle_t get_handle_of_step(const step_handle_t& step_handle) const; /// Returns a handle to the path that an step is on virtual path_handle_t get_path_handle_of_step(const step_handle_t& step_handle) const; /// Get a handle to the first step, which will be an arbitrary step in a circular path /// that we consider "first" based on our construction of the path. If the path is empty, /// then the implementation must return the same value as path_end(). virtual step_handle_t path_begin(const path_handle_t& path_handle) const; /// Get a handle to a fictitious position past the end of a path. This position is /// returned by get_next_step for the final step in a path in a non-circular path. /// Note: get_next_step will *NEVER* return this value for a circular path. virtual step_handle_t path_end(const path_handle_t& path_handle) const; /// Get a handle to the last step, which will be an arbitrary step in a circular path that /// we consider "last" based on our construction of the path. If the path is empty /// then the implementation must return the same value as path_front_end(). virtual step_handle_t path_back(const path_handle_t& path_handle) const; /// Get a handle to a fictitious position before the beginning of a path. This position is /// return by get_previous_step for the first step in a path in a non-circular path. /// Note: get_previous_step will *NEVER* return this value for a circular path. virtual step_handle_t path_front_end(const path_handle_t& path_handle) const; /// Returns true if the step is not the last step in a non-circular path. virtual bool has_next_step(const step_handle_t& step_handle) const; /// Returns true if the step is not the first step in a non-circular path. virtual bool has_previous_step(const step_handle_t& step_handle) const; /// Returns a handle to the next step on the path. If the given step is the final step /// of a non-circular path, this method has undefined behavior. In a circular path, /// the "last" step will loop around to the "first" step. virtual step_handle_t get_next_step(const step_handle_t& step_handle) const; /// Returns a handle to the previous step on the path. If the given step is the first /// step of a non-circular path, this method has undefined behavior. In a circular path, /// it will loop around from the "first" step (i.e. the one returned by path_begin) to /// the "last" step. virtual step_handle_t get_previous_step(const step_handle_t& step_handle) const; using PathHandleGraph::for_each_path_handle; using PathHandleGraph::for_each_step_on_handle; protected: /// Execute a function on each path in the graph. If it returns false, stop /// iteration. Returns true if we finished and false if we stopped early. virtual bool for_each_path_handle_impl(const std::function& iteratee) const; /// Execute a function on each step of a handle in any path. If it /// returns false, stop iteration. Returns true if we finished and false if /// we stopped early. virtual bool for_each_step_on_handle_impl(const handle_t& handle, const std::function& iteratee) const; //------------------------------------------------------------------------------ /* PathMetadata interface, actually exposing threads. */ public: /// What is the given path meant to be representing? virtual PathSense get_sense(const path_handle_t& handle) const; /// Get the name of the sample or assembly asociated with the /// path-or-thread, or NO_SAMPLE_NAME if it does not belong to one. virtual std::string get_sample_name(const path_handle_t& handle) const; /// Get the name of the contig or gene asociated with the path-or-thread, /// or NO_LOCUS_NAME if it does not belong to one. virtual std::string get_locus_name(const path_handle_t& handle) const; /// Get the haplotype number (0 or 1, for diploid) of the path-or-thread, /// or NO_HAPLOTYPE if it does not belong to one. virtual size_t get_haplotype(const path_handle_t& handle) const; /// Get the phase block number (contiguously phased region of a sample, /// contig, and haplotype) of the path-or-thread, or NO_PHASE_BLOCK if it /// does not belong to one. virtual size_t get_phase_block(const path_handle_t& handle) const; /// Get the bounds of the path-or-thread that are actually represented /// here. Should be NO_SUBRANGE if the entirety is represented here, and /// 0-based inclusive start and exclusive end positions of the stored /// region on the full path-or-thread if a subregion is stored. /// /// If no end position is stored, NO_END_POSITION may be returned for the /// end position. virtual subrange_t get_subrange(const path_handle_t& handle) const; protected: /// Loop through all the paths matching the given query. Query elements /// which are null match everything. Returns false and stops if the /// iteratee returns false. virtual bool for_each_path_matching_impl(const std::unordered_set* senses, const std::unordered_set* samples, const std::unordered_set* loci, const std::function& iteratee) const; /// Loop through all steps on the given handle for paths with the given /// sense. Returns false and stops if the iteratee returns false. virtual bool for_each_step_of_sense_impl(const handle_t& visited, const PathSense& sense, const std::function& iteratee) const; private: /// Internal iteration method to find all the GBWT edges and their path /// numbers on a node. Only looks at forward sequence for each path, but /// looks at both orientations of the node. bool for_each_edge_and_path_on_handle(const handle_t& handle, const std::function& iteratee) const; /// Get the sample number for the given sample name and senses. /// Returns the number of samples in the metadata if not found. gbwt::size_type sample_number_for_sample_name(const std::unordered_set* senses, const std::string& sample_name) const; /// Iterate over all paths of the given senses with the given sample (which /// can be NO_SAMPLE_NAME) and locus bool for_each_path_matching_sample_and_locus(const std::unordered_set* senses, const std::string& sample_name, const std::string& locus_name, const std::function& iteratee) const; /// Iterate over all paths of the given senses with the given sample (which /// can be NO_SAMPLE_NAME) bool for_each_path_matching_sample(const std::unordered_set* senses, const std::string& sample_name, const std::function& iteratee) const; /// Iterate over all paths of the given senses with the given locus bool for_each_path_matching_locus(const std::unordered_set* senses, const std::string& locus_name, const std::function& iteratee) const; //------------------------------------------------------------------------------ /* SerializableHandleGraph interface. Serialization / deserialization throws `std::runtime_error` on failure. */ public: // Set the GBWT index used for graph topology and cache named path information. // Call deserialize() before using the graph. // MUST be called before using the graph if the graph is deserialize()-ed. void set_gbwt(const gbwt::GBWT& gbwt_index); // Set the address of the GBWT index. This assumes that the new index is identical // to the one that was already in use. Intended for implementing containers that // store both GBWT and GBWTGraph and may get moved around. void set_gbwt_address(const gbwt::GBWT& gbwt_index); /// Return a magic number to identify serialized GBWTGraphs. virtual uint32_t get_magic_number() const; protected: // Underlying implementation for "serialize" method. // Serialize the sequences to the ostream in SDSL format. virtual void serialize_members(std::ostream& out) const; // Underlying implementation to "deserialize" method. // Load the sequences from the istream. // User must call set_gbwt() before using the graph. virtual void deserialize_members(std::istream& in); //------------------------------------------------------------------------------ // TODO: Is anyone actually using this? /* WIP SegmentHandleGraph interface. NOTE: The implementation stores the translations also for segments that were not used on any path. Because the corresponding nodes are missing from the graph, the methods will treat them like missing segments. */ public: /// Returns `true` if the graph contains a translation from node ids to segment names. virtual bool has_segment_names() const; /// Returns (GFA segment name, semiopen node id range) containing the handle. /// If there is no such translation, returns ("id", (id, id + 1)). virtual std::pair> get_segment(const handle_t& handle) const; /// Returns (GFA segment name, starting offset in the same orientation) for the handle. /// If there is no translation, returns ("id", 0). virtual std::pair get_segment_name_and_offset(const handle_t& handle) const; /// Returns the name of the original GFA segment corresponding to the handle. /// If there is no translation, returns the node id as a string. virtual std::string get_segment_name(const handle_t& handle) const; /// Returns the starting offset in the original GFA segment corresponding to the handle /// in the same orientation as the handle. /// If there is no translation, returns 0. virtual size_t get_segment_offset(const handle_t& handle) const; /// Calls `iteratee` with each segment name as a string, and the semiopen /// interval of node ids corresponding to it as a std::pair of nid_t /// values. Stops early if the call returns `false`. /// Returns false if iteration was stopped, and true otherwise. template bool for_each_segment(const Iteratee& iteratee, bool parallel = false) const { return for_each_segment_impl(handlegraph::BoolReturningWrapper::wrap(iteratee), parallel); } /// Calls `iteratee` with each inter-segment edge (as an edge_t) and the /// corresponding segment names in the canonical orientation (as two /// strings). Stops early if the call returns `false`. /// Returns false if iteration was stopped, and true otherwise. template bool for_each_link(const Iteratee& iteratee, bool parallel = false) const { return for_each_link_impl(handlegraph::BoolReturningWrapper::wrap(iteratee), parallel); } protected: // Calls `iteratee` with each segment name and the semiopen interval of node ids // corresponding to it. Stops early if the call returns `false`. // In GBWTGraph, the segments are visited in sorted order by node ids. virtual bool for_each_segment_impl(const std::function&)>& iteratee, bool parallel) const; // Calls `iteratee` with each inter-segment edge and the corresponding segment names // in the canonical orientation. Stops early if the call returns `false`. virtual bool for_each_link_impl(const std::function& iteratee, bool parallel) const; //------------------------------------------------------------------------------ /* NamedNodeBackTranslation interface. */ public: /// Translate a node range back to segment space. /// Return value is not defined if no segments exist or the node does not /// exist. virtual std::vector translate_back(const oriented_node_range_t& range) const; /// Get a segment name. Return value is not defined if no segments exist or /// if the segment is out of range. virtual std::string get_back_graph_node_name(const nid_t& back_node_id) const; //------------------------------------------------------------------------------ /* GBWTGraph specific interface. Serialization / deserialization throws `std::runtime_error` on failure. */ public: // Serialize the graph into the output stream in the Simple-SDS format. void simple_sds_serialize(std::ostream& out) const { this->simple_sds_serialize_version(out, Header::DEFAULT_VERSION); } // Serialize the graph into the output stream in the Simple-SDS format, with the given version. // Supported versions are `Header::MIN_SERIALIZE_VERSION` to `Header::VERSION`. void simple_sds_serialize_version(std::ostream& out, std::uint32_t version) const; // Deserialize or decompress the graph from the input stream and set the given // GBWT index. Note that the GBWT index is essential for loading the structure. void simple_sds_load(std::istream& in, const gbwt::GBWT& gbwt_index); // Returns the size of the serialized structure in elements. size_t simple_sds_size() const; // Convert gbwt::node_type to handle_t. static handle_t node_to_handle(gbwt::node_type node) { return handlegraph::as_handle(node); } // Convert handle_t to gbwt::node_type. static gbwt::node_type handle_to_node(const handle_t& handle) { return handlegraph::as_integer(handle); } // Convert GBWT path id to path_handle_t. path_handle_t path_to_handle(gbwt::size_type path) const; // Convert path_handle_t to GBWT path id. gbwt::size_type handle_to_path(const path_handle_t& path_handle) const; // Get node sequence as a pointer and length. std::string_view get_sequence_view(const handle_t& handle) const; // Determine if the node sequence starts with the given character. bool starts_with(const handle_t& handle, char c) const; // Determine if the node sequence ends with the given character. bool ends_with(const handle_t& handle, char c) const; // Convert handle_t to gbwt::SearchState. // Note that the state may be empty if the handle does not correspond to a real node. gbwt::SearchState get_state(const handle_t& handle) const { return this->index->find(handle_to_node(handle)); } // Convert handle_t to gbwt::BidirectionalState. // Note that the state may be empty if the handle does not correspond to a real node. gbwt::BidirectionalState get_bd_state(const handle_t& handle) const { return this->index->bdFind(handle_to_node(handle)); } // Get the search state corresponding to the vector of handles. gbwt::SearchState find(const std::vector& path) const; // Get the bidirectional search state corresponding to the vector of handles. gbwt::BidirectionalState bd_find(const std::vector& path) const; // Visit all successor states of this state and call iteratee for the state. // Stop and return false if the iteratee returns false. // Note that this does not visit empty successor states. bool follow_paths(gbwt::SearchState state, const std::function& iteratee) const { return this->follow_paths(this->get_single_cache(), state, iteratee); } // Visit all predecessor/successor states of this state and call iteratee for the state. // Stop and return false if the iteratee returns false. // Note that this does not visit empty predecessor/successor states. // Each state corresponds to a path. Going backward extends the path left, while going // extends it right. When going backward, the state is for the reverse orientation. bool follow_paths(gbwt::BidirectionalState state, bool backward, const std::function& iteratee) const { return this->follow_paths(this->get_single_cache(), state, backward, iteratee); } //------------------------------------------------------------------------------ /* Cached GBWTGraph specific interface. Each thread must use a separate cache. */ // Return a cache for the GBWT index. Note: The cache is not thread-safe. gbwt::CachedGBWT get_cache() const { return gbwt::CachedGBWT(*(this->index), false); } // Return a single-node cache for the GBWT index. Mostly for internal use. // Note: The cache is not thread-safe. gbwt::CachedGBWT get_single_cache() const { return gbwt::CachedGBWT(*(this->index), true); } // Convert handle_t to gbwt::SearchState. gbwt::SearchState get_state(const gbwt::CachedGBWT& cache, const handle_t& handle) const { return cache.find(handle_to_node(handle)); } // Convert handle_t to gbwt::BidirectionalState. gbwt::BidirectionalState get_bd_state(const gbwt::CachedGBWT& cache, const handle_t& handle) const { return cache.bdFind(handle_to_node(handle)); } // Get the search state corresponding to the vector of handles. gbwt::SearchState find(const gbwt::CachedGBWT& cache, const std::vector& path) const; // Get the bidirectional search state corresponding to the vector of handles. gbwt::BidirectionalState bd_find(const gbwt::CachedGBWT& cache, const std::vector& path) const; // Visit all successor states of this state and call iteratee for the state. // Stop and return false if the iteratee returns false. // Note that the state may be empty if no path continues to that node. bool follow_paths(const gbwt::CachedGBWT& cache, gbwt::SearchState state, const std::function& iteratee) const; // Visit all predecessor/successor states of this state and call iteratee for the state. // Stop and return false if the iteratee returns false. // Note that the state may be empty if no path continues to that node. // Each state corresponds to a path. Going backward extends the path left, while going // extends it right. When going backward, the state is for the reverse orientation. bool follow_paths(const gbwt::CachedGBWT& cache, gbwt::BidirectionalState state, bool backward, const std::function& iteratee) const; // Loop over all the handles to next/previous (right/left) nodes. Passes // them to a callback which returns false to stop iterating and true to // continue. Returns true if we finished and false if we stopped early. bool cached_follow_edges(const gbwt::CachedGBWT& cache, const handle_t& handle, bool go_left, const std::function& iteratee) const; //------------------------------------------------------------------------------ private: friend class CachedGBWTGraph; // Construction helpers. void determine_real_nodes(); void cache_named_paths(); void copy(const GBWTGraph& source); // Copies a translation over this graph's node IDs from a // NamedNodeBackTranslation into the internal representation used in a // GBWTGraph. // Returns `StringArray` of segment names and `sd_vector<>` mapping node ids to names. // Runs of nonexistent nodes become segments with empty names. // Throws if the translation cannot be represented (i.e. segments aren't // forward strands of contiguous ascending node ID ranges). std::pair> copy_translation(const NamedNodeBackTranslation& translation) const; size_t node_offset(gbwt::node_type node) const { return node - this->index->firstNode(); } size_t node_offset(const handle_t& handle) const { return this->node_offset(handle_to_node(handle)); } }; //------------------------------------------------------------------------------ /* Traverse all haplotype-consistent windows in the graph and call lambda() for each window. Uses multiple threads, so the lambda should be thread-safe. A window starts with the sequence of a node and is followed by window_size - 1 bases from subsequent nodes. If no extensions are possible, a shorter substring of length >= window_size also qualifies as a window. */ void for_each_haplotype_window(const GBWTGraph& graph, size_t window_size, const std::function&, const std::string&)>& lambda, bool parallel); /* As above, but prioritize more windows with less overall redundancy. If the length of the initial node is at least window_size, it becomes a separate window. Extension windows then take only the last window_size - 1 bases from it. */ void for_each_nonredundant_window( const GBWTGraph& graph, size_t window_size, const std::function&, const std::string&)>& lambda, bool parallel); //------------------------------------------------------------------------------ } // namespace gbwtgraph #endif // GBWTGRAPH_GBWTGRAPH_H jltsiren-gbwtgraph-e27bc43/include/gbwtgraph/gbz.h000066400000000000000000000224121524136466400223210ustar00rootroot00000000000000#ifndef GBWTGRAPH_GBZ_H #define GBWTGRAPH_GBZ_H #include /* gbz.h: GBZ file format. */ namespace gbwtgraph { //------------------------------------------------------------------------------ /* GBZ file format wrapper, as specified in SERIALIZATION.md. The wrapper owns the GBWT index and the GBWTGraph. Constructors, serialization, and loading throw `std::runtime_error` on failure. File format versions: 3 Contains GBZ version 4 with zstd-compressed BWT. Compatible with versions 1 and 2. 2 Contains GBWTGraph version 4 with zstd-compressed sequences. Compatible with version 1. 1 The initial version. */ class GBZ { public: // This is a valid graph, unlike the default GBWTGraph. GBZ(); GBZ(const GBZ& source); GBZ(GBZ&& source) noexcept; ~GBZ(); // Build GBZ from the structures returned by `gfa_to_gbwt()`. // Calls compute_pggname() internally. Resets the pointers to `nullptr`. GBZ(std::unique_ptr& index, std::unique_ptr& graph); // Build GBZ from a GBWT index and a graph. // Calls compute_pggname() internally. Note that the GBZ will store a // copy of the GBWT index. Mostly for testing. GBZ(const gbwt::GBWT& index, const NaiveGraph& graph); // Builds GBZ from a set of non-overlapping subgraphs. // Moves the subgraphs out of the provided vector. // Fails with `std::exit()` if the graphs have overlapping node ids. // Sets the union of reference samples in the GBWT indexes as reference // samples. Does not create a node-to-segment translation. // Calls compute_pggname() internally but does not set any relationships. explicit GBZ(std::vector&& subgraphs); // Builds a GBZ from a GBWT index and a GBZ supergraph. // Reference samples in the supergraph that are present in the GBWT // index will be set as reference samples. Calls compute_pggname() // internally. The provided GBWT index will be moved into the GBZ. GBZ(gbwt::GBWT&& index, const GBZ& supergraph); // Build GBZ from a GBWT index and a `HandleGraph`, with an optional // translation. Because the parent graph does not store GraphName // information, compute_pggname() must be called separately after // construction. The provided GBWT index will be moved into the GBZ. GBZ(gbwt::GBWT&& index, const HandleGraph& graph, const NamedNodeBackTranslation* segment_space); void swap(GBZ& another) noexcept; GBZ& operator=(const GBZ& source); GBZ& operator=(GBZ&& source) noexcept; //------------------------------------------------------------------------------ /* Stable graph names (pggname) and known relationships between graphs. See `GraphName` documentation in utils.h. */ enum class ParentGraphType { // Determine the relationship heuristically. HEURISTIC, // The parent graph is a supergraph of this graph, unless the names are the same. SUPERGRAPH, // The parent graph is a translation target of this graph, unless the names are the same. TRANSLATION_TARGET }; /* Computes the pggname for this graph and stores it in the tags. Returns true on success, false on failure. If a parent graph is given and it has a set name, adds the corresponding relationship and imports all known relationships from the other graph. If an explicit relationship type is not given, the following heuristic will be used: 1. If the GBWTGraph has a node-to-segment translation, the relationship is a translation to the parent graph. 2. Otherwise, if the parent graph's pggname is different from the computed name for this graph, the relationship is a subgraph relationship. 3. Otherwise, no relationship is added. When the GBZ is built from a SequenceSource or another GBZ, this function is called automatically by the constructor. When the parent graph is a generic HandleGraph, GraphName information cannot be imported and this function must be called separately after construction. */ bool compute_pggname(const GraphName* parent, ParentGraphType relationship = ParentGraphType::HEURISTIC); // Returns the graph name object for this graph based on the information // stored in the tags. GraphName graph_name() const { return GraphName(this->tags); } // Returns the pggname for this graph, or an empty string if not set. std::string pggname() const { return this->tags.get(GraphName::GBZ_NAME_TAG); } // Returns the pggname of the translation target for the node-to-segment // translation, or an empty string if not set. std::string translation_target() const { return this->tags.get(GraphName::GBZ_TRANSLATION_TARGET_TAG); } //------------------------------------------------------------------------------ /* Reference samples and named paths. Reference samples cannot be changed conveniently within GBWTGraph, because the pointer to the GBWT index is const. */ // Sets the given sample names as reference samples, but only if they are // present in the GBWT metadata. Returns the number of reference samples. // This is somewhat expensive, as the GBWTGraph must recache named paths. size_t set_reference_samples(const sample_name_set& samples); // Returns the set of reference samples. // Some of these samples may not exist in the GBWT metadata. const sample_name_set& get_reference_samples() const { return this->graph.reference_samples; } // Returns the number of paths in the graph. size_t paths() const { return this->index.metadata.paths(); } // Returns the number of reference and generic paths in the graph. size_t named_paths() const { return this->graph.named_paths.size(); } //------------------------------------------------------------------------------ struct Header { std::uint32_t tag, version; std::uint64_t flags; constexpr static std::uint32_t TAG = 0x205A4247; // "GBZ " constexpr static std::uint32_t VERSION = Version::GBZ_VERSION; constexpr static std::uint64_t FLAG_MASK = 0x0000; // Symbolic names for versions that may be relevant when examining files, // even when the version is current or obsolete. constexpr static std::uint32_t ZSTD_BWT_VERSION = 3; // Flag masks for old compatible versions. constexpr static std::uint32_t ZSTD_SEQUENCES_VERSION = 2; constexpr static std::uint64_t ZSTD_SEQUENCES_FLAG_MASK = 0x0000; constexpr static std::uint32_t OLD_VERSION = 1; constexpr static std::uint64_t OLD_FLAG_MASK = 0x0000; constexpr static std::uint32_t MIN_VERSION = OLD_VERSION; // The oldest version we can read. constexpr static std::uint32_t MIN_SERIALIZE_VERSION = OLD_VERSION; // The oldest version we can write. constexpr static std::uint32_t DEFAULT_VERSION = ZSTD_SEQUENCES_VERSION; // The version we write by default. Header(); // Throws `sdsl::simple_sds::InvalidData` if the header is invalid. void check() const; void set_version() { this->version = VERSION; } void set(std::uint64_t flag) { this->flags |= flag; } void unset(std::uint64_t flag) { this->flags &= ~flag; } bool get(std::uint64_t flag) const { return (this->flags & flag); } bool operator==(const Header& another) const; bool operator!=(const Header& another) const { return !(this->operator==(another)); } }; //------------------------------------------------------------------------------ Header header; gbwt::Tags tags; gbwt::GBWT index; GBWTGraph graph; const static std::string EXTENSION; // ".gbz" // Serialize the the GBZ into the output stream in the Simple-SDS format. void simple_sds_serialize(std::ostream& out) const { this->simple_sds_serialize_version(out, Header::DEFAULT_VERSION); } // Serialize the GBZ into the output stream in the Simple-SDS format, with the given version. // Supported versions are `Header::MIN_SERIALIZE_VERSION` to `Header::VERSION`. void simple_sds_serialize_version(std::ostream& out, std::uint32_t version) const; // Serialize the given GBWT and GBWTGraph objects in the GBZ format. // NOTE: GBZ tags will be empty, except for the source tag. static void simple_sds_serialize(const gbwt::GBWT& index, const GBWTGraph& graph, std::ostream& out); // Deserialize or decompress the GBZ from the input stream. void simple_sds_load(std::istream& in); // Returns the size of the serialized structure in elements. size_t simple_sds_size() const; // Returns the GBZ tags from the given file, without loading it entirely. static gbwt::Tags simple_sds_load_tags(const std::string& filename); // Serialize the GBWT (Simple-SDS format) and the GBWTGraph to separate files. // Default graph format is libhandlegraph / SDSL. // NOTE: GBZ tags are not serialized. void serialize_to_files(const std::string& gbwt_name, const std::string& graph_name, bool simple_sds_graph = false) const; // Loads the GBWT (Simple-SDS format) and the GBWTGraph from separate files. // Graph format is libhandlegraph / SDSL; the Simple-SDS format cannot be read. // NOTE: GBZ tags are not loaded. void load_from_files(const std::string& gbwt_name, const std::string& graph_name); private: void copy(const GBZ& source); void reset_tags(); void add_source(); void set_gbwt(); void set_gbwt_address(); }; //------------------------------------------------------------------------------ } // namespace gbwtgraph #endif // GBWTGRAPH_GBZ_H jltsiren-gbwtgraph-e27bc43/include/gbwtgraph/gfa.h000066400000000000000000000203471524136466400223010ustar00rootroot00000000000000#ifndef GBWTGRAPH_GFA_H #define GBWTGRAPH_GFA_H #include #include #include #include #include /* gfa.h: Tools for building GBWTGraph from GFA. */ namespace gbwtgraph { //------------------------------------------------------------------------------ // TODO: Add sanity checks. struct GFAParsingParameters { // GBWT construction parameters. `node_width` is no longer used, as it can be determined // automatically. There are no sanity checks for `batch_size`. gbwt::size_type node_width = gbwt::WORD_BITS; gbwt::size_type batch_size = gbwt::DynamicGBWT::INSERT_BATCH_SIZE; gbwt::size_type sample_interval = gbwt::DynamicGBWT::SAMPLE_INTERVAL; // Chop segments longer than this into multiple nodes. Use 0 to disable chopping. size_t max_node_length = MAX_NODE_LENGTH; // To avoid creating too many jobs, combine small consecutive components into jobs // of at most `num_nodes / approximate_num_jobs` nodes. Value 0 is interpreted as 1. constexpr static size_t APPROXIMATE_NUM_JOBS = 32; size_t approximate_num_jobs = APPROXIMATE_NUM_JOBS; // Try to run this may construction jobs in parallel. Value 0 is interpreted as 1. size_t parallel_jobs = 1; // Determine GBWT batch size automatically. If the length of the longest path is `N` // segments, batch size will be the maximum of the default (100 million) and // `gbwt::DynamicGBWT::MIN_SEQUENCES_PER_BATCH * (N + 1)` but no more than GFA file // size in bytes. This should ensure that each batch consists of at least 10 paths // and their reverse complements. With heavy chopping, path length in nodes may be // much larger than `N`, and hence it may be useful to set the batch size manually. bool automatic_batch_size = true; bool show_progress = false; /* To parse path names, we use a string regex and a string listing field types. For the regex, each submatch (part of the regex separated by parentheses) is a field. The fields are numbered according to preorder traversal from left to right, with 0 corresponding to the entire path name. fields[i] maps field i to a GBWT path name component. Possible values are: S sample name C contig name H haplotype identifier F fragment identifier The values are case-insensitive. Any other character indicates that the field should not be used. If the string is too short, subsequent fields are not used. Each component may occur only once in the string. */ const static std::string DEFAULT_REGEX; // .* const static std::string DEFAULT_FIELDS; // C const static PathSense DEFAULT_SENSE; // Defaults to generic const static std::string PAN_SN_REGEX; // (.*)#(.*)#(.*) const static std::string PAN_SN_FIELDS; // XSHC const static PathSense PAN_SN_SENSE; // panSN names should default to haplotype struct PathNameParsingParameters { std::string regex; std::string fields; PathSense sense; PathNameParsingParameters() = default; PathNameParsingParameters(const PathNameParsingParameters& other) = default; PathNameParsingParameters(PathNameParsingParameters&& other) noexcept = default; PathNameParsingParameters& operator=(const PathNameParsingParameters& other) = default; PathNameParsingParameters& operator=(PathNameParsingParameters&& other) noexcept = default; PathNameParsingParameters(const std::string& path_name_regex, const std::string& path_name_fields, PathSense sense = PathSense::GENERIC); }; // We consult these in order until one of the regular expressions matches. std::list path_name_formats {{DEFAULT_REGEX, DEFAULT_FIELDS, DEFAULT_SENSE}}; // We are familiar with these GFA versions. const static std::vector SUPPORTED_VERSIONS; // 1.0, 1.1, 1.2 }; //------------------------------------------------------------------------------ struct GFAExtractionParameters { // Use this many OpenMP threads for extracting paths and walks. Value 0 is interpreted // as 1. size_t num_threads = 1; size_t threads() const { return std::max(this->num_threads, size_t(1)); } // Cache GBWT records larger than this size to speed up decompression. constexpr static size_t LARGE_RECORD_BYTES = 1024; size_t large_record_bytes = LARGE_RECORD_BYTES; enum path_mode { mode_default, // Generic paths as P-lines, haplotype paths as W-lines. mode_pan_sn, // All paths as P-lines with PanSN names. mode_ref_only, // Generic paths as P-lines, reference paths as W-lines. }; path_mode mode = mode_default; static std::string mode_name(path_mode mode); static path_mode get_mode(const std::string& name); // Use the node-to-segment translation if it exists. bool use_translation = true; bool show_progress = false; }; //------------------------------------------------------------------------------ /* Build GBWT from GFA P-lines, W-lines, and/or Z-lines with the following assumptions: 1. Links and paths have no overlaps between segments. 2. There are no containments. If the construction fails, the function throws `std::runtime_error`. However, if there is an error during the multithreaded path/walk parsing stage, the construction exits with `std::exit()`. Before GBWT construction, the graph is partitioned into weakly connected components. The components are ordered by node ids, and contiguous ranges of components are assigned to jobs of roughly equal size. A separate GBWT index is built for each job, and the partial indexes are merged using the fast algorithm. Multiple jobs can be run in parallel. The construction is done in several passes over a memory-mapped GFA file. The function returns the GBWT index and a naive graph implementation that can be used for GBWTGraph construction. If there are segments longer than the maximum length specified in the parameters, such segments will be broken into nodes of that length. If segment identifiers are not positive integers, they will be translated into such identifiers. In both cases, the sequence source will contain a translation from segment names to ranges of node identifiers. If there is a stable graph name (pggname) and/or subgraph/translation relationships stored in the GFA headers, the information will be stored in NaiveGraph tags. TODO: Compute parent graph name (by extending EmptyGraph) if the name is not present in the headers. */ std::pair, std::unique_ptr> gfa_to_gbwt(const std::string& gfa_filename, const GFAParsingParameters& parameters = GFAParsingParameters()); /* Writes the graph as GFA into the output stream in a normalized form. The lines are ordered in the following way: 1. H-lines, with a file header with a version tag and possible reference samples tag first. If GraphName information is present, header lines for it will follow. 2. S-lines ordered by node ids. 3. L-lines in canonical order. When using a single thread, the edges (from, to) are ordered by tuples (id(from), is_reverse(from), id(to), is_reverse(to)). All overlaps are `*`. 4. P-lines for generic paths stored under the sample named `_gbwt_ref` (defined in `GENERIC_PATH_SAMPLE_NAME`). When using a single thread, the paths are ordered by GBWT path ids. All overlaps are `*`. 5. W-lines for other paths. When using a single thread, the paths are ordered by GBWT path ids. If the GBWT does not contain path names, all GBWT paths will be written as P-lines. NOTE: If a node-to-segment translation is present and used (according to the parameters), header lines corresponding to the graph name will not be written. This is because it cannot be determined efficiently if the resulting graph is the translation target or its subgraph. */ void gbwt_to_gfa ( const GBZ& gbz, std::ostream& out, const GFAExtractionParameters& parameters = GFAExtractionParameters() ); /* Writes the canonical GFA representation of the graph into the given output stream. See https://github.com/jltsiren/pggname for details. */ void gbwt_to_canonical_gfa(const GBWTGraph& graph, std::ostream& out); extern const std::string GFA_EXTENSION; // ".gfa" //------------------------------------------------------------------------------ } // namespace gbwtgraph #endif // GBWTGRAPH_GFA_H jltsiren-gbwtgraph-e27bc43/include/gbwtgraph/index.h000066400000000000000000000131031524136466400226430ustar00rootroot00000000000000#ifndef GBWTGRAPH_CONSTRUCTION_H #define GBWTGRAPH_CONSTRUCTION_H #include #include #include #include #include /* index.h: Minimizer index construction from GBWTGraph. */ namespace gbwtgraph { //------------------------------------------------------------------------------ /* Index the haplotypes in the graph. Insert the minimizers into the provided index. Function argument get_payload is used to generate the payload for each position stored in the index. The payload is provided as a pointer to index.payload_size() words of payload. The pointer must remain valid for the duration of the call. If payload size is zero, the pointer may be null. The number of threads can be set through OpenMP. If the graph contains GraphName information, it will be copied to the minimizer index. We do a lot of redundant work by traversing both orientations and finding almost the same minimizers in each orientation. If we consider only the windows starting in forward (reverse) orientation, we may miss windows that cross from a reverse node to a forward node (from a forward node to a reverse node). */ template void index_haplotypes ( const GBZ& gbz, MinimizerIndex& index, const std::function& get_payload ); // Minimizer index tag for the path name fields that define a haplotype in // index_haplotypes_with_paths(). extern const std::string PATH_NAME_FIELDS_TAG; // "path_name_fields" // Maximum number of samples where index_haplotypes_with_paths can still assign // distinct path IDs. extern const size_t MAX_PATH_IDS; /* Index the haplotypes in the graph. This version requires that index.payload_size() > 0. It gets the first index.payload_size() - 1 words of payload from the get_payload function and uses the last word to store the the set of haplotypes containing the minimizer hit. This uses PathIdMap to map path names to integers in [0, 64). Depending on the GBWT metadata, the mapping may be based on (sample, contig, haplotype), (sample, haplotype), or (sample). If there are too many samples (more than MAX_PATH_IDS), all paths map to 0. The fields used in the mapping are stored in the minimizer index tag PATH_NAME_FIELDS_TAG. The number of threads can be set through OpenMP. If the graph contains GraphName information, it will be copied to the minimizer index. Throws std::runtime_error if index.payload_size() == 0. */ template void index_haplotypes_with_paths ( const GBZ& gbz, MinimizerIndex& index, const std::function& get_payload ); //------------------------------------------------------------------------------ /* Returns all canonical kmers in the string specified by the iterators. A canonical kmer is the smallest of a kmer and its reverse complement. The return value is a vector of kmers sorted by their offsets. */ template std::vector> canonical_kmers(std::string::const_iterator begin, std::string::const_iterator end, size_t k); /* Returns all canonical kmers in the string. A canonical kmer is the smaller of a kmer and its reverse complement. The return value is a vector of kmers sorted by their offsets. */ template std::vector> canonical_kmers(const std::string& seq, size_t k) { return canonical_kmers(seq.begin(), seq.end(), k); } //------------------------------------------------------------------------------ /* Index the haplotypes in the graph. Insert the kmers into the provided index if include(key) returns true. The number of threads can be set through OpenMP. This is mostly intended for indexes without payload. If payload size is nonzero, the payload will be empty. */ template void build_kmer_index(const GBWTGraph& graph, KmerIndex& index, size_t k, const std::function& include); /* Index the haplotypes in the graph. Insert the kmers into the provided indexes according to the middle base. The number of threads can be set through OpenMP. This is mostly intended for indexes without payload. If payload size is nonzero, the payload will be empty. Throws std::runtime_error if the indexes do not have the same payload size. */ template void build_kmer_indexes(const GBWTGraph& graph, std::array, 4>& indexes, size_t k); /* Returns all kmers in the haplotypes with more than threshold hits in the graph in sorted order. Considers a kmer and its reverse complement the same. The number of threads can be set through OpenMP. If the number of kmers can be estimated in advance, providing a hash table size can save time and memory. There are two versions of this algorithm: * The fast version (default) uses four kmer indexes with the given hash table size and inserts each kmer into the index determined by the middle base. This version uses more memory and parallelizes better. * The space-efficient version does four passes over the graph. In each pass, it only considers the kmers with a specific middle base. This versions uses less memory, is slower, and does not parallelize as well. */ template std::vector frequent_kmers(const GBWTGraph& graph, size_t k, size_t threshold, bool space_efficient, size_t hash_table_size = KmerIndex::INITIAL_CAPACITY); //------------------------------------------------------------------------------ } // namespace gbwtgraph #endif // GBWTGRAPH_CONSTRUCTION_H jltsiren-gbwtgraph-e27bc43/include/gbwtgraph/internal.h000066400000000000000000000316161524136466400233610ustar00rootroot00000000000000#ifndef GBWTGRAPH_INTERNAL_H #define GBWTGRAPH_INTERNAL_H #include #include #include #include #include #include /* internal.h: Internal support structures. */ namespace gbwtgraph { //------------------------------------------------------------------------------ /* A buffered TSV writer for single-threaded situations. */ struct TSVWriter { explicit TSVWriter(std::ostream& out); ~TSVWriter(); void put(char c) { this->buffer.push_back(c); if(this->buffer.size() >= BUFFER_SIZE) { this->flush(); } } void newline() { this->put('\n'); } void newfield() { this->put('\t'); } void write(std::string_view view); void write(size_t value) { std::string str = std::to_string(value); this->write(str); } void flush(); // Buffer this many bytes; constexpr static size_t BUFFER_SIZE = 4 * 1048576; std::vector buffer; std::ostream& out; }; //------------------------------------------------------------------------------ /* A buffered TSV writer where the buffer grows as necessary and must be flushed explicitly. Intended for multi-threaded situations. */ struct ManualTSVWriter { explicit ManualTSVWriter(std::ostream& out); void put(char c) { this->buffer.push_back(c); } void newline() { this->put('\n'); } void newfield() { this->put('\t'); } void write(std::string_view view) { this->buffer.insert(this->buffer.end(), view.begin(), view.end()); } void write(size_t value) { std::string str = std::to_string(value); this->write(str); } bool full() const { return this->buffer.size() >= BUFFER_FULL; } void flush(); // Buffer this many bytes; constexpr static size_t BUFFER_SIZE = 4 * 1048576; // The buffer is (almost) full at this size. constexpr static size_t BUFFER_FULL = BUFFER_SIZE - 4096; std::vector buffer; std::ostream& out; }; //------------------------------------------------------------------------------ /* In the following, the writer can be either TSVWriter or ManualTSVWriter. We only deal with nodes and edges (rather than segments and links). We further assume that the graph is small enough that caching segment names is not necessary. */ // Write a GFA header to the given writer. // Set `reference_samples = nullptr` if there are no reference samples. template void write_gfa_header(Writer& writer, const std::string* reference_samples) { writer.put('H'); writer.newfield(); writer.write(std::string("VN:Z:1.1")); if(reference_samples != nullptr) { writer.newfield(); writer.write(REFERENCE_SAMPLE_LIST_GFA_TAG); writer.put(':'); writer.put('Z'); writer.put(':'); writer.write(*reference_samples); } writer.newline(); } // Write a node as a GFA segment to the given writer. template void write_gfa_segment(Writer& writer, nid_t node_id, std::string_view sequence) { writer.put('S'); writer.newfield(); writer.write(node_id); writer.newfield(); writer.write(sequence); writer.newline(); } // Write an edge as a GFA link to the given writer. template void write_gfa_link(Writer& writer, gbwt::node_type from, gbwt::node_type to) { writer.put('L'); writer.newfield(); writer.write(gbwt::Node::id(from)); writer.newfield(); writer.put(gbwt::Node::is_reverse(from) ? '-' : '+'); writer.newfield(); writer.write(gbwt::Node::id(to)); writer.newfield(); writer.put(gbwt::Node::is_reverse(to) ? '-' : '+'); writer.newfield(); writer.write("0M"); writer.newline(); } // Write a path as a GFA walk to the given writer. // Set `weight = nullptr` and/or `cigar = nullptr` if the information is not available. template void write_gfa_walk( Writer& writer, const gbwt::vector_type& path, const gbwt::FullPathName& path_name, size_t length, const size_t* weight, const std::string* cigar ) { writer.put('W'); writer.newfield(); writer.write(path_name.sample_name); writer.newfield(); // NOTE: It was a mistake to define NO_PHASE as unsigned -1. // GFA uses 0, which is much more convenient. size_t haplotype = (path_name.haplotype == GBWTGraph::NO_PHASE ? 0 : path_name.haplotype); writer.write(haplotype); writer.newfield(); writer.write(path_name.contig_name); writer.newfield(); writer.write(path_name.offset); writer.newfield(); writer.write(path_name.offset + length); writer.newfield(); for(gbwt::node_type node : path) { writer.put((gbwt::Node::is_reverse(node) ? '<' : '>')); writer.write(gbwt::Node::id(node)); } if(weight != nullptr) { writer.newfield(); writer.write(std::string("WT:i:")); writer.write(*weight); } if(cigar != nullptr) { writer.newfield(); writer.write(std::string("CG:Z:")); writer.write(*cigar); } writer.newline(); } //------------------------------------------------------------------------------ struct GFAGrammarIterator; /* A collection of grammar rules from GFA Q-lines. This class provides iterators over the expansions of any rule, both in forward and reverse complement orientations. TODO: Should we store the expansion length for each rule? */ class GFAGrammar { public: // The expansion of a rule is a list of (rule / segment name, is_reverse) pairs. using expansion_type = std::vector>; // A rule is a mapping from a rule name to its expansion. using rule_type = std::unordered_map::const_iterator; size_t size() const { return this->rules.size(); } bool empty() const { return this->rules.empty(); } // Inserts a new rule. Returns true if the insertion was successful // and false if the rule already existed. // No attempt is made to avoid cycles or empty expansions. bool insert(std::string&& rule_name, expansion_type&& expansion) { return this->rules.emplace(rule_name, expansion).second; } // Returns an iterator over the expansion of the given rule in the given direction. GFAGrammarIterator iter(const std::string& rule_name, bool is_reverse) const; // Returns an iterator to the expansion of the given rule, or `no_rule()` if the rule does not exist. rule_type expand(const std::string& rule_name) const { return this->rules.find(rule_name); } // Returns an iterator that indicates that the rule does not exist. rule_type no_rule() const { return this->rules.end(); } // Returns the first segment name in the expansion of the given symbol in forward orientation. // Returns the symbol itself if it is not a rule. // This can be used for assigning a rule to a graph component. std::string first_segment(const std::string& symbol) const; // Validates the grammar: // * there are no cycles; // * names do not clash with node/segment names (in the given graph); // * all names on the right side of productions exist as segments or rules; and // * all expansions are nontrivial (length >= 2). // Throws `std::runtime_error` if the grammar is invalid. void validate(const NaiveGraph& graph) const; private: std::unordered_map rules; }; struct GFAGrammarIterator { explicit GFAGrammarIterator(const GFAGrammar& grammar, const std::string& rule_name, bool is_reverse); // Returns the next segment in the expansion and its orientation, // or an empty string if the expansion is complete. std::pair next(); // Returns `true` if the iterator is empty. // An empty iterator either corresponds to a nonexistent rule or has already // returned an empty string. bool empty() const { return this->stack.empty(); } const GFAGrammar& grammar; // Stack of (grammar iterator, is_reverse, number of symbols processed). std::vector> stack; }; //------------------------------------------------------------------------------ /* A cache that stores GBWT records larger than `bytes` bytes as `DecompressedRecord` and supports faster path extraction from the index. */ struct LargeRecordCache { LargeRecordCache(const gbwt::GBWT& index, size_t bytes); size_t size() const { return this->cache.size(); } gbwt::size_type sequences() const { return this->index.sequences(); } gbwt::vector_type extract(gbwt::size_type sequence) const; const gbwt::GBWT& index; std::unordered_map cache; }; //------------------------------------------------------------------------------ // Sample (sequence offset, GBWT position) at the start of a node approximately // every `sample_interval` bp, with the first sample at offset 0. // If `length` is not nullptr, it will be set to the length of the path. // Sequence offsets are relative to the path, not the full haplotype. std::vector> sample_path_positions(const GBZ& gbz, path_handle_t path, size_t sample_interval, size_t* length = nullptr); //------------------------------------------------------------------------------ // A hasher for gbwt::PathName. struct PathNameHasher { static size_t hash(gbwt::PathName::path_name_type v) { return wang_hash_64(v); } static size_t combine(size_t h, size_t v) { h ^= v + 0x9e3779b9 + (h << 6) + (h >> 2); return h; } size_t operator() (const gbwt::PathName& key) const { size_t h = hash(key.sample); h = combine(h, hash(key.contig)); h = combine(h, hash(key.phase)); h = combine(h, hash(key.count)); return h; } }; // A hasher for GBWT search states. struct SearchStateHasher { static size_t hash(gbwt::size_type v) { return wang_hash_64(v); } static size_t combine(size_t h, size_t v) { h ^= v + 0x9e3779b9 + (h << 6) + (h >> 2); return h; } size_t operator() (const gbwt::SearchState& state) const { size_t h = hash(state.node); h = combine(h, hash(state.range.first)); h = combine(h, hash(state.range.second)); return h; } }; //------------------------------------------------------------------------------ /* Mapping from path names to integers in [0, 64). Depending on the GBWT metadata, the mapping may be based on (sample, contig, haplotype), (sample, haplotype), or (sample). If there are too many samples, all paths map to 0. Path names that map to the same integer are considered parts of the same haplotype. We can then use a 64-bit integer for storing subsets of haplotypes. The mapping depends on lexicographic order of path names and is therefore independent of the order of the paths in the GBWT index. */ struct PathIdMap { constexpr static size_t MAX_HAPLOTYPES = 64; enum class KeyType { NONE, SAMPLE, SAMPLE_HAPLOTYPE, SAMPLE_CONTIG_HAPLOTYPE }; // Build a PathIdMap from GBWT metatata. Assigns IDs to each distinguishable // combination of path name fields. If there are too many haplotypes we would // want to distinguish, produces an empty map. // // As long as there are MAX_HAPLOTYPES or fewer samples, the map will not be // empty. explicit PathIdMap(const gbwt::Metadata& metadata); // Returns the number of distinct haplotypes in the map. // Mapped values are in [0, size()). size_t size() const { return this->path_to_id.size(); } // Returns the haplotype identifier for the given path name. // Returns 0 if the identifier cannot be determined. size_t id(const gbwt::PathName& path) const { gbwt::PathName key = this->mask_name(path); auto iter = this->path_to_id.find(key); if(iter == this->path_to_id.end()) { return 0; } return iter->second; } // Returns the type of keys used in the mapping. KeyType key_type() const { return this->key; } // Returns a string representation of the given key type. static std::string key_type_str(KeyType key); std::unordered_map path_to_id; gbwt::PathName mask; KeyType key; private: gbwt::PathName mask_name(const gbwt::PathName& name) const { return gbwt::PathName ( name.sample & this->mask.sample, name.contig & this->mask.contig, name.phase & this->mask.phase, name.count & this->mask.count ); } bool build_map(const std::vector& sorted_paths); }; //------------------------------------------------------------------------------ /* Extracts the subpath corresponding to the given kmer. Assumes that the arguments are valid. Offset node_offset of node path[path_offset] is like a position returned by KmerIndex. If is_reverse is false, the kmer is on the forward strand, and the path extends forward from the position. If it is true, the kmer is on the reverse strand, and the path extends backward from the position. In the latter case, the returned path is a subpath of the reverse complement of the given path. */ std::vector extract_kmer_path(const GBWTGraph& graph, const std::vector& path, size_t path_offset, size_t node_offset, size_t k, bool is_reverse); //------------------------------------------------------------------------------ } // namespace gbwtgraph #endif // GBWTGRAPH_INTERNAL_H jltsiren-gbwtgraph-e27bc43/include/gbwtgraph/io.h000066400000000000000000000042461524136466400221530ustar00rootroot00000000000000#ifndef GBWTGRAPH_IO_H #define GBWTGRAPH_IO_H #include #include /* io.h: Internal I/O functions. */ namespace gbwtgraph { //------------------------------------------------------------------------------ namespace io { constexpr static size_t BLOCK_SIZE = 4 * 1024 * 1024; // Elements per block. // Serialize a simple element. template size_t serialize(std::ostream& out, const Element& element) { out.write(reinterpret_cast(&element), sizeof(element)); return sizeof(element); } // Load a simple element. template void load(std::istream& in, Element& element) { in.read(reinterpret_cast(&element), sizeof(element)); } // Serialize the size of a container. template size_t serialize_size(std::ostream& out, const Container& c) { size_t size = c.size(); return serialize(out, size); } // Resize the container to the serialized size. template void load_size(std::istream& in, Container& c) { size_t size = 0; load(in, size); c.resize(size); } // Serialize a vector of simple elements in blocks. template size_t serialize_vector(std::ostream& out, const std::vector& v) { size_t bytes = 0; bytes += serialize_size(out, v); // Data in blocks of BLOCK_SIZE elements. for(size_t i = 0; i < v.size(); i += BLOCK_SIZE) { size_t block_size = std::min(v.size() - i, BLOCK_SIZE); size_t byte_size = block_size * sizeof(Element); out.write(reinterpret_cast(v.data() + i), byte_size); bytes += byte_size; } return bytes; } // Load a serialized vector of simple elements. template void load_vector(std::istream& in, std::vector& v) { load_size(in, v); // Data in blocks of BLOCK_SIZE elements. for(size_t i = 0; i < v.size(); i += BLOCK_SIZE) { size_t block_size = std::min(v.size() - i, BLOCK_SIZE); size_t byte_size = block_size * sizeof(Element); in.read(reinterpret_cast(v.data() + i), byte_size); } } } // namespace io //------------------------------------------------------------------------------ } // namespace gbwtgraph #endif // GBWTGRAPH_IO_H jltsiren-gbwtgraph-e27bc43/include/gbwtgraph/minimizer.h000066400000000000000000002416311524136466400235500ustar00rootroot00000000000000#ifndef GBWTGRAPH_MINIMIZER_H #define GBWTGRAPH_MINIMIZER_H #include #include #include #include #include #include #include #include #include #include #include #include /* minimizer.h: Kmer indexes and minimizer indexes. */ namespace gbwtgraph { //------------------------------------------------------------------------------ /* A graph position encoded as a 64-bit integer. Uses the lowest 10 bits for node offset, limiting node length to 1024 bp. The next bit is used for node orientation and the remaining bits for node id. NOTE: This encoding must be consistent with the top-level MAX_NODE_LENGTH. */ struct Position { typedef std::uint64_t code_type; code_type value; // Constants that define the encoding. constexpr static size_t OFFSET_BITS = 10; constexpr static size_t ID_OFFSET = OFFSET_BITS + 1; constexpr static code_type REV_MASK = static_cast(1) << OFFSET_BITS; constexpr static code_type OFF_MASK = REV_MASK - 1; // As node id 0 is not allowed, we can use 0 as an empty position. constexpr static code_type NO_POSITION = 0; // Default constructor for technical purposes. Creates an empty position. constexpr Position() : value(NO_POSITION) {} // Returns an empty position. constexpr static Position no_pos() { return Position(); } // This reinterprets the encoded value as a Position. explicit constexpr Position(code_type value) : value(value) {} // Writes the position to the given array pointer. void write(code_type* array_ptr) const { *array_ptr = this->value; } // Converts pos_t to Position. explicit Position(pos_t pos) { this->value = (static_cast(gbwtgraph::id(pos)) << ID_OFFSET) | (static_cast(gbwtgraph::is_rev(pos)) << OFFSET_BITS) | (static_cast(gbwtgraph::offset(pos)) & OFF_MASK); } // Basic comparisons. bool operator==(Position another) const { return (this->value == another.value); } bool operator!=(Position another) const { return (this->value != another.value); } bool operator<(Position another) const { return (this->value < another.value); } bool operator>(Position another) const { return (this->value > another.value); } // Is the offset small enough to fit in the low-order bits of the encoding? static bool valid_offset(const pos_t& pos) { return (gbwtgraph::offset(pos) <= OFF_MASK); } // Returns the node identifier. nid_t id() const { return this->value >> ID_OFFSET; } // Returns the orientation. bool is_rev() const { return this->value & REV_MASK; } // Returns the node offset. size_t offset() const { return this->value & OFF_MASK; } // Decodes the position as pos_t. pos_t decode() const { return make_pos_t(this->id(), this->is_rev(), this->offset()); } }; //------------------------------------------------------------------------------ /* Kmer encoding using 2 bits per base in 64-bit integers. If two kmers have the same length, comparing the integers or tuples of integers gives the same result as lexicographic comparison of the kmers. Also provides some type definitions shared between the kmer and minimizer indexes. */ struct KmerEncoding { // The type used for storing encoded keys and values. typedef std::uint64_t code_type; // A value consisting of a position and a pointer to the payload, // which is assumed to have the same size as the index payload size. typedef std::pair value_type; // A pointer to a list of encoded values and the number of values. // Use get_value() in the index to access and decode individual values. typedef std::pair multi_value_type; // Conversion from characters to packed characters. const static std::vector CHAR_TO_PACK; // Conversion from packed characters to upper-case characters. const static std::vector PACK_TO_CHAR; // Masks for the bits used in the encoding. If kmer length is `k`, // `LOW_MASK[k]` marks the bits that are in use in the least significant word // of the encoding. `HIGH_MASK[k]` marks the bits that are in use in the // penultimate word of the encoding. const static std::vector LOW_MASK; const static std::vector HIGH_MASK; // Constants used in the encoding. constexpr static size_t FIELD_BITS = sizeof(code_type) * gbwt::BYTE_BITS; constexpr static size_t PACK_WIDTH = 2; constexpr static size_t PACK_OVERFLOW = FIELD_BITS - PACK_WIDTH; constexpr static size_t FIELD_CHARS = FIELD_BITS / PACK_WIDTH; constexpr static code_type PACK_MASK = 0x3; }; //------------------------------------------------------------------------------ /* A kmer encoded using 2 bits/character in a 64-bit integer. The encoding is only defined if all characters in the kmer are valid. The highest bit indicates whether the corresponding value in the hash table is a position or a pointer. */ struct Key64 { public: // Internal representation. typedef KmerEncoding::code_type code_type; typedef code_type value_type; code_type key; // Empty key. constexpr Key64() : key(EMPTY_KEY) {} // Reads the key starting from the given pointer. static Key64 read(const code_type* ptr) { return Key64(*ptr); } // Writes the key to the given pointer. void write(code_type* ptr) const { *ptr = this->key; } // No key, with 63 set bits. constexpr static Key64 no_key() { return Key64(NO_KEY & KEY_MASK); } // For testing. explicit constexpr Key64(code_type key) : key(key) {} // Get a representation of the actual key. value_type get_key() const { return (this->key & KEY_MASK); } // Comparisons. bool operator==(Key64 another) const { return (this->get_key() == another.get_key()); } bool operator!=(Key64 another) const { return (this->get_key() != another.get_key()); } bool operator<(Key64 another) const { return (this->get_key() < another.get_key()); } // Is the corresponding value a pointer? bool is_pointer() const { return (this->key & IS_POINTER); } void clear_pointer() { this->key &= ~IS_POINTER; } void set_pointer() { this->key |= IS_POINTER; } // Hash of the key. Do not call this directly, as the index may use a derived // hash function. size_t hash() const { return wang_hash_64(this->key & KEY_MASK); } // Move the kmer forward, with c as the next character. Update the key, assuming that // it encodes the kmer in forward orientation. void forward(size_t k, unsigned char c, size_t& valid_chars) { code_type packed = KmerEncoding::CHAR_TO_PACK[c]; if(packed > KmerEncoding::PACK_MASK) { this->key = EMPTY_KEY; valid_chars = 0; } else { this->key = ((this->key << KmerEncoding::PACK_WIDTH) | packed) & KmerEncoding::LOW_MASK[k]; valid_chars++; } } // Move the kmer forward, with c as the next character. Update the key, assuming that // it encodes the kmer in reverse orientation. void reverse(size_t k, unsigned char c) { code_type packed = KmerEncoding::CHAR_TO_PACK[c]; if(packed > KmerEncoding::PACK_MASK) { this->key = EMPTY_KEY; } else { packed ^= KmerEncoding::PACK_MASK; // The complement of the base. this->key = (packed << ((k - 1) * KmerEncoding::PACK_WIDTH)) | (this->key >> KmerEncoding::PACK_WIDTH); } } /// Encode a string of size k to a key. static Key64 encode(const std::string& sequence); /// Decode the key back to a string, given the kmer size used. std::string decode(size_t k) const; /// Returns the packed character value for kmer[i]. code_type access_raw(size_t k, size_t i) const { size_t offset = (k - 1 - i) * KmerEncoding::PACK_WIDTH; return (this->key >> offset) & KmerEncoding::PACK_MASK; } /// Returns kmer[i]. char access(size_t k, size_t i) const { return KmerEncoding::PACK_TO_CHAR[this->access_raw(k, i)]; } /// Returns the reverse complement of the kmer. Key64 reverse_complement(size_t k) const; // Required numeric constants. constexpr static std::size_t KEY_BITS = KmerEncoding::FIELD_BITS; constexpr static std::size_t KMER_LENGTH = 29; constexpr static std::size_t WINDOW_LENGTH = 11; constexpr static std::size_t SMER_LENGTH = KMER_LENGTH - WINDOW_LENGTH; constexpr static std::size_t KMER_MAX_LENGTH = 31; private: // Specific key values. Note that the highest bit is not a part of the key. constexpr static code_type EMPTY_KEY = 0; constexpr static code_type NO_KEY = std::numeric_limits::max(); constexpr static code_type KEY_MASK = NO_KEY >> 1; constexpr static code_type IS_POINTER = NO_KEY ^ KEY_MASK; // High bit. }; // Required for printing keys. std::ostream& operator<<(std::ostream& out, Key64 value); //------------------------------------------------------------------------------ /* A kmer encoded using 2 bits/character in a pair of 64-bit integers. The encoding is only defined if all characters in the kmer are valid. The highest bit indicates whether the corresponding value in the hash table is a position or a pointer. */ struct Key128 { public: // Internal representation. typedef KmerEncoding::code_type code_type; typedef std::pair value_type; code_type high, low; // Empty key. constexpr Key128() : high(EMPTY_KEY), low(EMPTY_KEY) {} // Reads the key starting from the given pointer. static Key128 read(const code_type* ptr) { return Key128(ptr[0], ptr[1]); } // Writes the key to the given pointer. void write(code_type* ptr) const { ptr[0] = this->high; ptr[1] = this->low; } // No key, with 127 set bits. constexpr static Key128 no_key() { return Key128(NO_KEY & KEY_MASK, NO_KEY); } // For testing. explicit constexpr Key128(code_type key) : high(EMPTY_KEY), low(key) {} // For testing. constexpr Key128(code_type high, code_type low) : high(high), low(low) {} // Get a representation of the actual key. value_type get_key() const { return value_type(this->high & KEY_MASK, this->low); } // Comparisons. bool operator==(Key128 another) const { return (this->get_key() == another.get_key()); } bool operator!=(Key128 another) const { return (this->get_key() != another.get_key()); } bool operator<(Key128 another) const { return (this->get_key() < another.get_key()); } // Is the corresponding value a pointer? bool is_pointer() const { return (this->high & IS_POINTER); } void clear_pointer() { this->high &= ~IS_POINTER; } void set_pointer() { this->high |= IS_POINTER; } // Hash of the key. Essentially boost::hash_combine. Do not call this directly, // as the index may use a derived hash function. size_t hash() const { size_t result = wang_hash_64(this->high & KEY_MASK); result ^= wang_hash_64(this->low) + 0x9e3779b9 + (result << 6) + (result >> 2); return result; } // Move the kmer forward, with c as the next character. Update the key, assuming that // it encodes the kmer in forward orientation. void forward(size_t k, unsigned char c, size_t& valid_chars) { code_type packed = KmerEncoding::CHAR_TO_PACK[c]; if(packed > KmerEncoding::PACK_MASK) { this->high = EMPTY_KEY; this->low = EMPTY_KEY; valid_chars = 0; } else { this->high = ((this->high << KmerEncoding::PACK_WIDTH) | (this->low >> KmerEncoding::PACK_OVERFLOW)) & KmerEncoding::HIGH_MASK[k]; this->low = ((this->low << KmerEncoding::PACK_WIDTH) | packed) & KmerEncoding::LOW_MASK[k]; valid_chars++; } } // Move the kmer forward, with c as the next character. Update the key, assuming that // it encodes the kmer in reverse orientation. void reverse(size_t k, unsigned char c) { code_type packed = KmerEncoding::CHAR_TO_PACK[c]; if(packed > KmerEncoding::PACK_MASK) { this->high = EMPTY_KEY; this->low = EMPTY_KEY; } else { packed ^= KmerEncoding::PACK_MASK; // The complement of the base. if(k > KmerEncoding::FIELD_CHARS) { this->low = ((this->high & KmerEncoding::PACK_MASK) << KmerEncoding::PACK_OVERFLOW) | (this->low >> KmerEncoding::PACK_WIDTH); this->high = (packed << ((k - KmerEncoding::FIELD_CHARS - 1) * KmerEncoding::PACK_WIDTH)) | (this->high >> KmerEncoding::PACK_WIDTH); } else // The entire kmer is in the lower part of the key. { this->low = (packed << ((k - 1) * KmerEncoding::PACK_WIDTH)) | (this->low >> KmerEncoding::PACK_WIDTH); } } } /// Encode a string of size k to a key. static Key128 encode(const std::string& sequence); /// Decode the key back to a string, given the kmer size used. std::string decode(size_t k) const; /// Returns the packed character value for kmer[i]. code_type access_raw(size_t k, size_t i) const { size_t offset = (k - 1 - i) * KmerEncoding::PACK_WIDTH; code_type shifted; if(offset >= KmerEncoding::FIELD_BITS) { shifted = this->high >> (offset - KmerEncoding::FIELD_BITS); } else { shifted = this->low >> offset; } return shifted & KmerEncoding::PACK_MASK; } /// Returns kmer[i]. char access(size_t k, size_t i) const { return KmerEncoding::PACK_TO_CHAR[this->access_raw(k, i)]; } /// Returns the reverse complement of the kmer. Key128 reverse_complement(size_t k) const; // Required numeric constants. constexpr static std::size_t KEY_BITS = 2 * KmerEncoding::FIELD_BITS; constexpr static std::size_t KMER_LENGTH = 39; constexpr static std::size_t WINDOW_LENGTH = 15; constexpr static std::size_t SMER_LENGTH = KMER_LENGTH - WINDOW_LENGTH; constexpr static std::size_t KMER_MAX_LENGTH = 63; private: // Specific key values. Note that the highest bit is not a part of the key. constexpr static code_type EMPTY_KEY = 0; constexpr static code_type NO_KEY = std::numeric_limits::max(); constexpr static code_type KEY_MASK = NO_KEY >> 1; constexpr static code_type IS_POINTER = NO_KEY ^ KEY_MASK; // High bit. }; // Required for printing keys. std::ostream& operator<<(std::ostream& out, Key128 value); //------------------------------------------------------------------------------ // Sequence offset in kmer extraction. typedef std::uint32_t offset_type; /* The sequence offset of a kmer is the base that corresponds to the start of the kmer. If the kmer is a reverse complement, that means the last base of the substring covered by the kmer. Values stored in a kmer index are graph positions for the start of the kmer used as the key. The kmer always extends forward from the position. When a kmer is in reverse orientation in the query sequence, the graph positions from the index must be flipped with reverse_base_pos(). The last base of the forward orientation of the query sequence covered by the kmer then matches the flipped graph position. */ template struct Kmer { typedef KeyType key_type; key_type key; // Encoded kmer. size_t hash; // Hash of the kmer. offset_type offset; // Sequence offset. bool is_reverse; // The kmer is a reverse complement. // Is the kmer empty? bool empty() const { return (this->key == key_type::no_key()); } // Sort by (offset, !is_reverse). When the offsets are equal, a reverse complement // kmer is earlier in the sequence than a forward kmer. bool operator<(const Kmer& another) const { return ((this->offset < another.offset) || (this->offset == another.offset && this->is_reverse > another.is_reverse)); } bool operator==(const Kmer& another) const { return (this->key == another.key && this->offset == another.offset && this->is_reverse == another.is_reverse); } bool operator!=(const Kmer& another) const { return !(this->operator==(another)); } }; template std::ostream& operator<<(std::ostream& out, const Kmer& kmer) { out << "(" << kmer.key << ", " << (kmer.is_reverse ? "-" : "+") << kmer.offset << ")"; return out; } //------------------------------------------------------------------------------ // Forward declarations for friend declarations. class MinimizerHeader; template class MinimizerIndex; /* This is a general-purpose kmer index parameterized with a key type. The key can be either Key64 (64 bits, up to 31 bp) or Key128 (128 bits, up to 63 bp). Values consist of a graph position (Position) and payload. Payload size (in 64-bit words) is defined when the index is constructed and can be 0. The kmers are stored in a hash table with 2^n cells that uses quadratic probing. If a kmer is associated with a single value, it is stored directly within the hash table. When there are multiple values, the hash table stores a pointer to a vector of values. The values are sorted by position. Limitations: * The index does not support serialization. * Any key not equal to KeyType::no_key() can be inserted into the index. There are no guarantees that all kmers have the same length. */ template class KmerIndex { public: typedef KmerEncoding::code_type code_type; typedef KeyType key_type; // A cell consists of cell_size() words that encode a key and an // encoded value using value_size() words. The encoded value may // contain either a pointer to a vector of values or a single value. // A value consists of a graph position (Position) and 0 to // MAX_PAYLOAD_SIZE words of payload. For any given index, you can // get payload size using payload_size(). typedef std::pair cell_type; typedef std::pair const_cell_type; // A value consisting of a position and a pointer to the payload, // which is assumed to have the same size as the index payload size. typedef KmerEncoding::value_type value_type; // A pointer to a list of encoded values and the number of values. // Use get_value() to access and decode individual values. typedef KmerEncoding::multi_value_type multi_value_type; // Public constants. constexpr static size_t INITIAL_CAPACITY = 1024; constexpr static double MAX_LOAD_FACTOR = 0.77; constexpr static size_t MAX_PAYLOAD_SIZE = 15; // Must be consistent with the number of bits reserved in MinimizerHeader. constexpr static size_t KEY_SIZE = sizeof(key_type) / sizeof(code_type); constexpr static size_t POS_SIZE = sizeof(Position) / sizeof(code_type); // Returns the size of the smallest hash table that can hold this many keys. static size_t minimum_size(size_t keys) { size_t bound = std::ceil(keys / MAX_LOAD_FACTOR); size_t capacity = INITIAL_CAPACITY; while(capacity < bound) { capacity *= 2; } return capacity; } //------------------------------------------------------------------------------ /* Constructors, destructors, and object handling. */ // Throws std::runtime_error if payload size exceeds MAX_PAYLOAD_SIZE. explicit KmerIndex(size_t payload_size) : KmerIndex(INITIAL_CAPACITY, payload_size) { } // Hash table size must be a power of 2 and at least INITIAL_CAPACITY. // Otherwise it will revert to INITIAL_CAPACITY. // Throws std::runtime_error if payload size exceeds MAX_PAYLOAD_SIZE. KmerIndex(size_t cell_count, size_t payload_size) : keys(0), values(0), unique(0), payload(payload_size), downweight(0), frequent_kmers() { if(sdsl::bits::cnt(cell_count) != 1) { std::cerr << "KmerIndex::KmerIndex(): Cell count (" << cell_count << ") must be a power of 2; reverting to " << INITIAL_CAPACITY << std::endl; cell_count = INITIAL_CAPACITY; } if(cell_count < INITIAL_CAPACITY) { std::cerr << "KmerIndex::KmerIndex(): Cell count (" << cell_count << ") is too small; reverting to " << INITIAL_CAPACITY << std::endl; cell_count = INITIAL_CAPACITY; } this->max_keys = cell_count * MAX_LOAD_FACTOR; if(payload_size > MAX_PAYLOAD_SIZE) { std::string msg = "KmerIndex::KmerIndex(): Payload size (" + std::to_string(payload_size) + ") exceeds maximum (" + std::to_string(MAX_PAYLOAD_SIZE) + ")"; throw std::runtime_error(msg); } this->hash_table = empty_hash_table(cell_count, this->cell_size()); } KmerIndex(const KmerIndex& source) { this->copy(source); } KmerIndex(KmerIndex&& source) noexcept { *this = std::move(source); } ~KmerIndex() { this->clear(); } void swap(KmerIndex& another) noexcept { if(&another == this) { return; } std::swap(this->keys, another.keys); std::swap(this->max_keys, another.max_keys); std::swap(this->values, another.values); std::swap(this->unique, another.unique); std::swap(this->payload, another.payload); this->hash_table.swap(another.hash_table); std::swap(this->downweight, another.downweight); this->frequent_kmers.swap(another.frequent_kmers); } KmerIndex& operator=(const KmerIndex& source) { if(&source != this) { this->copy(source); } return *this; } KmerIndex& operator=(KmerIndex&& source) noexcept { if(&source != this) { this->keys = std::move(source.keys); this->max_keys = std::move(source.max_keys); this->values = std::move(source.values); this->unique = std::move(source.unique); this->payload = std::move(source.payload); this->hash_table = std::move(source.hash_table); this->downweight = std::move(source.downweight); this->frequent_kmers = std::move(source.frequent_kmers); } return *this; } // For testing. bool operator==(const KmerIndex& another) const { if(this->keys != another.keys) { return false; } if(this->max_keys != another.max_keys) { return false; } if(this->values != another.values) { return false; } if(this->unique != another.unique) { return false; } if(this->payload != another.payload) { return false; } if(this->hash_table.size() != another.hash_table.size()) { return false; } for(size_t array_offset = 0; array_offset < this->hash_table.size(); array_offset += this->cell_size()) { const_cell_type a = this->const_cell(array_offset); const_cell_type b = another.const_cell(array_offset); if(a.first != b.first) { return false; } // Key comparison. if(a.first.is_pointer() != b.first.is_pointer()) { return false; } multi_value_type a_values = this->get_values(a); multi_value_type b_values = another.get_values(b); if(a_values.second != b_values.second) { return false; } for(size_t j = 0; j < a_values.second * this->value_size(); j++) { if(a_values.first[j] != b_values.first[j]) { return false; } } } if(this->downweight != another.downweight) { return false; } if(this->frequent_kmers != another.frequent_kmers) { return false; } return true; } bool operator!=(const KmerIndex& another) const { return !(this->operator==(another)); } //------------------------------------------------------------------------------ /* Statistics and iteration. */ // Number of keys in the index. size_t size() const { return this->keys; } // Is the index empty? bool empty() const { return (this->size() == 0); } // Number of values (kmer occurrences) in the index. size_t number_of_values() const { return this->values; } // Size of the hash table in cells. size_t cell_count() const { return this->hash_table.size() / this->cell_size(); } // Number of keys that can fit into the hash table. Exceeding it will initiate rehashing. size_t capacity() const { return this->max_keys; } // Current load factor of the hash table. double load_factor() const { return static_cast(this->size()) / static_cast(this->cell_count()); } // Number of kmers with a single occurrence. size_t unique_keys() const { return this->unique; } // Returns the size of a payload in words. size_t payload_size() const { return this->payload; } // Returns the size of a value in words. size_t value_size() const { return POS_SIZE + this->payload_size(); } // Returns the size of a cell in words. size_t cell_size() const { return KEY_SIZE + POS_SIZE + this->payload_size(); } // Call `callback` for every non-empty hash table cell. // The callback gets the key and encoded values. void for_each_kmer(const std::function& callback) const { for(size_t array_offset = 0; array_offset < this->hash_table.size(); array_offset += this->cell_size()) { const_cell_type cell = this->const_cell(array_offset); if(cell.first == key_type::no_key()) { continue; } multi_value_type values = this->get_values(cell); callback(cell.first, values); } } // Call `callback` for every non-empty hash table cell. // The callback gets the key and encoded values. // If callback returns false, then stop iterating. // Returns false if the iteration stopped early, true otherwise. bool for_each_kmer(const std::function& callback) const { for(size_t array_offset = 0; array_offset < this->hash_table.size(); array_offset += this->cell_size()) { const_cell_type cell = this->const_cell(array_offset); if(cell.first == key_type::no_key()) { continue; } multi_value_type values = this->get_values(cell); if(!callback(cell.first, values)) { return false; } } return true; } //------------------------------------------------------------------------------ /* Operations. */ /* Inserts (key, (position, payload)) into the index if it is not already there. Does not insert keys equal to key_type::no_key() or positions equal to Position::no_pos(). */ void insert(key_type key, value_type value) { this->insert(key, value, this->hash(key)); } /* Inserts (key, (position, payload)) into the index if it is not already there using the given hash value. Does not insert keys equal to key_type::no_key() or positions equal to Position::no_pos(). */ void insert(key_type key, value_type value, size_t hash) { if(key == key_type::no_key() || value.first == Position::no_pos()) { return; } size_t array_offset = this->find_offset(key, hash); cell_type cell = this->cell(array_offset); if(cell.first == key_type::no_key()) { this->insert_new(key, value, cell); } else if(cell.first == key) { this->append(value, cell); } } /* Like insert(), but if (key, value.first) is already present in the index and payload_size() > 0, the LAST word of the existing entry's payload is OR'd with the LAST word of the new value's payload. The position and the other payload words (e.g. zipcode) are left unchanged. This is the path-aware variant used by index_haplotypes_with_paths(), where multiple kmer-paths can produce the same (key, Position) pair (typically at node boundaries when alternative successor nodes share their first k-1 bp); the haplotype bitmap stored in the last payload word must be the OR of all contributing kmer-paths, not just the first inserter's. PRECONDITION: the first payload_size()-1 words of the payload must be a pure function of value.first (the Position) — i.e. all callers that may produce the same (key, Position) pair must compute identical leading payload words. index_haplotypes_with_paths() satisfies this because it derives those words from get_payload(pos_t), which is invoked with the same pos for both inserts. Only the last word (the haplotype bitmap) varies with the kmer-path, and that is what gets merged. Other consumers of this API must respect the precondition. For payload_size() == 0 this falls back to insert() (no last word to merge). */ void insert_with_last_payload_word_or(key_type key, value_type value) { this->insert_with_last_payload_word_or(key, value, this->hash(key)); } void insert_with_last_payload_word_or(key_type key, value_type value, size_t hash) { if(key == key_type::no_key() || value.first == Position::no_pos()) { return; } if(this->payload_size() == 0) { this->insert(key, value, hash); return; } size_t array_offset = this->find_offset(key, hash); cell_type cell = this->cell(array_offset); if(cell.first == key_type::no_key()) { this->insert_new(key, value, cell); } else if(cell.first == key) { this->append_or_merge_last_word(value, cell); } } // Returns the occurrence count for the kmer. size_t count(key_type key) const { return this->count(key, this->hash(key)); } // Returns the occurrence count for the kmer with the given hash value. size_t count(key_type key, size_t hash) const { if(key == key_type::no_key()) { return 0; } size_t array_offset = this->find_offset(key, hash); const_cell_type cell = this->const_cell(array_offset); if(cell.first == key) { multi_value_type values = this->get_values(cell); return values.second; } return 0; } /* Returns the sorted list of the occurrences for the given kmer and the number of occurrences. Any insertions into the index may invalidate the returned pointer. */ multi_value_type find(key_type key) const { return this->find(key, this->hash(key)); } /* Returns the sorted list of the occurrences for the kmer with the given hash value and the number of occurrences. Any insertions into the index may invalidate the returned pointer. */ multi_value_type find(key_type key, size_t hash) const { if(key == key_type::no_key()) { return multi_value_type(nullptr, 0); } size_t array_offset = this->find_offset(key, hash); const_cell_type cell = this->const_cell(array_offset); if(cell.first == key) { return this->get_values(cell); } return multi_value_type(nullptr, 0); } /* Decodes the given value from the given list of values. Returns an empty value with no position and an empty payload if the index is out of bounds. */ value_type get_value(multi_value_type values, size_t index) const { if(index >= values.second) { return std::make_pair(Position::no_pos(), nullptr); } const code_type* ptr = values.first + index * this->value_size(); Position pos(*ptr); const code_type* payload = ptr + POS_SIZE; return std::make_pair(pos, payload); } //------------------------------------------------------------------------------ /* Internal implementation. */ private: size_t keys, max_keys; size_t values, unique; size_t payload; // Payload size in words. std::vector hash_table; // Downweight hashes for frequent kmers. Note that frequent_kmers is a hash set similar to hash_table. size_t downweight; std::vector frequent_kmers; // Needed for serialization. friend class MinimizerHeader; friend class MinimizerIndex; size_t serialize_hash_table(std::ostream& out) const { size_t words_in_block = io::BLOCK_SIZE * this->cell_size(); size_t cell_size = this->cell_size(); size_t bytes = 0; bytes += io::serialize_size(out, this->hash_table); // Data in blocks of BLOCK_SIZE cells. Replace pointers with empty values // to ensure that the serialization is deterministic. for(size_t array_offset = 0; array_offset < this->hash_table.size(); array_offset += words_in_block) { size_t current_block_size = std::min(this->hash_table.size() - array_offset, words_in_block); size_t byte_size = current_block_size * sizeof(code_type); std::vector buffer(this->hash_table.begin() + array_offset, this->hash_table.begin() + array_offset + current_block_size); for(size_t buffer_offset = 0; buffer_offset < buffer.size(); buffer_offset += cell_size) { cell_type cell = hash_table_cell(buffer, buffer_offset); if(cell.first.is_pointer()) { this->clear_value(cell, false); } } out.write(reinterpret_cast(buffer.data()), byte_size); bytes += byte_size; } return bytes; } static std::vector empty_hash_table(size_t capacity, size_t cell_size) { std::vector result = std::vector(capacity * cell_size, 0); key_type no_key = key_type::no_key(); Position no_pos = Position::no_pos(); for(size_t array_offset = 0; array_offset < result.size(); array_offset += cell_size) { no_key.write(&result[array_offset]); no_pos.write(&result[array_offset + KEY_SIZE]); } return result; } void copy(const KmerIndex& source) { this->clear(); this->keys = source.keys; this->max_keys = source.max_keys; this->values = source.values; this->unique = source.unique; this->payload = source.payload; // First make a shallow copy and then copy the occurrence lists. this->hash_table = source.hash_table; for(size_t array_offset = 0; array_offset < this->hash_table.size(); array_offset += this->cell_size()) { cell_type cell = this->cell(array_offset); std::vector* old = get_vector_pointer(cell); if(old != nullptr) { this->write_vector_pointer(new std::vector(*old), cell); } } this->downweight = source.downweight; this->frequent_kmers = source.frequent_kmers; } // Clears the value stored in the given cell. // In the destructive mode, this also deletes any pointer stored in the cell // and clears the pointer bit in the key. void clear_value(cell_type cell, bool destructive) const { if(destructive && cell.first.is_pointer()) { std::vector* ptr = get_vector_pointer(cell); delete ptr; cell.first.clear_pointer(); } Position::no_pos().write(cell.second); for(size_t value_offset = 1; value_offset < this->value_size(); value_offset++) { cell.second[value_offset] = 0; } } // Delete all pointers in the hash table and replace them with empty values. void clear() { for(size_t array_offset = 0; array_offset < this->hash_table.size(); array_offset += this->cell_size()) { cell_type cell = this->cell(array_offset); this->clear_value(cell, true); } } // Returns a mutable view of the cell starting at the given array offset // in the given array. static cell_type hash_table_cell(std::vector& array, size_t array_offset) { key_type* key = reinterpret_cast(&array[array_offset]); code_type* values = &array[array_offset + KEY_SIZE]; return cell_type(*key, values); } // Returns a mutable view of the cell starting at the given array offset. cell_type cell(size_t array_offset) { return hash_table_cell(this->hash_table, array_offset); } // Returns an immutable view of the cell starting at the given array offset // in the given array. static const_cell_type hash_table_const_cell(const std::vector& array, size_t array_offset) { key_type key = key_type::read(&array[array_offset]); const code_type* values = &array[array_offset + KEY_SIZE]; return const_cell_type(key, values); } // Returns an immutable view of the cell starting at the given array offset. const_cell_type const_cell(size_t array_offset) const { return hash_table_const_cell(this->hash_table, array_offset); } // Converts a mutable cell to an immutable cell. static const_cell_type const_cell(cell_type cell) { return const_cell_type(cell.first, cell.second); } // Returns the values stored in given cell. The cell is assumed to be non-empty. multi_value_type get_values(const_cell_type cell) const { if(cell.first.is_pointer()) { const std::vector* ptr = reinterpret_cast*>(cell.second[0]); return multi_value_type(ptr->data(), ptr->size() / this->value_size()); } else { return multi_value_type(cell.second, 1); } } // Returns the vector storing the values for the hash table cell starting at the // given offset of the underlying array. This returns nullptr if the cell is empty // or if it contains a single value. static std::vector* get_vector_pointer(cell_type cell) { if(!cell.first.is_pointer()) { return nullptr; } return reinterpret_cast*>(cell.second[0]); } // Returns the vector storing the values for the hash table cell starting at the // given offset of the underlying array. This returns nullptr if the cell is empty // or if it contains a single value. static const std::vector* get_vector_pointer(const_cell_type cell) { if(!cell.first.is_pointer()) { return nullptr; } return reinterpret_cast*>(cell.second[0]); } // Find the array offset for the key with the given hash value. // The cell may either be empty or contain the key. size_t find_offset(key_type key, size_t hash) const { size_t cell_count = this->cell_count(); size_t cell_offset = hash & (cell_count - 1); size_t array_offset = cell_offset * this->cell_size(); for(size_t attempt = 0; attempt < cell_count; attempt++) { const_cell_type cell = this->const_cell(array_offset); if(cell.first == key_type::no_key() || cell.first == key) { return array_offset; } // Quadratic probing with triangular numbers. cell_offset = (cell_offset + attempt + 1) & (cell_count - 1); array_offset = cell_offset * this->cell_size(); } // This should not happen. std::cerr << "KmerIndex::find_offset(): Cannot find the offset for key " << key << std::endl; assert(false); return 0; } // Encodes the given value to the given array pointer. void encode_value(value_type value, code_type* array_ptr) const { value.first.write(array_ptr); for(size_t i = 0; i < this->payload_size(); i++) { array_ptr[i + POS_SIZE] = value.second[i]; } } // Stores the given vector pointer in the given cell. Assumes that // the cell does not already contain a pointer. void write_vector_pointer(std::vector* ptr, cell_type cell) const { static_assert(sizeof(std::vector*) == sizeof(code_type), "Pointer size does not match code_type size."); cell.second[0] = reinterpret_cast(ptr); for(size_t i = 1; i < this->value_size(); i++) { cell.second[i] = 0; } } // Swaps the values starting at the given pointers, assuming that the values do not overlap. void swap_values(code_type* a, code_type* b) const { for(size_t i = 0; i < this->value_size(); i++) { std::swap(a[i], b[i]); } } // Encodes the given value, appends it to the given vector, and bubbles it down to its correct position. void insert_value(value_type value, std::vector& vec) const { size_t value_size = this->value_size(); assert(vec.size() % value_size == 0); size_t curr_offset = vec.size(); vec.resize(vec.size() + value_size, 0); this->encode_value(value, vec.data() + curr_offset); while(curr_offset >= value_size) { size_t prev_offset = curr_offset - value_size; Position candidate(vec[prev_offset]); if(candidate > value.first) { this->swap_values(vec.data() + prev_offset, vec.data() + curr_offset); } else { break; } curr_offset = prev_offset; } } // Insert (key, value) the given cell, which is assumed to be empty. // Rehashing may be necessary, which invalidates the cell. void insert_new(key_type key, value_type value, cell_type cell) { cell.first = key; this->encode_value(value, cell.second); this->keys++; this->values++; this->unique++; if(this->size() > this->capacity()) { this->rehash(); } } // Inserts the given value into the given cell, which is assumed to // already contain a key. void append(value_type value, cell_type cell) { if(this->contains(const_cell(cell), value.first)) { return; } std::vector* ptr = get_vector_pointer(cell); if(ptr != nullptr) { // We already have multiple occurrences. this->insert_value(value, *ptr); } else { // Now we have two occurrences of this kmer. std::vector* occs = new std::vector(cell.second, cell.second + this->value_size()); this->insert_value(value, *occs); this->write_vector_pointer(occs, cell); cell.first.set_pointer(); this->unique--; } this->values++; } // Like append(), but if value.first is already in the cell, OR the last word of // the new payload into the existing entry's last word instead of dropping the // new value. Caller must guarantee payload_size() > 0. void append_or_merge_last_word(value_type value, cell_type cell) { // Locate value.first in the cell's sorted occurrence list. We mirror the // binary search in contains() but also compute the array offset of the match. multi_value_type values = this->get_values(const_cell(cell)); size_t low = 0, high = values.second; while(low < high) { size_t mid = low + (high - low) / 2; size_t mid_offset = mid * this->value_size(); Position candidate(values.first[mid_offset]); if(candidate < value.first) { low = mid + 1; } else if(candidate > value.first) { high = mid; } else { // Found. Cast away const: values.first either points into hash_table[] // (mutable storage) or into a heap-allocated std::vector // owned by the cell (also mutable). Other code paths (e.g. clear_value) // mutate the same memory through cell.second / vector pointers. code_type* dst = const_cast(values.first + mid_offset + POS_SIZE); dst[this->payload_size() - 1] |= value.second[this->payload_size() - 1]; return; } } // Position not in the cell: regular append (mirrors append() body). std::vector* ptr = get_vector_pointer(cell); if(ptr != nullptr) { this->insert_value(value, *ptr); } else { std::vector* occs = new std::vector(cell.second, cell.second + this->value_size()); this->insert_value(value, *occs); this->write_vector_pointer(occs, cell); cell.first.set_pointer(); this->unique--; } this->values++; } // Returns true if the given cell contains the given graph position. bool contains(const_cell_type cell, Position pos) const { multi_value_type values = this->get_values(cell); size_t low = 0, high = values.second; while(low < high) { size_t mid = low + (high - low) / 2; size_t array_offset = mid * this->value_size(); Position candidate(values.first[array_offset]); if(candidate < pos) { low = mid + 1; } else if(candidate > pos) { high = mid; } else { return true; } } return false; } // Get the hash value for the key. If downweighting is not in use, or // the kmer is not frequent, the hash value reported by the key is used // directly. Otherwise we downweight the hash value by the specified // number of iterations. size_t hash(key_type key) const { if(this->frequent_kmers.empty()) { return key.hash(); } size_t h = key.hash(); size_t offset = h & (this->frequent_kmers.size() - 1); for(size_t attempt = 0; attempt < this->frequent_kmers.size(); attempt++) { if(this->frequent_kmers[offset] == key_type::no_key()) { break; } if(this->frequent_kmers[offset] == key) { h = std::numeric_limits::max() - h; for(size_t i = 0; i < this->downweight; i++) { h = h >> 32; h = h * h; } return std::numeric_limits::max() - h; } // Quadratic probing with triangular numbers. offset = (offset + attempt + 1) & (this->frequent_kmers.size() - 1); } return h; } // Double the size of the hash table. void rehash() { // Reinitialize with a larger hash table. size_t cell_size = this->cell_size(); std::vector old_hash_table = empty_hash_table(this->cell_count() * 2, cell_size); this->hash_table.swap(old_hash_table); this->max_keys = this->cell_count() * MAX_LOAD_FACTOR; // Move the keys to the new hash table. for(size_t old_array_offset = 0; old_array_offset < old_hash_table.size(); old_array_offset += cell_size) { cell_type source = hash_table_cell(old_hash_table, old_array_offset); if(source.first == key_type::no_key()) { continue; } size_t offset = this->find_offset(source.first, this->hash(source.first)); cell_type target = this->cell(offset); assert(target.first == key_type::no_key()); target.first = source.first; this->swap_values(target.second, source.second); } } // Inserts a key into the frequent kmers hash table. void insert_frequent(key_type key) { size_t h = key.hash(); size_t offset = h & (this->frequent_kmers.size() - 1); for(size_t attempt = 0; attempt < this->frequent_kmers.size(); attempt++) { if(this->frequent_kmers[offset] == key) { return; } if(this->frequent_kmers[offset] == key_type::no_key()) { this->frequent_kmers[offset] = key; return; } // Quadratic probing with triangular numbers. offset = (offset + attempt + 1) & (this->frequent_kmers.size() - 1); } // This should not happen. std::cerr << "KmerIndex::insert_frequent(): Cannot find the offset for key " << key << std::endl; assert(false); } }; //------------------------------------------------------------------------------ struct MinimizerHeader { std::uint32_t tag, version; std::uint64_t k, w_or_s; // Minimizer parameters. std::uint64_t keys; // Number of keys in the hash table. std::uint64_t unused; // Currently unused. std::uint64_t capacity; // Number of keys that can fit in the hash table without initiating rehashing. std::uint64_t values; // Number of values in the index. std::uint64_t unique; // Number of keys with a single value. std::uint64_t flags; constexpr static std::uint32_t TAG = 0x31513151; constexpr static std::uint32_t VERSION = Version::MINIMIZER_VERSION; constexpr static std::uint64_t FLAG_MASK = 0xFFFF; constexpr static std::uint64_t FLAG_KEY_MASK = 0x00FF; constexpr static size_t FLAG_KEY_OFFSET = 0; constexpr static std::uint64_t FLAG_SYNCMERS = 0x0100; constexpr static std::uint64_t FLAG_WEIGHT_MASK = 0x0E00; constexpr static size_t FLAG_WEIGHT_OFFSET = 9; constexpr static std::uint64_t FLAG_PAYLOAD_MASK = 0xF000; constexpr static size_t FLAG_PAYLOAD_OFFSET = 12; // Symbolic names for versions that may be relevant when examining files, // even when the version is current or obsolete. constexpr static std::uint32_t TAGS_VERSION = 11; MinimizerHeader(); MinimizerHeader(size_t kmer_length, size_t window_length, size_t key_bits, size_t payload_size); void sanitize(size_t kmer_max_length); // Throws `sdsl::simple_sds::InvalidData` if the header is invalid. void check() const; void update_version(); // Boolean flags. void set(std::uint64_t flag) { this->flags |= flag; } void unset(std::uint64_t flag) { this->flags &= ~flag; } bool get_flag(std::uint64_t flag) const { return (this->flags & flag); } // Integer flags. void set_int(std::uint64_t mask, size_t offset, size_t value); size_t get_int(std::uint64_t mask, size_t offset) const; size_t key_bits() const; // If weighted minimizers are in use, deprioritize frequent kmers by // this many iterations. size_t downweight() const; // Returns payload size in 64-bit words. size_t payload_size() const; // These do not compare the statistics fields, because the actual values are // now stored within the kmer index. bool operator==(const MinimizerHeader& another) const; bool operator!=(const MinimizerHeader& another) const { return !(this->operator==(another)); } // Read the statistics from the kmer index. template void set_statistics(const KmerIndex& index) { this->keys = index.size(); this->unused = 0; this->capacity = index.capacity(); this->values = index.number_of_values(); this->unique = index.unique_keys(); this->set_int(FLAG_WEIGHT_MASK, FLAG_WEIGHT_OFFSET, index.downweight); this->set_int(FLAG_PAYLOAD_MASK, FLAG_PAYLOAD_OFFSET, index.payload_size()); } // Write the statistics into the kmer index. template void fill_statistics(KmerIndex& index) const { index.keys = this->keys; index.max_keys = this->capacity; index.values = this->values; index.unique = this->unique; index.downweight = this->downweight(); index.payload = this->payload_size(); } }; //------------------------------------------------------------------------------ /* A class that implements the minimizer index as a hash table mapping kmers to sets of pos_t. For each stored position, we also store 64 bits of payload for external purposes. The hash table uses quadratic probing with power-of-two size. We encode kmers using 2 bits/character and hash the encoding. A minimizer is the kmer with the smallest hash in a window of w consecutive kmers and their reverse complements. The index can also use closed syncmers: Edgar: Syncmers are more sensitive than minimizers for selecting conserved k-mers in biological sequences. PeerJ 9:e10805, 2021. A closed syncmer is a kmer where one of the occurrences of the smer (or its reverse complement) with the smallest hash is the first or the last. Because we want to select the same kmers in both orientations, we use the kmer or its reverse complement, depending on which has the smaller hash. Minimizers and closed syncmers should have roughly the same seed density when w = k - s. There is also an option to use weighted minimizers: Jain, Rhie, Zhang, Chu, Walenz, Koren, and Philippy: Weighted minimizer sampling improves long read mapping. Bioinformatics, 2020. Normally, a minimizer is the kmer with the smallest hash value. With weighted minimizers, we want to discourage selecting some (frequent) kmers as minimizers. Let x be the hash value interpreted as a number in [0.0, 1.0]. If the kmer is listed as one to be avoided, we downweight it by replacing its hash value with 1.0 - (1.0 - x)^(2^iterations), where iterations is a parameter between 1 and 7 (default 3). Index versions (this should be in the wiki): 1 The initial version. 2 Minimizer selection is based on hashes instead of lexicographic order. A sequence and its reverse complement have the same minimizers, reducing index size by 50%. Not compatible with version 1. 3 Construction-time hit cap is no longer used. Compatible with version 2. 4 Use SDSL data structures where appropriate. Not compatible with version 3. 5 An option to use 128-bit keys. Compatible with version 4. 6 Store the position/pointer bit in the key instead of a separate bit vector. Store 64 bits of payload for each position. Not compatible with version 5. 7 Option to use closed syncmers instead of minimizers. Compatible with version 6. 8 Payload is now 128 bits per position. Not compatible with earlier versions. 9 Option to provide a set of frequent kmers that should be avoided as minimizers. The capacity field in the header is no longer in use. Compatible with version 8. 10 Replace the old distance index payload with zipcodes. Not compatible with earlier versions. 11 The index is no longer parameterized by the value type. Instead, the value consists of a position and 0-15 words of payload. Payload size is specified when creating the index. There is also a tag (key-value) structure that can be used for storing the type of payload. Not compatible with earlier versions. */ template class MinimizerIndex { public: typedef KeyType key_type; typedef typename KeyType::code_type code_type; typedef typename KmerIndex::cell_type cell_type; typedef typename KmerIndex::const_cell_type const_cell_type; typedef typename KmerIndex::value_type value_type; typedef typename KmerIndex::multi_value_type multi_value_type; typedef Kmer minimizer_type; const static std::string EXTENSION; // ".min" //------------------------------------------------------------------------------ explicit MinimizerIndex(size_t payload_size, bool use_syncmers = false) : header(KeyType::KMER_LENGTH, (use_syncmers ? KeyType::SMER_LENGTH : KeyType::WINDOW_LENGTH), KeyType::KEY_BITS, payload_size), tags(), index(payload_size) { if(use_syncmers) { this->header.set(MinimizerHeader::FLAG_SYNCMERS); } this->header.sanitize(KeyType::KMER_MAX_LENGTH); } MinimizerIndex(size_t kmer_length, size_t window_or_smer_length, size_t payload_size, bool use_syncmers = false) : header(kmer_length, window_or_smer_length, KeyType::KEY_BITS, payload_size), tags(), index(payload_size) { if(use_syncmers) { this->header.set(MinimizerHeader::FLAG_SYNCMERS); } this->header.sanitize(KeyType::KMER_MAX_LENGTH); } MinimizerIndex(const MinimizerIndex& source) : index(source.payload_size()) { this->copy(source); } MinimizerIndex(MinimizerIndex&& source) noexcept : index(source.payload_size()) { *this = std::move(source); } void swap(MinimizerIndex& another) noexcept { if(&another == this) { return; } std::swap(this->header, another.header); this->tags.swap(another.tags); this->index.swap(another.index); } MinimizerIndex& operator=(const MinimizerIndex& source) { if(&source != this) { this->copy(source); } return *this; } MinimizerIndex& operator=(MinimizerIndex&& source) noexcept { if(&source != this) { this->header = std::move(source.header); this->tags = std::move(source.tags); this->index = std::move(source.index); } return *this; } // Serializes the index to the ostream. Returns the number of bytes written. size_t serialize(std::ostream& out) const { size_t bytes = 0; // Get statistics from the index and serialize the header. MinimizerHeader copy = this->header; copy.set_statistics(this->index); bytes += io::serialize(out, copy); // Serialize the tags in the SDSL format. bytes += this->tags.serialize(out); // Serialize the hash table and the occurrence lists. bytes += this->index.serialize_hash_table(out); // Serialize the occurrence lists. for(size_t array_offset = 0; array_offset < this->index.hash_table.size(); array_offset += this->index.cell_size()) { const_cell_type cell = this->index.const_cell(array_offset); const std::vector* ptr = KmerIndex::get_vector_pointer(cell); if(ptr != nullptr) { bytes += io::serialize_vector(out, *ptr); } } // Serialize the frequent kmers. bytes += io::serialize_vector(out, this->index.frequent_kmers); return bytes; } // Load the index from the istream. // Throws sdsl::simple_sds::InvalidData if sanity checks fail. void deserialize(std::istream& in) { // Get rid of the existing pointers in the hash table. this->index.clear(); // Load and check the header and fill in the statistics in the kmer index. io::load(in, this->header); this->header.check(); if(this->header.key_bits() != KeyType::KEY_BITS) { std::string msg = "MinimizerIndex::deserialize(): Expected " + std::to_string(KeyType::KEY_BITS) + "-bit keys, got " + std::to_string(this->header.key_bits()) + "-bit keys"; throw sdsl::simple_sds::InvalidData(msg); } this->header.update_version(); this->header.fill_statistics(this->index); // Load the tags in the SDSL format. this->tags.load(in); // Load the hash table and the occurrence lists. io::load_vector(in, this->index.hash_table); size_t cell_size = this->index.cell_size(); for(size_t array_offset = 0; array_offset < this->index.hash_table.size(); array_offset += cell_size) { cell_type cell = this->index.cell(array_offset); if(cell.first.is_pointer()) { std::vector* ptr = new std::vector(); io::load_vector(in, *ptr); this->index.write_vector_pointer(ptr, cell); } } // Load the frequent kmers. io::load_vector(in, this->index.frequent_kmers); } // For testing. bool operator==(const MinimizerIndex& another) const { if(this->header != another.header) { return false; } if(this->tags != another.tags) { return false; } return (this->index == another.index); } bool operator!=(const MinimizerIndex& another) const { return !(this->operator==(another)); } //------------------------------------------------------------------------------ /* Tags. */ /* Sets the given tag to the given value, overwriting any existing value. Note that keys are case insensitive. */ void set_tag(const std::string& key, const std::string& value) { this->tags.set(key, value); } /* Returns the value for the given tag, or an empty string if the tag is not present. Note that keys are case insensitive. */ std::string get_tag(const std::string& key) const { return this->tags.get(key); } /* Removes the given tag. Does nothing if the tag is not present. Note that keys are case insensitive. */ void remove_tag(const std::string& key) { this->tags.unset(key); } /* Returns an iterator at the first tag. The tags are (key, value) pairs sorted by key. */ auto tags_begin() const { return this->tags.tags.begin(); } /* Returns an iterator past the last tag. */ auto tags_end() const { return this->tags.tags.end(); } /* Returns a GraphName object built from the information stored in the tags. */ GraphName graph_name() const { return GraphName(this->tags); } /* Stores the given GraphName object in the tags. */ void set_graph_name(const GraphName& graph_name) { graph_name.set_tags(this->tags); } //------------------------------------------------------------------------------ /* A circular buffer of size 2^i for all minimizer candidates. The candidates are sorted by both key and sequence offset. The candidate at offset j is removed when we reach offset j + w. Candidates at the tail are removed when we advance with a smaller hash. */ struct CircularBuffer { const KmerIndex& parent; std::vector buffer; size_t head, tail; size_t w; constexpr static size_t BUFFER_SIZE = 16; CircularBuffer(const KmerIndex& parent, size_t capacity) : parent(parent), buffer(), head(0), tail(0), w(capacity) { size_t buffer_size = BUFFER_SIZE; while(buffer_size < this->w) { buffer_size *= 2; } this->buffer.resize(buffer_size); } bool empty() const { return (this->head >= this->tail); } size_t size() const { return this->tail - this->head; } size_t begin() const { return this->head; } size_t end() const { return this->tail; } minimizer_type& at(size_t i) { return this->buffer[i & (this->buffer.size() - 1)]; } minimizer_type& front() { return this->at(this->begin()); } minimizer_type& back() { return this->at(this->end() - 1); } // Advance to the next offset (pos) with a valid kmer. void advance(offset_type pos, key_type forward_key, key_type reverse_key) { if(!(this->empty()) && this->front().offset + this->w <= pos) { this->head++; } size_t forward_hash = this->parent.hash(forward_key), reverse_hash = this->parent.hash(reverse_key); size_t hash = std::min(forward_hash, reverse_hash); while(!(this->empty()) && this->back().hash > hash) { this->tail--; } this->tail++; if(reverse_hash < forward_hash) { this->back() = { reverse_key, reverse_hash, pos, true }; } else { this->back() = { forward_key, forward_hash, pos, false }; } } // Advance to the next offset (pos) without a valid kmer. void advance(offset_type pos) { if(!(this->empty()) && this->front().offset + this->w <= pos) { this->head++; } } }; //------------------------------------------------------------------------------ /* Finding minimizers. */ /* Returns all minimizers in the string specified by the iterators. The return value is a vector of minimizers sorted by their offsets. If there are multiple occurrences of one or more minimizer keys with the same hash in a window, return all of them. Calls syncmers() if the index uses closed syncmers. */ std::vector minimizers(std::string::const_iterator begin, std::string::const_iterator end) const { if(this->uses_syncmers()) { return this->syncmers(begin, end); } std::vector result; size_t window_length = this->window_bp(), total_length = end - begin; if(total_length < window_length) { return result; } // Find the minimizers. CircularBuffer buffer(this->index, this->w()); size_t valid_chars = 0, start_pos = 0; size_t next_read_offset = 0; // The first read offset that may contain a new minimizer. key_type forward_key, reverse_key; std::string::const_iterator iter = begin; while(iter != end) { forward_key.forward(this->k(), *iter, valid_chars); reverse_key.reverse(this->k(), *iter); if(valid_chars >= this->k()) { buffer.advance(start_pos, forward_key, reverse_key); } else { buffer.advance(start_pos); } ++iter; if(static_cast(iter - begin) >= this->k()) { start_pos++; } // We have a full window with a minimizer. if(static_cast(iter - begin) >= window_length && !buffer.empty()) { // Insert the candidates if: // 1) this is the first minimizer we encounter; // 2) the last reported minimizer had the same hash (we may have new occurrences); or // 3) the first candidate is located after the last reported minimizer. if(result.empty() || result.back().hash == buffer.front().hash || result.back().offset < buffer.front().offset) { // Insert all new occurrences of the minimizer in the window. for(size_t i = buffer.begin(); i < buffer.end() && buffer.at(i).hash == buffer.front().hash; i++) { if(buffer.at(i).offset >= next_read_offset) { result.emplace_back(buffer.at(i)); next_read_offset = buffer.at(i).offset + 1; } } } } } // It was more convenient to use the first offset of the kmer, regardless of the orientation. // If the minimizer is a reverse complement, we must return the last offset instead. for(minimizer_type& minimizer : result) { if(minimizer.is_reverse) { minimizer.offset += this->k() - 1; } } std::sort(result.begin(), result.end()); return result; } /* Returns all minimizers in the string. The return value is a vector of minimizers sorted by their offsets. Calls syncmers() if the index uses closed syncmers. */ std::vector minimizers(const std::string& str) const { return this->minimizers(str.begin(), str.end()); } /* Returns all minimizers in the string specified by the iterators, together with the start and length of the run of windows they arise from. The return value is a vector of tuples of minimizers, starts, and lengths, sorted by minimizer offset. Calls syncmers() if the index uses closed syncmers but leaves the start and length fields empty. */ std::vector> minimizer_regions(std::string::const_iterator begin, std::string::const_iterator end) const { std::vector> result; if(this->uses_syncmers()) { std::vector res = this->syncmers(begin, end); result.reserve(res.size()); for(const minimizer_type& m : res) { result.emplace_back(m, 0, 0); } return result; } size_t window_length = this->window_bp(), total_length = end - begin; if(total_length < window_length) { return result; } // Find the minimizers. CircularBuffer buffer(this->index, this->w()); // Note that start_pos isn't meaningfully the start of the window we are // looking at. size_t valid_chars = 0, start_pos = 0; size_t next_read_offset = 0; // The first read offset that may contain a new minimizer. // All results before this are finished and have their lengths filled in. // All results after are current winning minimizers of the current window. size_t finished_through = 0; key_type forward_key, reverse_key; std::string::const_iterator iter = begin; while(iter != end) { // Get the forward and reverse strand minimizer candidates forward_key.forward(this->k(), *iter, valid_chars); reverse_key.reverse(this->k(), *iter); // If they don't have any Ns or anything in them, throw them into the sliding window tracked by buffer. // Otherwise just slide it along. if(valid_chars >= this->k()) { buffer.advance(start_pos, forward_key, reverse_key); } else { buffer.advance(start_pos); } ++iter; if(static_cast(iter - begin) >= this->k()) { start_pos++; } // We have a full window. if(static_cast(iter - begin) >= window_length) { // Work out where the window we are minimizing in began size_t window_start = static_cast(iter - begin) - window_length; // Work out the past-the-end index of the window we have just finished (not the current window) size_t prev_past_end_pos = window_start + window_length - 1; // Finish off end positions for results that weren't replaced but are going out of range while(finished_through < result.size() && std::get<0>(result[finished_through]).offset < window_start) { // Compute region length based on it stopping at the previous step std::get<2>(result[finished_through]) = prev_past_end_pos - std::get<1>(result[finished_through]); finished_through++; } // Our full window has a minimizer in it if (!buffer.empty()) { // Insert the candidates if: // 1) this is the first minimizer we encounter; // 2) the last reported minimizer had the same hash (we may have new occurrences); or // 3) the first candidate is located after the last reported minimizer. if(result.empty() || std::get<0>(result.back()).hash == buffer.front().hash || std::get<0>(result.back()).offset < buffer.front().offset) { // Insert all new occurrences of the minimizer in the window. for(size_t i = buffer.begin(); i < buffer.end() && buffer.at(i).hash == buffer.front().hash; i++) { if(buffer.at(i).offset >= next_read_offset) { // Insert the minimizer instance, with its region starting // where the window covered by the buffer starts. result.emplace_back(buffer.at(i), window_start, 0); // There can only ever really be one minimizer at a given start // position. So look for the next one 1 base to the right. next_read_offset = buffer.at(i).offset + 1; } } // If new minimizers beat out old ones, finish off the old ones. while(!result.empty() && finished_through < result.size() && std::get<0>(result.back()).hash != std::get<0>(result[finished_through]).hash) { // The window before the one we are looking at was the last one for this minimizer. std::get<2>(result[finished_through]) = prev_past_end_pos - std::get<1>(result[finished_through]); finished_through++; } } } } } // Now close off the minimizers left active when we hit the end of the string. while(finished_through < result.size()) { // The region length is from the region start to the string end std::get<2>(result[finished_through]) = total_length - std::get<1>(result[finished_through]); finished_through++; } // It was more convenient to use the first offset of the kmer, regardless of the orientation. // If the minimizer is a reverse complement, we must return the last offset instead. for(auto& record : result) { if(std::get<0>(record).is_reverse) { std::get<0>(record).offset += this->k() - 1; } } std::sort(result.begin(), result.end()); return result; } /* Returns all minimizers in the string. The return value is a vector of minimizers, region starts, and region lengths sorted by their offsets. Calls syncmers() if the index uses closed syncmers but leaves the start and length fields empty. */ std::vector> minimizer_regions(const std::string& str) const { return this->minimizer_regions(str.begin(), str.end()); } //------------------------------------------------------------------------------ /* Finding syncmers. */ /* Returns all closed syncmers in the string specified by the iterators. The return value is a vector of minimizers sorted by their offsets. Calls minimizers() if the index uses minimizers. */ std::vector syncmers(std::string::const_iterator begin, std::string::const_iterator end) const { if(!(this->uses_syncmers())) { return this->minimizers(begin, end); } std::vector result; size_t total_length = end - begin; if(total_length < this->k()) { return result; } // Find the closed syncmers. CircularBuffer buffer(this->index, this->k() + 1 - this->s()); size_t processed_chars = 0, dummy_valid_chars = 0, valid_chars = 0, smer_start = 0; key_type forward_smer, reverse_smer, forward_kmer, reverse_kmer; std::string::const_iterator iter = begin; while(iter != end) { forward_smer.forward(this->s(), *iter, valid_chars); reverse_smer.reverse(this->s(), *iter); forward_kmer.forward(this->k(), *iter, dummy_valid_chars); reverse_kmer.reverse(this->k(), *iter); if(valid_chars >= this->s()) { buffer.advance(smer_start, forward_smer, reverse_smer); } else { buffer.advance(smer_start); } ++iter; processed_chars++; if(processed_chars >= this->s()) { smer_start++; } // We have a full kmer with a closed syncmer. if(valid_chars >= this->k()) { // Insert the kmer if the first or the last smer is among the smallest, i.e. if // 1) the first smer in the buffer is at the start of the kmer; // 2) the first smer in the buffer is at the end of the kmer; or // 3) the last smer in the buffer is at the end of the kmer and shares shares // the hash with the first smer. if(buffer.front().offset == processed_chars - this->k() || buffer.front().offset == smer_start - 1 || (buffer.back().offset == smer_start - 1 && buffer.back().hash == buffer.front().hash)) { size_t forward_hash = this->index.hash(forward_kmer), reverse_hash = this->index.hash(reverse_kmer); offset_type pos = processed_chars - this->k(); if(reverse_hash < forward_hash) { result.push_back({ reverse_kmer, reverse_hash, pos, true }); } else { result.push_back({ forward_kmer, forward_hash, pos, false }); } } } } // It was more convenient to use the first offset of the kmer, regardless of the orientation. // If the closed syncmer is a reverse complement, we must return the last offset instead. for(minimizer_type& minimizer : result) { if(minimizer.is_reverse) { minimizer.offset += this->k() - 1; } } std::sort(result.begin(), result.end()); return result; } /* Returns all closed syncmers in the string. The return value is a vector of minimizers sorted by their offsets. Calls minimizers() if the index uses minimizers. */ std::vector syncmers(const std::string& str) const { return this->syncmers(str.begin(), str.end()); } //------------------------------------------------------------------------------ /* Operations. */ /* Adds a set of frequent kmers that should be avoided (in both orientations) when using weighted minimizers. The index must be empty. Clears the frequent kmers if the set of keys is empty or the number of iterations is 0. Throws std::runtime_error on failure. */ void add_frequent_kmers(const std::vector& kmers, size_t iterations) { if(!this->empty()) { throw std::runtime_error("MinimizerIndex::add_frequent_kmers(): The index is not empty"); } if(this->uses_syncmers()) { throw std::runtime_error("MinimizerIndex::add_frequent_kmers(): Cannot use frequent kmers with syncmers"); } size_t max_iterations = MinimizerHeader::FLAG_WEIGHT_MASK >> MinimizerHeader::FLAG_WEIGHT_OFFSET; if(iterations > max_iterations) { std::string msg = "MinimizerIndex::add_frequent_kmers(): Number of iterations (" + std::to_string(iterations) + ") must be at most " + std::to_string(max_iterations); throw std::runtime_error(msg); } if(kmers.empty() || iterations == 0) { this->index.frequent_kmers.clear(); this->index.downweight = 0; this->header.set_int(MinimizerHeader::FLAG_WEIGHT_MASK, MinimizerHeader::FLAG_WEIGHT_OFFSET, 0); } // We oversize the hash table a bit to make it more likely to find an empty // cell when querying with a non-frequent kmer. size_t capacity = KmerIndex::minimum_size(4 * kmers.size()); this->index.frequent_kmers = std::vector(capacity, key_type::no_key()); for(key_type key : kmers) { this->index.insert_frequent(key); this->index.insert_frequent(key.reverse_complement(this->k())); } this->index.downweight = iterations; this->header.set_int(MinimizerHeader::FLAG_WEIGHT_MASK, MinimizerHeader::FLAG_WEIGHT_OFFSET, iterations); } /* Inserts the value into the index, using minimizer.key as the key and minimizer.hash as its hash. Does not insert keys equal to key_type::no_key() or positions equal to Position::no_pos(). Does not update the payload if the value has already been inserted with the same key. Use minimizer() or minimizers() to get the minimizer. */ void insert(const minimizer_type& minimizer, value_type value) { this->index.insert(minimizer.key, value, minimizer.hash); } /* Path-aware insert: like insert(), but if (minimizer.key, value.first) is already present, the LAST word of the existing payload is OR'd with the LAST word of the new payload, instead of dropping the new value silently. See KmerIndex::insert_with_last_payload_word_or() for the precondition on the leading payload words and the rationale. */ void insert_with_last_payload_word_or(const minimizer_type& minimizer, value_type value) { this->index.insert_with_last_payload_word_or(minimizer.key, value, minimizer.hash); } /* Returns the occurrence count of the minimizer. Use minimizer() or minimizers() to get the minimizer. */ size_t count(const minimizer_type& minimizer) const { return this->index.count(minimizer.key, minimizer.hash); } /* Returns a pointer to the internal representation of the occurrences of the given minimizer and the occurrence count. The occurrences are in sorted order. The pointer may be invalidated if new positions are inserted into the index. Use minimizer() or minimizers() to get the minimizer and position.decode() to decode the positions within the occurrences. If the minimizer is in reverse orientation, use reverse_base_pos() to reverse the reported occurrences. */ multi_value_type find(const minimizer_type& minimizer) const { return this->index.find(minimizer.key, minimizer.hash); } /* Decodes the given value from the given list of values. Returns an empty value with no position and an empty payload if the index is out of bounds. */ value_type get_value(multi_value_type values, size_t index) const { return this->index.get_value(values, index); } //------------------------------------------------------------------------------ /* Statistics. */ // Length of the kmers in the index. size_t k() const { return this->header.k; } // Window length for the minimizers. Does not make sense when using closed syncmers. size_t w() const { return this->header.w_or_s; } // Length of the smers when using closed syncmers. Does not make sense when using minimizers. size_t s() const { return this->header.w_or_s; } // Does the index use closed syncmers instead of minimizers. bool uses_syncmers() const { return this->header.get_flag(MinimizerHeader::FLAG_SYNCMERS); } // Does the index use weighted minimizers. bool uses_weighted_minimizers() const { return !(this->index.frequent_kmers.empty()); } // Window length in bp. We are guaranteed to have at least one kmer from the window if // all characters within it are valid. size_t window_bp() const { return (this->uses_syncmers() ? 2 * this->k() - this->s() - 1 : this->k() + this->w() - 1); } // Number of keys in the index. size_t size() const { return this->index.size(); } // Is the index empty. bool empty() const { return (this->size() == 0); } // Number of values (minimizer occurrences) in the index. size_t number_of_values() const { return this->index.number_of_values(); } // Size of the hash table. size_t cell_count() const { return this->index.cell_count(); } // Number of keys that can fit into the hash table. Exceeding it will initiate rehashing. size_t capacity() const { return this->index.capacity(); } // Current load factor of the hash table. double load_factor() const { return this->index.load_factor(); } // Number of minimizers with a single occurrence. size_t unique_keys() const { return this->index.unique_keys(); } // Returns the size of a payload in words. size_t payload_size() const { return this->index.payload_size(); } // Returns the size of a value in words. size_t value_size() const { return KmerIndex::POS_SIZE + this->payload_size(); } // Returns the size of a cell in words. size_t cell_size() const { return KmerIndex::KEY_SIZE + KmerIndex::POS_SIZE + this->payload_size(); } // Call `callback` for every non-empty hash table cell in the index. // The callback gets the key and encoded values. // If callback returns false, then stop iterating. // Returns false if the iteration stopped early, true otherwise. bool for_each_minimizer(const std::function& callback) const { return this->index.for_each_kmer(callback); } //------------------------------------------------------------------------------ /* Internal implementation. */ private: MinimizerHeader header; gbwt::Tags tags; KmerIndex index; void copy(const MinimizerIndex& source) { this->header = source.header; this->tags = source.tags; this->index = source.index; } }; //------------------------------------------------------------------------------ /* Decode the subset of minimizer hits and their payloads in the given subgraph induced by node identifiers. This version should only be used when the number of hits is small. If the minimizer is in reverse orientation, use reverse_base_pos() to reverse the reported occurrences. */ template void hits_in_subgraph ( const MinimizerIndex& index, KmerEncoding::multi_value_type hits, const std::unordered_set& subgraph, const std::function& report_hit ); /* Decode the subset of minimizer hits and their payloads in the given subgraph induced by node identifiers. The set of node ids must be in sorted order. This version uses exponential search on the larger list, so it should be efficient regardless of the size of the subgraph and the number of hits. If the minimizer is in reverse orientation, use reverse_base_pos() to reverse the reported occurrences. */ template void hits_in_subgraph ( const MinimizerIndex& index, KmerEncoding::multi_value_type hits, const std::vector& subgraph, const std::function& report_hit ); //------------------------------------------------------------------------------ // Choose the default index type. using DefaultMinimizerIndex = MinimizerIndex; //------------------------------------------------------------------------------ // Numerical template class constants. template constexpr size_t KmerIndex::INITIAL_CAPACITY; template constexpr double KmerIndex::MAX_LOAD_FACTOR; template constexpr size_t KmerIndex::MAX_PAYLOAD_SIZE; template constexpr size_t KmerIndex::KEY_SIZE; template constexpr size_t KmerIndex::POS_SIZE; // Other template class variables. template const std::string MinimizerIndex::EXTENSION = ".min"; //------------------------------------------------------------------------------ } // namespace gbwtgraph #endif // GBWTGRAPH_MINIMIZER_H jltsiren-gbwtgraph-e27bc43/include/gbwtgraph/naive_graph.h000066400000000000000000000244471524136466400240340ustar00rootroot00000000000000#ifndef GBWTGRAPH_NAIVE_GRAPH_H #define GBWTGRAPH_NAIVE_GRAPH_H #include #include /* naive_graph.h: A naive HandleGraph implementation used as a helper in GBWTGraph construction (e.g. from GFA). */ namespace gbwtgraph { //------------------------------------------------------------------------------ /* A naive HandleGraph implementation based on a hash map of node records and concatenated sequences. Nodes can be added with `create_node(id, sequence)` or translated from GFA segments with `translate_segment(name, sequence, max_bp)`. These two approaches must not be mixed. In either case, GFA segment names can be translated into half-open ranges of node ids with `translate(name)`. Inserted sequences are normalized to upper case. The tags object is intended for storing GraphName information for the parent graph. If a translation is used, this will be for the translation target of the GBWTGraph being built. Otherwise it is either for the supergraph or the same graph, depending on whether the paths in the GBWT use all nodes and edges in the graph. */ class NaiveGraph : public HandleGraph { /* Construction. */ public: NaiveGraph(); NaiveGraph(const NaiveGraph& another) = default; NaiveGraph(NaiveGraph&& another) noexcept = default; NaiveGraph& operator=(const NaiveGraph& another) = default; NaiveGraph& operator=(NaiveGraph&& another) noexcept = default; virtual ~NaiveGraph() {} // Creates a new node with the given sequence. // Cannot be used with `translate_segment()`. // Throws `std::runtime_error` if the node already exists or if the sequence is empty. void create_node(nid_t node_id, std::string_view sequence); // Takes a GFA segment with the given name and sequence. // Translates it into one or more nodes of at most `max_length` bp each, assigning them the next unused node ids. // Returns the half-open node id range for the translated segment. // Cannot be used with `create_node()`. // Throws `std::runtime_error` if the segment has already been translated or if the sequence is empty. std::pair translate_segment(const std::string& name, std::string_view sequence, size_t max_length); // Creates a new edge between the two oriented nodes. // Throws `std::runtime_error` if the nodes do not exist. void create_edge(gbwt::node_type from, gbwt::node_type to); // Removes all duplicate edges. void remove_duplicate_edges(); //------------------------------------------------------------------------------ /* HandleGraph interface. Methods return default values when the node does not exist. */ public: // Method to check if a node exists by ID. virtual bool has_node(nid_t node_id) const; // Look up the handle for the node with the given ID in the given orientation. virtual handle_t get_handle(const nid_t& node_id, bool is_reverse = false) const; // Get the ID from a handle. virtual nid_t get_id(const handle_t& handle) const; // Get the orientation of a handle. virtual bool get_is_reverse(const handle_t& handle) const; // Invert the orientation of a handle (potentially without getting its ID). virtual handle_t flip(const handle_t& handle) const; // Get the length of a node. virtual size_t get_length(const handle_t& handle) const; // Get the sequence of a node, presented in the handle's local forward // orientation. virtual std::string get_sequence(const handle_t& handle) const; // Returns one base of a handle's sequence, in the orientation of the // handle. virtual char get_base(const handle_t& handle, size_t index) const; // Returns a substring of a handle's sequence, in the orientation of the // handle. If the indicated substring would extend beyond the end of the // handle's sequence, the return value is truncated to the sequence's end. virtual std::string get_subsequence(const handle_t& handle, size_t index, size_t size) const; // Return the number of nodes in the graph. virtual size_t get_node_count() const; // Return the smallest ID in the graph, or some smaller number if the // smallest ID is unavailable. Return value is unspecified if the graph is empty. virtual nid_t min_node_id() const; // Return the largest ID in the graph, or some larger number if the // largest ID is unavailable. Return value is unspecified if the graph is empty. virtual nid_t max_node_id() const; protected: // Loop over all the handles to next/previous (right/left) nodes. Passes // them to a callback which returns false to stop iterating and true to // continue. Returns true if we finished and false if we stopped early. virtual bool follow_edges_impl(const handle_t& handle, bool go_left, const std::function& iteratee) const; // Loop over all the nodes in the graph in their local forward // orientations, in their internal stored order. Stop if the iteratee // returns false. Can be told to run in parallel, in which case stopping // after a false return value is on a best-effort basis and iteration // order is not defined. Returns true if we finished and false if we // stopped early. virtual bool for_each_handle_impl(const std::function& iteratee, bool parallel = false) const; public: // Get the number of edges on the right (go_left = false) or left (go_left // = true) side of the given handle. virtual size_t get_degree(const handle_t& handle, bool go_left) const; //------------------------------------------------------------------------------ /* Data members. */ private: struct Node { std::vector predecessors, successors; size_t sequence_offset; // Offset into NaiveGraph::sequences size_t sequence_length; }; // This should be called after a node is created. void normalize_sequence(size_t from_offset, size_t length); std::unordered_map nodes; std::vector sequences; nid_t min_id, max_id; // For each segment name, a half-open range of node ids. std::unordered_map> segment_translation; // Tags for storing GraphName information. gbwt::Tags tags; //------------------------------------------------------------------------------ /* Custom interface. */ public: // Extension for a file storing the node-to-segment translation. const static std::string TRANSLATION_EXTENSION; // ".trans" // Returns `true` if the graph contains a node with the given id. // Potentially faster than `has_node()`, as this is not virtual. bool has_node_impl(nid_t node_id) const { return (this->nodes.find(node_id) != this->nodes.end()); } // Returns an iterator to the specified node by node id. std::unordered_map::const_iterator get_node(nid_t node_id) const { return this->nodes.find(node_id); } // Returns an iterator to the specified node by handle. std::unordered_map::const_iterator get_node(const handle_t& handle) const { return this->nodes.find(gbwt::Node::id(handle_to_node(handle))); } // Returns an iterator to the specified node by node id. std::unordered_map::iterator get_node_mut(nid_t node_id) { return this->nodes.find(node_id); } // Returns an iterator to the specified node by handle. std::unordered_map::iterator get_node_mut(const handle_t& handle) { return this->nodes.find(gbwt::Node::id(handle_to_node(handle))); } // Returns a view of the sequence of the specified node in the forward orientation. // Returns an empty view if the node does not exist. std::string_view get_sequence_view(nid_t node_id) const; // Convert gbwt::node_type to handle_t. static handle_t node_to_handle(gbwt::node_type node) { return handlegraph::as_handle(node); } // Convert handle_t to gbwt::node_type. static gbwt::node_type handle_to_node(const handle_t& handle) { return handlegraph::as_integer(handle); } // Returns `true` if the graph uses node-to-segment translation. // That means that nodes were created using `translate_segment()` rather than `add_node()`. bool uses_translation() const { return !(this->segment_translation.empty()); } // Returns the number of segments in the translation. size_t get_segment_count() const { return this->segment_translation.size(); } // Returns `true` if the graph contains a translation for a segment with the given name. bool has_segment(const std::string& name) const { return (this->segment_translation.find(name) != this->segment_translation.end()); } // Returns `true` if the graph contains a node/segment with the given name, // depending on whether translation is used. bool has_node_or_segment(const std::string& name) const; // An empty node id range indicating that the translation failed. constexpr static std::pair no_translation() { return std::pair(0, 0); } // Returns an iterator to the translations in an arbitrary order. std::unordered_map>::const_iterator translation_begin() const { return this->segment_translation.cbegin(); } // Returns an iterator to the end of the translations. std::unordered_map>::const_iterator translation_end() const { return this->segment_translation.cend(); } // Translates the segment name into a half-open range of node identifiers. // Returns (id, id + 1) if the graph does not use translation and the segment name // is a valid integer node id. // Returns `no_translation()` if the translation fails. std::pair translate(const std::string& segment_name) const; // Returns `StringArray` of segment names and `sd_vector<>` mapping node ids to names. // If `is_present` returns false, the corresponding segment name will be empty. // Uses multiple OpenMP threads. std::pair> invert_translation(const std::function)>& is_present) const; // Stores the given GraphName information in the tags. void set_graph_name(const GraphName& graph_name) { graph_name.set_tags(this->tags); } // Retrieves the GraphName information from the tags. GraphName graph_name() const { return GraphName(this->tags); } }; //------------------------------------------------------------------------------ } // namespace gbwtgraph #endif // GBWTGRAPH_NAIVE_GRAPH_Hjltsiren-gbwtgraph-e27bc43/include/gbwtgraph/path_cover.h000066400000000000000000000154431524136466400236770ustar00rootroot00000000000000#ifndef GBWTGRAPH_PATH_COVER_H #define GBWTGRAPH_PATH_COVER_H #include #include /* path_cover.h: Build GBWT from a path cover of a HandleGraph. */ namespace gbwtgraph { //------------------------------------------------------------------------------ constexpr size_t LOCAL_HAPLOTYPES_DEFAULT_N = 64; constexpr size_t PATH_COVER_DEFAULT_N = 16; constexpr size_t PATH_COVER_DEFAULT_K = 4; constexpr size_t PATH_COVER_MIN_K = 2; constexpr size_t PATH_COVER_DEFAULT_JOBS = 32; //------------------------------------------------------------------------------ // TODO: Remove this. VG no longer uses it. /* Store the named paths from the given graph into the given GBWT builder. Generic named paths will go under a sample with the special GENERIC_PATH_SAMPLE_NAME name. Creates new contigs with the appropriate names, or re-uses any found in the metadata. Skips empty paths. If the given filter function is set, and returns false for a path, that path is not added. */ [[deprecated("Use assign_paths and insert_paths instead")]] void store_named_paths(gbwt::GBWTBuilder& builder, const PathHandleGraph& graph, const std::function* path_filter = nullptr); // TODO: Remove this after haplotype_indexer in vg has been updated. /* Store paths from the given graph into the given GBWT builder. Generic named paths will go under a sample with the special GENERIC_PATH_SAMPLE_NAME name. Creates new contigs with the appropriate names, or re-uses any found in the metadata. Only stores paths with the senses in the given set. Skips empty paths. If the given filter function is set, and returns false for a path, that path is not added. */ [[deprecated("Use assign_paths and insert_paths instead")]] void store_paths(gbwt::GBWTBuilder& builder, const PathHandleGraph& graph, const std::unordered_set& senses, const std::function* path_filter = nullptr); //------------------------------------------------------------------------------ struct PathCoverParameters { // Number of paths per component. size_t num_paths = PATH_COVER_DEFAULT_N; // Length of the window in nodes. size_t context = PATH_COVER_DEFAULT_K; // Batch size for GBWT construction (in nodes). gbwt::size_type batch_size = gbwt::DynamicGBWT::INSERT_BATCH_SIZE; // Sample interval for GBWT construction (in nodes). gbwt::size_type sample_interval = gbwt::DynamicGBWT::SAMPLE_INTERVAL; // Approximate number of construction jobs. size_t approximate_num_jobs = PATH_COVER_DEFAULT_JOBS; // Number of parallel GBWT construction jobs. size_t parallel_jobs = 1; // Show progress information. bool show_progress = false; }; //------------------------------------------------------------------------------ /* Find a path cover of the graph with n paths per component and return a GBWT of the paths. The path cover is built greedily. Each time we extend a path, we choose the extension where the coverage of the k >= 2 node window is the lowest. Note that this is a maximum coverage algorithm that tries to maximize the number of windows covered by a fixed number of paths. This algorithm has been inspired by: Ghaffaari and Marschall: Fully-sensitive seed finding in sequence graphs using a hybrid index. Bioinformatics, 2019. Because the graph here may have cycles and orientation flips, some changes were necessary: - If the component is not a DAG, we start from an arbitrary node with minimal coverage and extend in both directions. - In a DAG, we start from the head node with the lowest coverage so far. - We stop when the length of the path reaches the size of the component. - The length of the window is in nodes instead of base pairs. We expect a sparse graph, where the nodes between variants are long. - If the component is not a DAG and the path is shorter than k - 1 nodes, we consider the coverage of individual nodes instead of windows. - When determining window coverage, we consider the window equivalent to its reverse complement. If include_named_paths is set, named paths from the graph will be stored. If a path_filter is supplied, only paths matching it will be stored. NOTE: The resulting GBWT does not have any samples assigned as reference samples. If reference paths were included, the REFERENCE_SAMPLE_LIST_GBWT_TAG tag should be set separately when appropriate. However, the subgraph constructors for GBWTGraph and GBZ will do this automatically. */ gbwt::GBWT path_cover_gbwt( const PathHandleGraph& graph, const PathCoverParameters& parameters, bool include_named_paths = false, const std::function* path_filter = nullptr); //------------------------------------------------------------------------------ /* As above, but build the path cover using local haplotypes. Each window must be consistent with the haplotypes in the GBWT index. Instead of choosing the extension with the lowest coverage, this algorithm maximizes the D'Hondt ratio: true_coverage / (selected_coverage + 1). In graph components without haplotypes in the GBWT index, this algorithm will revert to the regular path cover algorithm. If include_named_paths is set, named paths from the graph will be stored. If a path_filter is supplied, only paths matching it will be stored. NOTE: The resulting GBWT does not have any samples assigned as reference samples. If reference paths were included, the REFERENCE_SAMPLE_LIST_GBWT_TAG tag should be set separately when appropriate. However, the subgraph constructors for GBWTGraph and GBZ will do this automatically. */ gbwt::GBWT local_haplotypes( const PathHandleGraph& graph, const gbwt::GBWT& index, const PathCoverParameters& parameters, bool include_named_paths = false, const std::function* path_filter = nullptr); //------------------------------------------------------------------------------ /* Augment the given GBWT index with a path cover of the components that do not have any paths. This will add n new samples but no new haplotypes into the metadata. If the metadata contains sample/contig names, the path cover will use names path_cover_i and component_i. Returns the number of components that received a path cover. This algorithm is effectively a single construction job, as it takes an existing dynamic index. We further assume that most components are already covered by the GBWT and we only need to augment a few small ones. */ size_t augment_gbwt( const HandleGraph& graph, gbwt::DynamicGBWT& index, const PathCoverParameters& parameters); //------------------------------------------------------------------------------ } // namespace gbwtgraph #endif // GBWTGRAPH_PATH_COVER_H jltsiren-gbwtgraph-e27bc43/include/gbwtgraph/subgraph.h000066400000000000000000000172631524136466400233620ustar00rootroot00000000000000#ifndef GBWTGRAPH_SUBGRAPH_H #define GBWTGRAPH_SUBGRAPH_H #include #include #include #include #include /* subgraph.h: Subgraph extraction from a GBZ graph as in GBZ-base. */ namespace gbwtgraph { //------------------------------------------------------------------------------ /* A data structure for indexing reference and generic paths in a GBZ graph for random access by sequence offset. For each path, the index stores GBWT positions corresponding to node starts once every `sample_interval` bp (default 1024 bp). If the path is a haplotype fragment, the positions are given relative to the start of the fragment, not the full haplotype. The index relies on the fact that path handles for generic / reference paths in a GBWTGraph are integers in the range [0, n), where n is the number of such paths in the graph. */ class PathIndex { public: // Sample node starts once every 1024 bp by default. constexpr static size_t DEFAULT_SAMPLE_INTERVAL = 1024; // Build the index for the paths in the given GBZ graph. // The work is parallelized over the paths using OpenMP threads. explicit PathIndex(const GBZ& gbz, size_t sample_interval = DEFAULT_SAMPLE_INTERVAL); PathIndex(const PathIndex& source) = default; PathIndex(PathIndex&& source) noexcept = default; PathIndex& operator=(const PathIndex& source) = default; PathIndex& operator=(PathIndex&& source) noexcept = default; // For each `NamedPath` in the graph, mark the sampled sequence positions. std::vector> sequence_positions; // Sampled GBWT positions for each `NamedPath` in the graph. std::vector> gbwt_positions; size_t paths() const { return this->sequence_positions.size(); } size_t path_length(path_handle_t handle) const { return this->sequence_positions[handlegraph::as_integer(handle)].size(); } // Returns the last sampled position at or before `offset` in the path, // as a pair (sequence offset, GBWT position). If there is no such path, // returns (0, gbwt::invalid_edge()). std::pair sampled_position(path_handle_t handle, size_t offset) const; }; //------------------------------------------------------------------------------ /* A query for extracting a subgraph from a GBZ graph. The subgraph contains all nodes and edges within the given context (in bp) around a path offset, path interval, or a node. The query can also extract all path fragments within the subgraph. If the query is based on a path, the corresponding fragment will have true metadata. All other paths are reported as unknown paths. It is possible to output all paths, only distinct paths (with the number of copies as the weight), or only the reference path. */ struct SubgraphQuery { enum QueryType { path_offset_query, path_interval_query, node_query, invalid_query }; enum HaplotypeOutput { all_haplotypes, distinct_haplotypes, reference_only }; // Type of the query. QueryType type; // Which haplotypes to output. HaplotypeOutput haplotype_output; // A path offset/interval query as (sample, contig, haplotype, offset). gbwt::FullPathName path_query; // Node id for the node query. nid_t node_id; // Context length around the query position (in bp). size_t context; // Creates a subgraph query based on a haplotype offset. // The offset stored in `path_name` will be ignored. static SubgraphQuery path_offset(const gbwt::FullPathName& path_name, size_t offset, size_t context, HaplotypeOutput output); // Creates a subgraph query based on a haplotype interval [`from`, `to`). // The offset stored in `path_name` will be ignored. // Throws `std::runtime_error` if the interval is empty. static SubgraphQuery path_interval(const gbwt::FullPathName& path_name, size_t from, size_t to, size_t context, HaplotypeOutput output); // Creates a subgraph query based on a node id. static SubgraphQuery node(nid_t node, size_t context, HaplotypeOutput output); // Returns a string representation of the query. std::string to_string() const; }; //------------------------------------------------------------------------------ /* A subgraph extracted from a GBZ graph using a `SubgraphQuery`. The subgraph is defined as the induced subgraph based on a stored set of node identifiers. There may be a defined reference path. There may be a weight (the number of duplicates) for each path and a CIGAR string relative to the reference for each non-reference path. At the moment, the only supported functionality is converting the subgraph to GFA format. */ class Subgraph { public: // Build a subgraph from the given query. // If the query is based on a path, a path index must be provided. // Throws `std::runtime_error` on error. Subgraph(const GBZ& gbz, const PathIndex* path_index, const SubgraphQuery& query); Subgraph(const Subgraph& source) = default; Subgraph(Subgraph&& source) noexcept = default; Subgraph& operator=(const Subgraph& source) = default; Subgraph& operator=(Subgraph&& source) noexcept = default; // Node identifiers in the subgraph. std::set nodes; // Paths in the subgraph (in canonical orientation). std::vector paths; // Length of each path (in bp). std::vector path_lengths; // Weight (the number of duplicates) of each path. std::vector path_weights; // CIGAR string for each path relative to the reference path. std::vector path_cigars; // Offset of the reference path in `paths`. size_t reference_path; // Metadata for the reference path. gbwt::FullPathName reference_name; // Starting offset of the reference path fragment in this subgraph. // This is relative to the path fragment corresponding to `reference_handle`, not the full haplotype. // The starting offset of the path fragment in the full haplotype is `reference_name.offset`. size_t reference_start; // Convert the subgraph to GFA. void to_gfa(const GBZ& gbz, std::ostream& out) const; private: // Extract the paths within the subgraph and determine reference path information. void extract_paths(const GBZ& gbz, const SubgraphQuery& query, size_t query_offset, const std::pair& ref_pos); // Update the paths according to query type. void update_paths(const SubgraphQuery& query); const size_t* weight(size_t path_id) const; const std::string* cigar(size_t path_id) const; }; //------------------------------------------------------------------------------ /* An algorithm for aligning a subpath relative to a reference subpath. The two subpaths are assumed to be diverging. The alignment is done under the following assumptions: * There may be matching bases at the beginning and the end. * Any matching bases in the middle are considered accidental and treated as mismatches. * Scoring parameters are the same as in vg: match = 1, mismatch = -4, gap open = -6, gap extend = -1. * If multiple operations are used for the middle, the order is mismatch, insertion, deletion. The alignment is appended to the given vector of edit operation as pairs ('M', length), ('I', length), or ('D', length), as in CIGAR strings. Successive edits of the same type are merged. */ void align_diverging( const GBWTGraph& graph, subpath_type path, subpath_type reference, std::vector>& edits ); /* Converts a sequence of edit operations to a CIGAR string. The edits are given as pairs (type, length). */ std::string to_cigar(const std::vector>& edits); //------------------------------------------------------------------------------ } // namespace gbwtgraph #endif // GBWTGRAPH_SUBGRAPH_H jltsiren-gbwtgraph-e27bc43/include/gbwtgraph/utils.h000066400000000000000000000622131524136466400227020ustar00rootroot00000000000000#ifndef GBWTGRAPH_UTILS_H #define GBWTGRAPH_UTILS_H #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* utils.h: Common utilities and definitions. */ namespace gbwtgraph { //------------------------------------------------------------------------------ // We should be using std::hardware_destructive_interference_size, but Apple // Clang does not support it and GCC really hates it. So we just use 128, // which is a bit wasteful on Intel. inline constexpr size_t CACHE_LINE_SIZE = 128; //------------------------------------------------------------------------------ // Import type definitions from libhandlegraph. using nid_t = handlegraph::nid_t; using offset_t = handlegraph::offset_t; using subrange_t = handlegraph::subrange_t; using pos_t = handlegraph::pos_t; using handle_t = handlegraph::handle_t; using path_handle_t = handlegraph::path_handle_t; using PathSense = handlegraph::PathSense; using step_handle_t = handlegraph::step_handle_t; using edge_t = handlegraph::edge_t; using oriented_node_range_t = handlegraph::oriented_node_range_t; using HandleGraph = handlegraph::HandleGraph; using PathHandleGraph = handlegraph::PathHandleGraph; using PathMetadata = handlegraph::PathMetadata; using SerializableHandleGraph = handlegraph::SerializableHandleGraph; using NamedNodeBackTranslation = handlegraph::NamedNodeBackTranslation; //------------------------------------------------------------------------------ // Split a view into subviews using the given separator character. std::vector split_view(std::string_view str_view, char separator); // This is the equivalent of `std::string_view` for paths // represented as `gbwt::vector_type`. typedef std::pair subpath_type; // Returns a subpath corresponding the half-open interval [from, to) in the given path. inline subpath_type get_subpath(const gbwt::vector_type& path, size_t from, size_t to) { return subpath_type(path.data() + from, to - from); } // Parses an unsigned integer from a view. // Returns (0, false) if parsing failed. // We could also use std::from_chars, but these will remain for now. template std::pair parse_unsigned(std::string_view str_view) { IntegerType result = 0; for(auto cursor = str_view.begin(); cursor != str_view.end(); ++cursor) { char c = *cursor; if(c < '0' || c > '9') { return std::make_pair(0, false); } IntegerType digit = static_cast(c - '0'); result = result * 10 + digit; } return std::make_pair(result, true); } // Parses an unsigned integer. // Throws std::runtime_error with `msg + str` if parsing fails. template IntegerType parse_unsigned_or_throw(const std::string& str, const std::string& msg) { auto parse = parse_unsigned(str); if(!parse.second) { throw std::runtime_error(msg + str); } return parse.first; } //------------------------------------------------------------------------------ // Custom exception that tells that something is wrong with the GBWT index. class InvalidGBWT : public std::runtime_error { public: explicit InvalidGBWT(const std::string& message) : std::runtime_error(message) {} explicit InvalidGBWT(const char* message) : std::runtime_error(message) {} }; //------------------------------------------------------------------------------ // Some tools may not work with nodes longer than this. constexpr size_t MAX_NODE_LENGTH = 1024; // Paths with this sample name (`_gbwt_ref`) are interpreted as generic named // paths. All generic paths must have distinct contig names that can be used as // path names. extern const std::string GENERIC_PATH_SAMPLE_NAME; [[deprecated("Use gbwtgraph::GENERIC_PATH_SAMPLE_NAME instead")]] extern const std::string REFERENCE_PATH_SAMPLE_NAME; // Which paths are reference vs. haplotype paths is determined by storing the // sample names in this GBWT tag extern const std::string REFERENCE_SAMPLE_LIST_GBWT_TAG; // Which paths are reference vs. haplotype paths is determined by storing the // sample names in this GFA tag extern const std::string REFERENCE_SAMPLE_LIST_GFA_TAG; // What separator character is used in lists of reference sample names? extern const char REFERENCE_SAMPLE_LIST_SEPARATOR; // What sample name prefix is used for generated path cover paths that do not represent observed haplotypes? extern const std::string COVER_PATH_SAMPLE_PREFIX; // Cached information for a named path. struct NamedPath { gbwt::size_type id; // Original path id. gbwt::edge_type from, to; // First / last position on the path, or `gbwt::invalid_edge()` if empty. size_t length; PathSense sense; }; //------------------------------------------------------------------------------ struct Version { static std::string str(bool verbose = false); static void print(std::ostream& out, const std::string& tool_name, bool verbose = false, size_t new_lines = 2); constexpr static size_t MAJOR_VERSION = 1; constexpr static size_t MINOR_VERSION = 5; constexpr static size_t PATCH_VERSION = 0; constexpr static size_t GBZ_VERSION = 3; constexpr static size_t GRAPH_VERSION = 4; constexpr static size_t MINIMIZER_VERSION = 11; const static std::string SOURCE_KEY; // source const static std::string SOURCE_VALUE; // jltsiren/gbwtgraph }; //------------------------------------------------------------------------------ // Keep the sample names in order to guarantee consistent conversions between tags and sets. typedef std::set sample_name_set; // Parse a path sense tag value to a collection of reference-sense sample names. sample_name_set parse_reference_samples_tag(const char* cursor, const char* end); // Parse a path sense tag value to a collection of reference-sense sample names. sample_name_set parse_reference_samples_tag(std::string_view tag_value); // Parse the reference samples tag embedded in a GBWT index. sample_name_set parse_reference_samples_tag(const gbwt::GBWT& index); // Returns the subset of the given sample names that are present in the given GBWT index. sample_name_set present_sample_names(const sample_name_set& sample_names, const gbwt::GBWT& index); // Compose a reference sample name tag from a collection of reference-sense sample names. std::string compose_reference_samples_tag(const sample_name_set& reference_samples); //------------------------------------------------------------------------------ // Because we want to be able to work with path metadata with just the GBWT, we // expose the utility functions for dealing with it. The packing format only // has to touch these functions and MetadataBuilder, and some of the GBWTGraph // search methods. // Determine the sense that a path ought to have, from stored metadata. // Takes a reference path set from parse_reference_samples_tag, and handles identifying // the string sample name and defaulting un-mentioned samples. // Tolerates incomplete metadata. PathSense get_path_sense(const gbwt::Metadata& metadata, const gbwt::PathName& path_name, const sample_name_set& reference_samples); // Determine the sense a path ought to have, given an index and the reference // samples set parsed from it with parse_reference_samples_tag. // Tolerates missing metadata. PathSense get_path_sense(const gbwt::GBWT& index, gbwt::size_type path_number, const sample_name_set& reference_samples); // Determine the sense that paths for a sample ought to have, given the sample number. // Tolerates incomplete metadata. PathSense get_sample_sense(const gbwt::Metadata& metadata, gbwt::size_type sample, const sample_name_set& reference_samples); // Determine the sense that paths for a sample ought to have, given the string sample name. // The metadata is assumed to be populated with sample names. PathSense get_sample_sense(const std::string& sample_name, const sample_name_set& reference_samples); // Determine the sample name that a path ought to present, from stored metadata. // Tolerates incomplete metadata. std::string get_path_sample_name(const gbwt::Metadata& metadata, const gbwt::PathName& path_name, PathSense sense); // Determine the sample name that a path ought to present, from stored metadata. // Tolerates missing metadata. std::string get_path_sample_name(const gbwt::GBWT& index, gbwt::size_type path_number, PathSense sense); // Determine the locus name that a path ought to present, from stored metadata. // Tolerates incomplete metadata. std::string get_path_locus_name(const gbwt::Metadata& metadata, const gbwt::PathName& path_name, PathSense sense); // Determine the locus name that a path ought to present, from stored metadata. // Tolerates missing metadata. std::string get_path_locus_name(const gbwt::GBWT& index, gbwt::size_type path_number, PathSense sense); // Determine the haplotype phase number that a path ought to present, from stored metadata. // Tolerates incomplete metadata. size_t get_path_haplotype(const gbwt::Metadata& metadata, const gbwt::PathName& path_name, PathSense sense); // Determine the haplotype phase number that a path ought to present, from stored metadata. // Tolerates missing metadata. size_t get_path_haplotype(const gbwt::GBWT& index, gbwt::size_type path_number, PathSense sense); // Determine the phase block number a path ought to present, from stored metadata. // Tolerates incomplete metadata. size_t get_path_phase_block(const gbwt::Metadata& metadata, const gbwt::PathName& path_name, PathSense sense); // Determine the phase block number a path ought to present, from stored metadata. // Tolerates missing metadata. size_t get_path_phase_block(const gbwt::GBWT& index, gbwt::size_type path_number, PathSense sense); // Determine the subrange that a path ought to present, from stored metadata. // Tolerates incomplete metadata. subrange_t get_path_subrange(const gbwt::Metadata& metadata, const gbwt::PathName& path_name, PathSense sense); // Determine the subrange that a path ought to present, from stored metadata. // Tolerates missing metadata. subrange_t get_path_subrange(const gbwt::GBWT& index, gbwt::size_type path_number, PathSense sense); // Get a libhandlegraph path name string from the given stored path metadata. // Tolerates incomplete metadata. std::string compose_path_name(const gbwt::Metadata& metadata, const gbwt::PathName& path_name, PathSense sense); // Get a libhandlegraph path name string from the given stored path metadata. // Tolerates missing metadata. std::string compose_path_name(const gbwt::GBWT& index, gbwt::size_type path_number, PathSense sense); // We also have write accessors. // For write access to sample name, locus, haplotype, phase block, and // subrange, see MetadataBuilder::add_path(). // Set the senses for samples' paths, into the given GBWT tag set. void set_sample_path_senses(gbwt::Tags& tags, const std::unordered_map& senses); //------------------------------------------------------------------------------ // Functions for pos_t manipulation. inline pos_t make_pos_t(nid_t id, bool is_rev, offset_t off) { return std::make_tuple(id, is_rev, off); } inline nid_t id(const pos_t& pos) { return std::get<0>(pos); } inline bool is_rev(const pos_t& pos) { return std::get<1>(pos); } inline offset_t offset(const pos_t& pos) { return std::get<2>(pos); } inline nid_t& get_id(pos_t& pos) { return std::get<0>(pos); } inline bool& get_is_rev(pos_t& pos) { return std::get<1>(pos); } inline offset_t& get_offset(pos_t& pos) { return std::get<2>(pos); } inline bool is_empty(const pos_t& pos) { return (id(pos) == 0); } // Get a pos_t for the same base in the other orientation. inline pos_t reverse_base_pos(const pos_t& pos, size_t node_length) { return make_pos_t(id(pos), !is_rev(pos), (node_length - 1) - offset(pos)); } inline std::ostream& operator<<(std::ostream& out, const pos_t& pos) { return out << id(pos) << (is_rev(pos) ? "-" : "+") << offset(pos); } //------------------------------------------------------------------------------ /* Thomas Wang's integer hash function. In many implementations, std::hash is identity function for integers, which leads to performance issues. */ inline size_t wang_hash_64(size_t key) { key = (~key) + (key << 21); // key = (key << 21) - key - 1; key = key ^ (key >> 24); key = (key + (key << 3)) + (key << 8); // key * 265 key = key ^ (key >> 14); key = (key + (key << 2)) + (key << 4); // key * 21 key = key ^ (key >> 28); key = key + (key << 31); return key; } // Essentially boost::hash_combine. inline size_t hash(nid_t id, bool is_rev, size_t offset) { size_t result = wang_hash_64(id); result ^= wang_hash_64(is_rev) + 0x9e3779b9 + (result << 6) + (result >> 2); result ^= wang_hash_64(offset) + 0x9e3779b9 + (result << 6) + (result >> 2); return result; } inline size_t hash(const pos_t& pos) { return hash(id(pos), is_rev(pos), offset(pos)); } //------------------------------------------------------------------------------ // Stream buffer that encapsulates OpenSSL message digest calculation. // Uses SHA-256 by default. class DigestBuf : public std::streambuf { public: explicit DigestBuf(const EVP_MD* algorithm = EVP_sha256()); ~DigestBuf(); DigestBuf(const DigestBuf&) = delete; DigestBuf& operator=(const DigestBuf&) = delete; DigestBuf(DigestBuf&&) = delete; DigestBuf& operator=(DigestBuf&&) = delete; bool good() const { return (this->context != nullptr); } // Finish the digest calculation, close the stream, and get the digest as a hex string. // Returns an empty string on failure. std::string finish(); protected: int overflow(int_type ch) override; std::streamsize xsputn(const char* s, std::streamsize n) override; EVP_MD_CTX* context; }; // Output stream that calculates OpenSSL message digests. // Uses SHA-256 by default. class DigestStream : public std::ostream { public: explicit DigestStream(const EVP_MD* algorithm = EVP_sha256()); DigestStream(const DigestStream&) = delete; DigestStream& operator=(const DigestStream&) = delete; DigestStream(DigestStream&&) = delete; DigestStream& operator=(DigestStream&&) = delete; // Finish the digest calculation, close the stream, and get the digest as a hex string. // Returns an empty string on failure. std::string finish(); protected: DigestBuf buffer; }; //------------------------------------------------------------------------------ /* An object that stores graph names and known relationships between graphs. Graph names are assumed to be pggname strings (https://github.com/jltsiren/pggname), but they can be any non-empty strings. Both subgraph and coordinate translation relations are supported. Graph names and relationships can be extracted from and stored as gbwt::Tags, GFA header lines, and GAF header lines. */ class GraphName { public: GraphName() = default; // Constructor with a name but no known relationships. explicit GraphName(const std::string& name) : pggname(name) {}; // Constructor from GBZ or MinimizerIndex tags. // Throws std::runtime_error on malformed tags. explicit GraphName(const gbwt::Tags& tags); // Constructor from GFA or GAF header lines. // Throws std::runtime_error on malformed headers. explicit GraphName(const std::vector& header_lines); GraphName(const GraphName&) = default; GraphName& operator=(const GraphName&) = default; GraphName(GraphName&&) noexcept = default; GraphName& operator=(GraphName&&) noexcept = default; // Adds a new subgraph relationship. // No effect if the names are the same or either of them is empty. void add_subgraph(std::string_view subgraph, std::string_view supergraph); // Adds a new translation relationship. // No effect if the names are the same or either of them is empty. void add_translation(std::string_view from, std::string_view to); // Adds all relationships from another GraphName object. void add_relationships(const GraphName& another); // Returns the name of the graph. const std::string& name() const { return this->pggname; } // Returns true if the name of the graph has been set. bool has_name() const { return !(this->pggname.empty()); } // Sets the GBZ / MinimizerIndex tags according to this object. void set_tags(gbwt::Tags& tags) const; // Returns this object as GFA header lines. std::vector gfa_header_lines() const; // Returns this object as GAF header lines. std::vector gaf_header_lines() const; // Equality operator for the objects. bool operator==(const GraphName& another) const { return (this->pggname == another.pggname && this->subgraph == another.subgraph && this->translation == another.translation); } // Returns true if these graph names refer to the same graph // (if they have the same non-empty pggname). bool same(const GraphName& another) const { return (!(this->pggname.empty()) && this->pggname == another.pggname); } // Returns true if this graph is a subgraph of the given graph. bool subgraph_of(const GraphName& another) const; // Returns true if coordinates in this graph translate to the given graph. bool translates_to(const GraphName& another) const; /* Returns a textual description of the relationship between this graph and another graph. The relationship can be in either direction. The format is: Name 1 is for Graph i [is a subgraph of|translates to] graph i+1 Name N is for With graph names: i If a graph lacks a name, "(no name)" will be used. */ std::string describe_relationship(const GraphName& another, const std::string& this_desc, const std::string& another_desc) const; private: std::string pggname; std::map> subgraph; // (A, { B }) for "A is subgraph of B". std::map> translation; // (A, { B }) for "coordinates in A translate to B". // Returns a vector of subgraph relationships between the given graphs, // such that v[i] is subgraph of v[i + 1], or an empty vector if there is no such path. // Uses relationships in the current object. std::vector find_subgraph_path(const GraphName& from, const GraphName& to) const; // Returns a vector of subgraph or translation relationships between the given graphs, // such that v[i].first is a subgraph of v[i + 1].first (if v[i].second is false) or // translates to v[i + 1].first (if v[i].second is true), // or an empty vector if there is no such path. // Uses relationships in the current object and prioritizes subgraph relationships. std::vector> find_path(const GraphName& from, const GraphName& to) const; public: const static std::string GBZ_NAME_TAG; // "pggname" const static std::string GBZ_SUBGRAPH_TAG; // "subgraph" const static std::string GBZ_TRANSLATION_TAG; // "translation" const static std::string GBZ_TRANSLATION_TARGET_TAG; // "translation_target"; not stored in GraphName const static std::string GFA_NAME_TAG; // "NM" const static std::string GAF_NAME_TAG; // "RN" const static std::string GFA_GAF_SUBGRAPH_TAG; // "SG" const static std::string GFA_GAF_TRANSLATION_TAG; // "TL" constexpr static char GFA_HEADER_PREFIX = 'H'; constexpr static char GAF_HEADER_PREFIX = '@'; constexpr static char GFA_GAF_FIELD_SEPARATOR = '\t'; constexpr static char GFA_GAF_TAG_SEPARATOR = ':'; constexpr static char GFA_GAF_TAG_STR_TYPE = 'Z'; constexpr static char RELATIONSHIP_SEPARATOR = ','; // Between graph names in subgraph/translation tags. constexpr static char RELATIONSHIP_LIST_SEPARATOR = ';'; // Between subgraph/translation entries in GBZ tags. }; //------------------------------------------------------------------------------ // Utility functions. char complement(char base); std::string reverse_complement(std::string_view seq); void reverse_complement_in_place(std::string& seq); /* An edge is in canonical orientation, if: 1. The destination node has a higher identifier than the source node. 2. It is a self-loop with at least one end in forward orientation. */ bool edge_is_canonical(gbwt::node_type from, gbwt::node_type to); /* A path is in canonical orientation, if: 1. It is empty. 2. Both the first and the last node are in forward orientation. 3. The edge from the first node to the last node would be in canonical orientation. */ bool path_is_canonical(const gbwt::vector_type& path); //------------------------------------------------------------------------------ /* A structure for building GBWT metadata in a way that is compatible with vg and the libhandlegraph path metadata model. All paths are assigned to a job, with 0 as the default job. When the GBWT metadata is extracted using get_metadata(), the paths are ordered by job. This corresponds to building partial GBWTs for each job and merging them in the same order. Note that this class is not thread-safe. When using multiple GBWT construction jobs, the best practice is to build the metadata first. Constructor and the methods for handling paths/walks throw `std::runtime_error` on failure. */ class MetadataBuilder { public: // Construct a MetadataBuilder with no path name formats. MetadataBuilder(); // Construct a MetadataBuilder from pre-existing metadata and no path name formats. // Throws `std::runtime_error` if the metadata does not contain sample, contig, // and path names. MetadataBuilder(const gbwt::Metadata& source); // Construct a MetadataBuilder with one path name format. MetadataBuilder( const std::string& path_name_regex, const std::string& path_name_prefix, PathSense path_sense = PathSense::GENERIC); // An empty value. constexpr static size_t NO_FIELD = std::numeric_limits::max(); // A structure that parses path metadata from a string name. struct PathMetadataBuilder { std::regex parser; // Mapping from regex submatches to GBWT path name components. size_t sample_field, contig_field, haplotype_field, fragment_field; PathSense sense; PathMetadataBuilder(const std::string& path_name_regex, const std::string& path_name_prefix, PathSense path_sense); }; // List of possible path name formats ordered by priority. std::vector path_name_formats; // GBWT metadata. std::map sample_names, contig_names; std::set> haplotypes; // (sample id, phase id) std::vector> path_names; std::map counts; // This can be set to true to enable warnings about the use of sample name // GENERIC_PATH_SAMPLE_NAME in inappropriate situations. bool generic_path_sample_warning; // Register a format for parsing path names. Formats are tried in order until one matches. void add_path_name_format(const std::string& path_name_regex, const std::string& path_name_fields, PathSense path_sense); // Add a path defined by libhandlegraph metadata to the given job. // Doesn't create metadata for samples or contigs if the no-name sentinel is // passed for a sense that usually has them. void add_path(PathSense sense, const std::string& sample_name, const std::string& locus_name, size_t haplotype, size_t phase_block, const handlegraph::subrange_t& subrange, size_t job = 0); // Parse a path name using a regex to determine metadata, and assign it to the given job. void add_path(const std::string& name, size_t job = 0); // Add a path based on GFA walk metadata and assign it to the given job. void add_walk(const std::string& sample, const std::string& haplotype, const std::string& contig, const std::string& start, size_t job = 0); // Add a GBWT path and assign it to the given job. void add_gbwt_path(const gbwt::FullPathName& path_name, size_t job = 0); // Add a haplotype path and assign it to the given job. void add_haplotype(const std::string& sample, const std::string& contig, size_t haplotype, size_t fragment, size_t job = 0); // Add a generic path and assign it to the given job. void add_generic_path(const std::string& name, size_t job = 0); bool empty() const { return this->path_names.empty(); } // Build GBWT metadata from the current contents. // Paths come out ordered by job. gbwt::Metadata get_metadata() const; // Clears the existing metadata but leaves the path name formats intact. void clear(); private: // Add a path name to the given job, increasing the number of jobs if necessary. void add_path_name(const gbwt::PathName& path_name, size_t job); // Returns a vector of strings `result` such that `result[i] = "value"` if // `source["value"] = i`. The strings are ordered by the values. The // mapped values are assumed to be distinct and from `0` to `source.size() - 1`. static std::vector map_to_vector(const std::map& source); }; //------------------------------------------------------------------------------ } // namespace gbwtgraph #endif // GBWTGRAPH_UTILS_H jltsiren-gbwtgraph-e27bc43/install.sh000077500000000000000000000034301524136466400177620ustar00rootroot00000000000000#!/bin/bash # ./install.sh [directory [jobs]] # Install GBWTGraph headers and library to the home directory or to the # specified directory. The number of parallel make jobs is equal to the number # of logical CPU cores unless specified otherwise. GBWTGRAPH_DIR=$(pwd) INCLUDE_DIR=include INCLUDE=gbwtgraph LIBRARY_DIR=lib LIBRARY=libgbwtgraph.a # Determine the directory where we will install GBWTGraph. if [ $# -ge 1 ]; then PREFIX=${1} else PREFIX=${HOME} fi mkdir -p "${PREFIX}" 2> /dev/null cd "${PREFIX}" > /dev/null 2>&1 if [ $? != 0 ]; then echo "Error: Directory '${PREFIX}' does not exist and cannot be created." exit 1 else PREFIX=$(pwd -P) fi echo "Installing GBWTGraph to '${PREFIX}'." # Determine the number of jobs. if [ $# -ge 2 ]; then JOBS=${2} else JOBS=$(getconf _NPROCESSORS_ONLN) fi echo "Running ${JOBS} parallel make jobs." # Create the include/library directories. mkdir -p "${INCLUDE_DIR}" 2> /dev/null if [ $? != 0 ]; then echo "Error: Directory '${PREFIX}/${INCLUDE_DIR}' does not exist and cannot be created." exit 1 fi mkdir -p "${LIBRARY_DIR}" 2> /dev/null if [ $? != 0 ]; then echo "Error: Directory '${PREFIX}/${LIBRARY_DIR}' does not exist and cannot be created." exit 1 fi # Build GBWTGraph. cd "${GBWTGRAPH_DIR}" make clean > /dev/null if [ $? != 0 ]; then echo "Error: Cleanup failed." exit 1 fi make -j "${JOBS}" if [ $? != 0 ]; then echo "Error: Could not compile GBWTGraph." exit 1 fi # Install GBWTGraph. cp -R "${INCLUDE_DIR}/${INCLUDE}" "${PREFIX}/${INCLUDE_DIR}" 2> /dev/null if [ $? != 0 ]; then echo "Error: Could not install the headers." exit 1 fi cp "${LIBRARY_DIR}/${LIBRARY}" "${PREFIX}/${LIBRARY_DIR}" 2> /dev/null if [ $? != 0 ]; then echo "Error: Could not install the library." exit 1 fi echo "GBWTGraph installed to '${PREFIX}'." jltsiren-gbwtgraph-e27bc43/src/000077500000000000000000000000001524136466400165445ustar00rootroot00000000000000jltsiren-gbwtgraph-e27bc43/src/algorithms.cpp000066400000000000000000000605141524136466400214270ustar00rootroot00000000000000#include #include #include #include #include namespace gbwtgraph { //------------------------------------------------------------------------------ // A quick-and-dirty union-find data structure using path splitting and // union by rank. struct DisjointSets { std::vector parent; std::vector rank; // Rank is at most ~log(size). nid_t offset; // Node i is stored at [i - offset]. DisjointSets(size_t n, nid_t offset) : parent(n, 0), rank(n, 0), offset(offset) { for(size_t i = 0; i < this->size(); i++) { this->parent[i] = i; } } size_t size() const { return this->parent.size(); } void clear() { this->parent = std::vector(); this->rank = std::vector(); } size_t find(nid_t node) { size_t element = node - this->offset; while(this->parent[element] != element) { size_t next = this->parent[element]; this->parent[element] = this->parent[next]; element = next; } return element; } void set_union(nid_t node_a, nid_t node_b) { size_t a = this->find(node_a), b = this->find(node_b); if(a == b) { return; } if(this->rank[a] < this->rank[b]) { std::swap(a, b); } this->parent[b] = a; if(this->rank[b] == this->rank[a]) { this->rank[a]++; } } std::vector> sets(const std::function& include_node) { std::vector> result; std::unordered_map root_to_set; for(nid_t node = this->offset; node < static_cast(this->offset + this->size()); node++) { if(!include_node(node)) { continue; } size_t root = this->find(node); if(root_to_set.find(root) == root_to_set.end()) { root_to_set[root] = result.size(); result.emplace_back(); } result[root_to_set[root]].push_back(node); } return result; } }; //------------------------------------------------------------------------------ std::vector> weakly_connected_components(const HandleGraph& graph) { if(graph.get_node_count() == 0) { return std::vector>(); } nid_t min_id = graph.min_node_id(), max_id = graph.max_node_id(); sdsl::bit_vector found(max_id + 1 - min_id, 0); DisjointSets components(max_id + 1 - min_id, min_id); graph.for_each_handle([&](const handle_t& handle) { nid_t start_id = graph.get_id(handle); if(found[start_id - min_id]) { return; } std::stack handles; handles.push(handle); while(!(handles.empty())) { handle_t h = handles.top(); handles.pop(); nid_t id = graph.get_id(h); if(found[id - min_id]) { continue; } found[id - min_id] = true; auto handle_edge = [&](const handle_t& next) { components.set_union(id, graph.get_id(next)); handles.push(next); }; graph.follow_edges(h, false, handle_edge); graph.follow_edges(h, true, handle_edge); } }, false); return components.sets([&](nid_t node) -> bool { return graph.has_node(node); }); } //------------------------------------------------------------------------------ std::vector is_nice_and_acyclic(const HandleGraph& graph, const std::vector& component) { std::vector head_nodes; if(component.empty()) { return head_nodes; } constexpr size_t NOT_SEEN = std::numeric_limits::max(); std::unordered_map> nodes; // (remaining indegree, orientation) std::stack active; size_t found = 0; // Number of nodes that have become head nodes. // Find the head nodes. size_t missing_nodes = 0; for(nid_t node : component) { if(!(graph.has_node(node))) { missing_nodes++; continue; } handle_t handle = graph.get_handle(node, false); size_t indegree = graph.get_degree(handle, true); if(indegree == 0) { nodes[node] = std::make_pair(indegree, false); head_nodes.push_back(node); active.push(handle); found++; } else { nodes[node] = std::make_pair(NOT_SEEN, false); } } // Active nodes are the current head nodes. Process the successors, determine their // orientations, and decrement their indegrees. If the indegree becomes 0, activate // the node. bool ok = true; while(!(active.empty())) { handle_t curr = active.top(); active.pop(); graph.follow_edges(curr, false, [&](const handle_t& next) -> bool { nid_t next_id = graph.get_id(next); bool next_orientation = graph.get_is_reverse(next); auto iter = nodes.find(next_id); if(iter->second.first == NOT_SEEN) // First visit to the node. { iter->second.first = graph.get_degree(next, true); iter->second.second = next_orientation; } else if(next_orientation != iter->second.second) // Already visited, wrong orientation. { ok = false; return false; } iter->second.first--; if(iter->second.first == 0) { active.push(next); found++; } return true; }); if(!ok) { break; } } if(found != component.size() - missing_nodes) { ok = false; } if(!ok) { head_nodes.clear(); } return head_nodes; } //------------------------------------------------------------------------------ std::vector topological_order(const HandleGraph& graph, const std::unordered_set& subgraph) { std::vector result; result.reserve(2 * subgraph.size()); if(subgraph.empty()) { return result; } std::unordered_map indegrees; indegrees.reserve(2 * subgraph.size()); std::stack active; // Add all handles to the map. size_t missing_nodes = 0; for(nid_t node : subgraph) { if(!(graph.has_node(node))) { missing_nodes++; continue; } indegrees[graph.get_handle(node, false)] = 0; indegrees[graph.get_handle(node, true)] = 0; } // Determine indegrees and activate head nodes. for(auto iter = indegrees.begin(); iter != indegrees.end(); ++iter) { graph.follow_edges(iter->first, true, [&](const handle_t& next) -> bool { if(indegrees.find(next) != indegrees.end()) { iter->second++; } return true; }); if(iter->second == 0) { active.push(iter->first); result.push_back(iter->first); } } // Follow edges from active nodes and activate the nodes we have reached using // all incoming edges. while(!(active.empty())) { handle_t curr = active.top(); active.pop(); graph.follow_edges(curr, false, [&](const handle_t& next) -> bool { auto iter = indegrees.find(next); if(iter == indegrees.end()) { return true; } iter->second--; if(iter->second == 0) { active.push(iter->first); result.push_back(iter->first); } return true; }); } if(result.size() != 2 * (subgraph.size() - missing_nodes)) { result.clear(); } return result; } //------------------------------------------------------------------------------ std::pair, std::vector> chunk_graph(const GBZ& gbz, const ChunkParameters& params) { if(params.verbose) { std::cerr << "Chunking the graph" << std::endl; } // Find the weakly connected components. ConstructionJobs jobs = gbwt_construction_jobs(gbz.graph, 1); std::vector contig_names = jobs.contig_names(gbz.graph); if(params.verbose) { std::cerr << "Found " << jobs.components() << " weakly connected components" << std::endl; } // Determine the components we want to extract, or leave it empty to extract all of them. std::map selected_components; // (original component id, selected component rank) if(!params.contig_name.empty()) { // The given contig name may not be for a reference path. gbwt::size_type contig_id = gbz.index.metadata.contig(params.contig_name); auto path_ids = gbz.index.metadata.pathsForContig(contig_id); for(gbwt::size_type path_id : path_ids) { gbwt::edge_type start = gbz.index.start(gbwt::Path::encode(path_id, false)); size_t component_id = jobs.component(gbwt::Node::id(start.first)); if(component_id < jobs.components()) { // We cannot use operator[], as that would increase the size before we access it. selected_components.insert(std::make_pair(component_id, selected_components.size())); } } if(params.verbose) { std::cerr << "Selected " << selected_components.size() << " components for contig " << params.contig_name << std::endl; } if(selected_components.empty()) { return std::pair, std::vector>(); } } // Split the GBWT. size_t components = (selected_components.empty() ? jobs.components() : selected_components.size()); auto node_to_component = [&](gbwt::node_type node) -> gbwt::size_type { nid_t id = gbwt::Node::id(node); size_t component_id = jobs.component(id); if(selected_components.empty()) { return component_id; } auto iter = selected_components.find(component_id); if(iter != selected_components.end()) { return iter->second; } else { // Skip this node. return components; } }; std::vector component_gbwts = gbz.index.split(components, node_to_component); if(params.verbose) { std::cerr << "Split the GBWT into " << component_gbwts.size() << " parts" << std::endl; } // Initialize the result. std::pair, std::vector> result; result.first.reserve(components); result.second.reserve(components); if(selected_components.empty()) { for(size_t i = 0; i < components; i++) { result.second.emplace_back(contig_names[i]); } } else { size_t index = 0; for(auto iter = selected_components.begin(); iter != selected_components.end(); ++iter, index++) { result.second.emplace_back(contig_names[iter->first]); } } // And now build the GBZs. Note that we need to pass the reference samples manually, // as the GBWT library does not know about them. auto ref_samples = parse_reference_samples_tag(gbz.index); for(size_t i = 0; i < components; i++) { auto present_ref_samples = present_sample_names(ref_samples, component_gbwts[i]); if(!present_ref_samples.empty()) { component_gbwts[i].tags.set(REFERENCE_SAMPLE_LIST_GBWT_TAG, compose_reference_samples_tag(present_ref_samples)); } result.first.emplace_back(std::move(component_gbwts[i]), gbz); } if(params.verbose) { std::cerr << "Extracted " << result.first.size() << " chunks" << std::endl; } return result; } //------------------------------------------------------------------------------ /* A point in the dynamic programming matrix for LCS computation. Each point represents the furthest point reached with the given number of weighted edits on a weighted diagonal. The point itself points to the first unmatched pair of nodes. TODO: We could save space by using 32-bit integers. TODO: We could derive the weight using the invariant. TODO: One of the offsets is redunant. */ struct LCSPoint { // Twice the total length of the matched nodes. // The invariant is: weight + edits = a_prefix_sum + b_prefix_sum. size_t weight; // Non-weighted offsets in the two sequences. size_t a_offset, b_offset; // Length of the non-weighted run of matches ending at this point. size_t matches; }; /* Dynamic programming matrix for LCS computation. Because the LCS is weighted by sequence length, we precompute prefix sums of weights over both paths. The algorithm is a variant of Myers' O(nd) algorithm. We therefore store only the furthest points reached with each (weighted) number of edits on each (weighted) diagonal. TODO: We could save space by using 32-bit integers. */ struct LCSMatrix { const gbwt::vector_type& a; const gbwt::vector_type& b; // Prefix sums of sequence lengths on the two paths. std::vector a_prefix_sum, b_prefix_sum; // The final result if we have found it. LCSPoint result; // Furthest point reached for each (weighted number of edits, weighted diagonal). // A diagonal is a_prefix_sum - b_prefix_sum. // The point refers to the first unmatched pair of nodes. std::map, LCSPoint> points; LCSMatrix(const GBWTGraph& graph, const gbwt::vector_type& a, const gbwt::vector_type& b) : a(a), b(b), a_prefix_sum(a.size() + 1, 0), b_prefix_sum(b.size() + 1, 0), result({ 0, 0, 0, 0 }) { for(size_t i = 1; i <= a.size(); i++) { this->a_prefix_sum[i] = this->a_prefix_sum[i - 1] + graph.get_length(GBWTGraph::node_to_handle(a[i - 1])); } for(size_t i = 1; i <= b.size(); i++) { this->b_prefix_sum[i] = this->b_prefix_sum[i - 1] + graph.get_length(GBWTGraph::node_to_handle(b[i - 1])); } this->points[std::make_pair(0, 0)] = { 0, 0, 0, 0 }; } // Returns the sorted set of diagonals for the given number of edits. std::vector diagonals(size_t edits) const { auto low = this->points.lower_bound(std::make_pair(edits, std::numeric_limits::min())); auto high = this->points.lower_bound(std::make_pair(edits + 1, std::numeric_limits::min())); std::vector result; for(auto iter = low; iter != high; ++iter) { result.push_back(iter->first.second); } return result; } // Inserts the point if it is better than the existing one. void try_insert(size_t edits, std::int64_t diagonal, LCSPoint point) { auto iter = this->points.find(std::make_pair(edits, diagonal)); if(iter == this->points.end()) { this->points[std::make_pair(edits, diagonal)] = point; } else if(point.weight > iter->second.weight) { iter->second = point; } } size_t a_weight(size_t offset) const { return this->a_prefix_sum[offset + 1] - this->a_prefix_sum[offset]; } size_t b_weight(size_t offset) const { return this->b_prefix_sum[offset + 1] - this->b_prefix_sum[offset]; } // Extends matches over all diagonals for the given number of weighted edits. // Also adds successor points reachable with a single edit from each extension. // Returns true and sets the point if we reached the end. bool extend(size_t edits) { std::vector diagonals = this->diagonals(edits); for(std::int64_t diagonal : diagonals) { auto iter = this->points.find(std::make_pair(edits, diagonal)); if(iter == this->points.end()) { continue; } LCSPoint point = iter->second; // Copy the point to avoid iterator invalidation. while(point.a_offset < this->a.size() && point.b_offset < this->b.size() && this->a[point.a_offset] == this->b[point.b_offset]) { point.weight += 2 * this->a_weight(point.a_offset); point.a_offset++; point.b_offset++; point.matches++; } if(point.matches > 0) { this->points[std::make_pair(edits, diagonal)] = point; } if(point.a_offset == this->a.size() && point.b_offset == this->b.size()) { this->result = point; return true; } if(point.a_offset < this->a.size()) { size_t weight = this->a_weight(point.a_offset); LCSPoint new_point { point.weight, point.a_offset + 1, point.b_offset, 0 }; this->try_insert(edits + weight, diagonal + std::int64_t(weight), new_point); } if(point.b_offset < this->b.size()) { size_t weight = this->b_weight(point.b_offset); LCSPoint new_point { point.weight, point.a_offset, point.b_offset + 1, 0 }; this->try_insert(edits + weight, diagonal - std::int64_t(weight), new_point); } } return false; } // Returns the next possible number of edits after the given number of edits. // Assumes that extend() has already been called for the given number of edits. // Returns std::numeric_limits::max() if there are no more possible edits. size_t next_edits(size_t edits) const { auto iter = this->points.lower_bound(std::make_pair(edits + 1, std::numeric_limits::min())); return (iter == this->points.end() ? std::numeric_limits::max() : iter->first.first); } // Returns the predecessor point and number of edits for the given point and number of edits. std::pair predecessor(size_t a_offset, size_t b_offset, size_t edits) const { std::int64_t diagonal = std::int64_t(this->a_prefix_sum[a_offset]) - std::int64_t(this->b_prefix_sum[b_offset]); auto prev = this->points.end(); if(a_offset > 0 && this->a_weight(a_offset - 1) > 0) { size_t weight = this->a_weight(a_offset - 1); prev = this->points.find(std::make_pair(edits - weight, diagonal - std::int64_t(weight))); } auto next = this->points.end(); if(b_offset > 0 && this->b_weight(b_offset - 1) > 0) { size_t weight = this->b_weight(b_offset - 1); next = this->points.find(std::make_pair(edits - weight, diagonal + std::int64_t(weight))); } if(prev != this->points.end() && next != this->points.end()) { return (prev->second.weight > next->second.weight ? std::make_pair(prev->second, prev->first.first) : std::make_pair(next->second, next->first.first)); } else if(prev != this->points.end()) { return std::make_pair(prev->second, prev->first.first); } else if(next != this->points.end()) { return std::make_pair(next->second, next->first.first); } else { return std::make_pair(LCSPoint { 0, 0, 0, 0 }, 0); } } }; std::vector> path_lcs(const GBWTGraph& graph, const gbwt::vector_type& a, const gbwt::vector_type& b) { if(a.empty() || b.empty()) { return {}; } // Find the furthest point on each diagonal with each possible number of edits, until we reach the end. LCSMatrix matrix(graph, a, b); size_t edits = 0; while(true) { if(matrix.extend(edits)) { break; } edits = matrix.next_edits(edits); if(edits == std::numeric_limits::max()) { break; } } // Trace back the LCS. std::vector> result; LCSPoint point = matrix.result; while(point.a_offset > 0 || point.b_offset > 0) { for(size_t i = 0; i < point.matches; i++) { point.a_offset--; point.b_offset--; result.push_back(std::make_pair(point.a_offset, point.b_offset)); } std::tie(point, edits) = matrix.predecessor(point.a_offset, point.b_offset, edits); } std::reverse(result.begin(), result.end()); return result; } //------------------------------------------------------------------------------ std::vector ConstructionJobs::contig_names(const PathHandleGraph& graph) const { std::function no_filter = [](const path_handle_t&) -> bool { return true; }; return this->contig_names(graph, no_filter); } std::vector ConstructionJobs::contig_names( const PathHandleGraph& graph, const std::function& filter) const { std::vector result(this->components(), ""); auto try_contig_name = [&](const path_handle_t& path) { if(!(filter(path))) { return; } nid_t node = graph.get_id(graph.get_handle_of_step(graph.path_begin(path))); size_t component = this->component(node); if(component >= result.size() || !(result[component].empty())) { return; } std::string contig_name = graph.get_locus_name(path); if(contig_name != PathMetadata::NO_LOCUS_NAME) { result[component] = contig_name; } }; // Try to get the contig names from reference paths and generic paths. graph.for_each_path_of_sense(PathSense::REFERENCE, try_contig_name); graph.for_each_path_of_sense(PathSense::GENERIC, try_contig_name); // Fallback: Component ids. for(size_t i = 0; i < this->components(); i++) { if(result[i].empty()) { result[i] = "component_" + std::to_string(i); } } return result; } std::vector> ConstructionJobs::components_per_job() const { std::vector> result(this->size()); for(size_t i = 0; i < this->components(); i++) { size_t job_id = this->job_for_component(i); if(job_id < this->size()) { result[job_id].push_back(i); } } return result; } void ConstructionJobs::clear() { this->nodes_per_job = {}; this->weakly_connected_components = {}; this->node_to_component = {}; this->component_to_job = {}; } //------------------------------------------------------------------------------ ConstructionJobs gbwt_construction_jobs(const HandleGraph& graph, size_t size_bound) { ConstructionJobs jobs; jobs.weakly_connected_components = weakly_connected_components(graph); size_t nodes = graph.get_node_count(); jobs.node_to_component.reserve(nodes); jobs.component_to_job.reserve(jobs.components()); for(size_t i = 0; i < jobs.components(); i++) { const std::vector& component = jobs.weakly_connected_components[i]; if(jobs.nodes_per_job.empty() || jobs.nodes_per_job.back() + component.size() > size_bound) { jobs.nodes_per_job.push_back(0); } jobs.nodes_per_job.back() += component.size(); for(nid_t node_id : component) { jobs.node_to_component[node_id] = i; } jobs.component_to_job[i] = jobs.size() - 1; } return jobs; } std::vector> assign_paths( const PathHandleGraph& graph, const ConstructionJobs& jobs, MetadataBuilder* metadata, const std::function* path_filter) { std::vector> result(jobs.size()); std::unordered_set senses = { PathSense::GENERIC, PathSense::REFERENCE }; graph.for_each_path_matching(&senses, nullptr, nullptr, [&](const path_handle_t& path) { // Check the path filter if we have one. if(path_filter != nullptr && !(*path_filter)(path)) { return; } // Find the job for this path. nid_t node = graph.get_id(graph.get_handle_of_step(graph.path_begin(path))); size_t job = jobs.job(node); if(job >= jobs.size()) { return; } result[job].push_back(path); if(metadata != nullptr) { metadata->add_path( graph.get_sense(path), graph.get_sample_name(path), graph.get_locus_name(path), graph.get_haplotype(path), graph.get_phase_block(path), graph.get_subrange(path), job ); } }); return result; } // Inserts the selected paths into the GBWT builder. void insert_paths( const PathHandleGraph& graph, const std::vector& paths, gbwt::GBWTBuilder& builder, size_t job_id, bool show_progress) { if(show_progress && paths.size() > 0) { #pragma omp critical { std::cerr << "Job " << job_id << ": Inserting " << paths.size() << " paths" << std::endl; } } for(const path_handle_t& path : paths) { gbwt::vector_type buffer; for(handle_t handle : graph.scan_path(path)) { buffer.push_back(gbwt::Node::encode(graph.get_id(handle), graph.get_is_reverse(handle))); } builder.insert(buffer, true); // Insert in both orientations. } } //------------------------------------------------------------------------------ std::vector> partition_chains(const handlegraph::SnarlDecomposition& snarls, const HandleGraph& graph, const ConstructionJobs& jobs) { std::vector> result(jobs.size()); size_t unassigned = 0; size_t offset = 0; snarls.for_each_child(snarls.get_root(), [&](const handlegraph::net_handle_t& chain) { bool assigned = false; snarls.for_each_child(chain, [&](const handlegraph::net_handle_t& child) -> bool { if(snarls.is_node(child)) { handle_t handle = snarls.get_handle(child, &graph); size_t job_id = jobs.job(graph.get_id(handle)); if(job_id < jobs.size()) { result[job_id].push_back({ chain, handle, offset }); assigned = true; } return false; } else { return true; } }); if(!assigned) { unassigned++; } offset++; }); if (offset != jobs.components()) { std::cerr << "partition_chains(): Warning: Found " << offset << " top-level chains in a graph with " << jobs.components() << " components" << std::endl; } if(unassigned > 0) { std::cerr << "partition_chains(): Warning: Could not assign " << unassigned << " chains to jobs" << std::endl; } return result; } //------------------------------------------------------------------------------ } // namespace gbwtgraph jltsiren-gbwtgraph-e27bc43/src/cached_gbwtgraph.cpp000066400000000000000000000120501524136466400225220ustar00rootroot00000000000000#include #include namespace gbwtgraph { //------------------------------------------------------------------------------ CachedGBWTGraph::CachedGBWTGraph() : graph(nullptr), cache() { } CachedGBWTGraph::CachedGBWTGraph(const CachedGBWTGraph& source) { this->copy(source); } CachedGBWTGraph::CachedGBWTGraph(CachedGBWTGraph&& source) noexcept { *this = std::move(source); } CachedGBWTGraph::~CachedGBWTGraph() { } void CachedGBWTGraph::swap(CachedGBWTGraph& another) noexcept { if(&another == this) { return; } std::swap(this->graph, another.graph); this->cache.swap(another.cache); } CachedGBWTGraph& CachedGBWTGraph::operator=(const CachedGBWTGraph& source) { if(&source != this) { this->copy(source); } return *this; } CachedGBWTGraph& CachedGBWTGraph::operator=(CachedGBWTGraph&& source) noexcept { if(&source != this) { this->graph = std::move(source.graph); this->cache = std::move(source.cache); } return *this; } void CachedGBWTGraph::copy(const CachedGBWTGraph& source) { this->graph = source.graph; this->cache = source.cache; } //------------------------------------------------------------------------------ CachedGBWTGraph::CachedGBWTGraph(const GBWTGraph& graph) : graph(&graph), cache(this->graph->get_cache()) { } //------------------------------------------------------------------------------ bool CachedGBWTGraph::has_node(nid_t node_id) const { size_t offset = this->graph->node_offset(gbwt::Node::encode(node_id, false)) / 2; return (offset < this->graph->real_nodes.size() && this->graph->real_nodes[offset]); } handle_t CachedGBWTGraph::get_handle(const nid_t& node_id, bool is_reverse) const { return node_to_handle(gbwt::Node::encode(node_id, is_reverse)); } nid_t CachedGBWTGraph::get_id(const handle_t& handle) const { return gbwt::Node::id(handle_to_node(handle)); } bool CachedGBWTGraph::get_is_reverse(const handle_t& handle) const { return gbwt::Node::is_reverse(handle_to_node(handle)); } handle_t CachedGBWTGraph::flip(const handle_t& handle) const { return node_to_handle(gbwt::Node::reverse(handle_to_node(handle))); } size_t CachedGBWTGraph::get_length(const handle_t& handle) const { size_t offset = this->graph->node_offset(handle); return this->graph->sequences.length(offset); } std::string CachedGBWTGraph::get_sequence(const handle_t& handle) const { size_t offset = this->graph->node_offset(handle); return this->graph->sequences.str(offset); } char CachedGBWTGraph::get_base(const handle_t& handle, size_t index) const { size_t offset = this->graph->node_offset(handle); std::string_view view = this->graph->sequences.view(offset); return view[index]; } std::string CachedGBWTGraph::get_subsequence(const handle_t& handle, size_t index, size_t size) const { size_t offset = this->graph->node_offset(handle); std::string_view view = this->graph->sequences.view(offset); index = std::min(index, view.size()); size = std::min(size, view.size() - index); return std::string(view.data() + index, view.data() + index + size); } size_t CachedGBWTGraph::get_node_count() const { return this->graph->header.nodes; } nid_t CachedGBWTGraph::min_node_id() const { if(this->get_node_count() == 0) { return std::numeric_limits::max(); } return gbwt::Node::id(this->cache.firstNode()); } nid_t CachedGBWTGraph::max_node_id() const { if(this->get_node_count() == 0) { return 0; } nid_t next_id = gbwt::Node::id(this->cache.sigma()); return next_id - 1; } bool CachedGBWTGraph::follow_edges_impl(const handle_t& handle, bool go_left, const std::function& iteratee) const { return this->graph->cached_follow_edges(this->cache, handle, go_left, iteratee); } bool CachedGBWTGraph::for_each_handle_impl(const std::function& iteratee, bool parallel) const { // This is expensive enough that we can afford using virtual functions. return this->graph->for_each_handle_impl(iteratee, parallel); } size_t CachedGBWTGraph::get_degree(const handle_t& handle, bool go_left) const { // Cache the node. gbwt::node_type curr = handle_to_node(handle); if(go_left) { curr = gbwt::Node::reverse(curr); } gbwt::size_type cache_index = this->cache.findRecord(curr); // The outdegree reported by GBWT might account for the endmarker, which is // always the first successor. size_t result = this->cache.outdegree(cache_index); if(result > 0 && this->cache.successor(cache_index, 0) == gbwt::ENDMARKER) { result--; } return result; } bool CachedGBWTGraph::has_edge(const handle_t& left, const handle_t& right) const { // Cache the node. gbwt::node_type curr = handle_to_node(left); gbwt::size_type cache_index = this->cache.findRecord(curr); for(gbwt::rank_type outrank = 0; outrank < this->cache.outdegree(cache_index); outrank++) { gbwt::node_type next = this->cache.successor(cache_index, outrank); if(node_to_handle(next) == right) { return true; } } return false; } //------------------------------------------------------------------------------ } // namespace gbwtgraph jltsiren-gbwtgraph-e27bc43/src/canonical_gfa.cpp000066400000000000000000000055111524136466400220160ustar00rootroot00000000000000#include #include #include #include using namespace gbwtgraph; //------------------------------------------------------------------------------ const std::string tool_name = "Canonical GFA"; struct Config { Config(int argc, char** argv); bool digest = false; bool integer_ids = false; bool handlegraph_algorithm = false; std::string filename; }; //------------------------------------------------------------------------------ int main(int argc, char** argv) { Config config(argc, argv); // Load the graph. GBZ gbz; sdsl::simple_sds::load_from(gbz, config.filename); // Choose the output stream. DigestStream digest_stream(EVP_sha256()); std::ostream& out = (config.digest ? digest_stream : std::cout); // Generate the GFA. if(config.integer_ids && !config.handlegraph_algorithm) { gbwt_to_canonical_gfa(gbz.graph, out); } else { handlegraph::algorithms::canonical_gfa(gbz.graph, out, config.integer_ids); } // Print the digest if needed. if(config.digest) { std::string digest = digest_stream.finish(); std::cout << digest << std::endl; } return 0; } //------------------------------------------------------------------------------ void printUsage(int exit_code) { Version::print(std::cerr, tool_name); std::cerr << "Usage: canonical_gfa [options] graph.gbz" << std::endl; std::cerr << std::endl; std::cerr << "Options:" << std::endl; std::cerr << " -d, --digest print the SHA-256 digest of the output instead of the output itself" << std::endl; std::cerr << " -i, --integer-ids order the nodes by integer ids" << std::endl; std::cerr << " -H, --handlegraph always use the libhandlegraph algorithm" << std::endl; std::cerr << std::endl; std::exit(exit_code); } //------------------------------------------------------------------------------ Config::Config(int argc, char** argv) { if(argc < 2) { printUsage(EXIT_SUCCESS); } // Data for `getopt_long()`. int c = 0, option_index = 0; option long_options[] = { { "digest", no_argument, 0, 'd' }, { "integer-ids", no_argument, 0, 'i' }, { "handlegraph", no_argument, 0, 'H' }, { 0, 0, 0, 0 } }; // Process options. while((c = getopt_long(argc, argv, "diH", long_options, &option_index)) != -1) { switch(c) { case 'd': this->digest = true; break; case 'i': this->integer_ids = true; break; case 'H': this->handlegraph_algorithm = true; break; case '?': std::exit(EXIT_FAILURE); default: std::exit(EXIT_FAILURE); } } // Sanity checks. if(optind >= argc) { printUsage(EXIT_FAILURE); } this->filename = argv[optind]; optind++; } //------------------------------------------------------------------------------ jltsiren-gbwtgraph-e27bc43/src/chunk_gbz.cpp000066400000000000000000000063201524136466400212230ustar00rootroot00000000000000#include #include #include #include #include using namespace gbwtgraph; //------------------------------------------------------------------------------ const std::string tool_name = "Chunk GBZ"; const std::string default_prefix = "chunk"; struct Config { Config(int argc, char** argv); std::string input; std::string output_prefix = default_prefix; ChunkParameters params; }; //------------------------------------------------------------------------------ int main(int argc, char** argv) { double start = gbwt::readTimer(); Config config(argc, argv); GBZ gbz; if(config.params.verbose) { std::cerr << "Loading the graph from " << config.input << std::endl; } sdsl::simple_sds::load_from(gbz, config.input); auto chunks = chunk_graph(gbz, config.params); if(config.params.verbose) { std::cerr << "Writing the output files" << std::endl; } for(size_t i = 0; i < chunks.first.size(); i++) { std::string filename = config.output_prefix + "_" + std::to_string(i) + "_" + chunks.second[i] + ".gbz"; sdsl::simple_sds::serialize_to(chunks.first[i], filename); } if(config.params.verbose) { double seconds = gbwt::readTimer() - start; std::cerr << "Used " << seconds << " seconds, " << gbwt::inGigabytes(gbwt::memoryUsage()) << " GiB" << std::endl; } return 0; } //------------------------------------------------------------------------------ void printUsage(int exit_code) { Version::print(std::cerr, tool_name); std::cerr << "Usage: chunk_gbz [options] graph.gbz" << std::endl; std::cerr << std::endl; std::cerr << "Chunks the given graph into weakly connected components." << std::endl; std::cerr << "Output files are named __.gbz." << std::endl; std::cerr << std::endl; std::cerr << "Options:" << std::endl; std::cerr << " -c, --contig X only extract components with this contig name" << std::endl; std::cerr << " -p, --prefix X use this output prefix (default: " << default_prefix << ")" << std::endl; std::cerr << " -v, --verbose print progress information" << std::endl; std::cerr << std::endl; std::exit(exit_code); } //------------------------------------------------------------------------------ Config::Config(int argc, char** argv) { if(argc < 2) { printUsage(EXIT_SUCCESS); } // Data for `getopt_long()`. int c = 0, option_index = 0; option long_options[] = { { "contig", required_argument, nullptr, 'c' }, { "prefix", required_argument, nullptr, 'p' }, { "verbose", no_argument, nullptr, 'v' }, { 0, 0, 0, 0 } }; // Process options. while((c = getopt_long(argc, argv, "c:p:v", long_options, &option_index)) != -1) { switch(c) { case 'c': this->params.contig_name = optarg; break; case 'p': this->output_prefix = optarg; break; case 'v': this->params.verbose = true; break; case '?': std::exit(EXIT_FAILURE); default: std::exit(EXIT_FAILURE); } } // Sanity checks. if(optind >= argc) { printUsage(EXIT_FAILURE); } this->input = argv[optind]; optind++; } //------------------------------------------------------------------------------ jltsiren-gbwtgraph-e27bc43/src/gbwtgraph.cpp000066400000000000000000002104041524136466400212360ustar00rootroot00000000000000#include #include #include #include #include #include #include #include #include #include namespace gbwtgraph { //------------------------------------------------------------------------------ // Numerical class constants. constexpr size_t GBWTGraph::CHUNK_SIZE; constexpr std::uint32_t GBWTGraph::Header::TAG; constexpr std::uint32_t GBWTGraph::Header::VERSION; constexpr std::uint64_t GBWTGraph::Header::FLAG_MASK; constexpr std::uint64_t GBWTGraph::Header::FLAG_TRANSLATION; constexpr std::uint64_t GBWTGraph::Header::FLAG_SIMPLE_SDS; constexpr std::uint32_t GBWTGraph::Header::ZSTD_VERSION; constexpr std::uint32_t GBWTGraph::Header::SIMPLE_SDS_VERSION; constexpr std::uint64_t GBWTGraph::Header::SIMPLE_SDS_FLAG_MASK; constexpr std::uint32_t GBWTGraph::Header::TRANS_VERSION; constexpr std::uint64_t GBWTGraph::Header::TRANS_FLAG_MASK; constexpr std::uint32_t GBWTGraph::Header::OLD_VERSION; constexpr std::uint64_t GBWTGraph::Header::OLD_FLAG_MASK; constexpr std::uint32_t GBWTGraph::Header::MIN_VERSION; constexpr std::uint32_t GBWTGraph::Header::MIN_SERIALIZE_VERSION; constexpr std::uint32_t GBWTGraph::Header::DEFAULT_VERSION; //------------------------------------------------------------------------------ // Other class variables. const std::string GBWTGraph::EXTENSION = ".gg"; //------------------------------------------------------------------------------ GBWTGraph::Header::Header() : tag(TAG), version(VERSION), nodes(0), flags(0) { } void GBWTGraph::Header::check() const { if(this->tag != TAG) { throw sdsl::simple_sds::InvalidData("GBWTGraph: Invalid tag"); } if(this->version > VERSION || this->version < OLD_VERSION) { std::string msg = "GBWTGraph: Expected version " + std::to_string(OLD_VERSION) + " to version " + std::to_string(VERSION) + ", got version " + std::to_string(this->version); throw sdsl::simple_sds::InvalidData(msg); } std::uint64_t mask = 0; switch(this->version) { case VERSION: mask = FLAG_MASK; break; case SIMPLE_SDS_VERSION: mask = SIMPLE_SDS_FLAG_MASK; break; case TRANS_VERSION: mask = TRANS_FLAG_MASK; break; case OLD_VERSION: mask = OLD_FLAG_MASK; break; } if((this->flags & mask) != this->flags) { throw sdsl::simple_sds::InvalidData("GBWTGraph: Invalid flags"); } } bool GBWTGraph::Header::operator==(const Header& another) const { return (this->tag == another.tag && this->version == another.version && this->nodes == another.nodes && this->flags == another.flags); } //------------------------------------------------------------------------------ GBWTGraph::GBWTGraph() : index(nullptr), header() { } GBWTGraph::GBWTGraph(const GBWTGraph& source) { this->copy(source); } GBWTGraph::GBWTGraph(GBWTGraph&& source) noexcept { *this = std::move(source); } GBWTGraph::~GBWTGraph() { } void GBWTGraph::swap(GBWTGraph& another) noexcept { if(&another == this) { return; } std::swap(this->index, another.index); std::swap(this->header, another.header); this->sequences.swap(another.sequences); this->real_nodes.swap(another.real_nodes); this->segments.swap(another.segments); this->node_to_segment.swap(another.node_to_segment); this->named_paths.swap(another.named_paths); this->name_to_path.swap(another.name_to_path); this->id_to_path.swap(another.id_to_path); this->reference_samples.swap(another.reference_samples); } GBWTGraph& GBWTGraph::operator=(const GBWTGraph& source) { if(&source != this) { this->copy(source); } return *this; } GBWTGraph& GBWTGraph::operator=(GBWTGraph&& source) noexcept { if(&source != this) { this->index = std::move(source.index); this->header = std::move(source.header); this->sequences = std::move(source.sequences); this->real_nodes = std::move(source.real_nodes); this->segments = std::move(source.segments); this->node_to_segment = std::move(source.node_to_segment); this->named_paths = std::move(source.named_paths); this->name_to_path = std::move(source.name_to_path); this->id_to_path = std::move(source.id_to_path); this->reference_samples = std::move(source.reference_samples); } return *this; } void GBWTGraph::copy(const GBWTGraph& source) { this->index = source.index; this->header = source.header; this->sequences = source.sequences; this->real_nodes = source.real_nodes; this->segments = source.segments; this->node_to_segment = source.node_to_segment; this->named_paths = source.named_paths; this->name_to_path = source.name_to_path; this->id_to_path = source.id_to_path; this->reference_samples = source.reference_samples; } void GBWTGraph::sanity_checks() { size_t nodes = sdsl::util::cnt_one_bits(this->real_nodes); if(nodes != this->header.nodes) { std::string msg = "GBWTGraph: " + std::to_string(nodes) + " nodes in real_nodes, " + std::to_string(this->header.nodes) + " in header"; throw sdsl::simple_sds::InvalidData(msg); } size_t potential_nodes = this->sequences.size(); if(this->index != nullptr && !(this->index->empty())) { potential_nodes = this->index->sigma() - this->index->firstNode(); } if(this->sequences.size() != potential_nodes) { std::string msg = "GBWTGraph: " + std::to_string(this->sequences.size()) + " sequences, " + std::to_string(potential_nodes) + " potential nodes"; throw sdsl::simple_sds::InvalidData(msg); } if(this->real_nodes.size() != potential_nodes / 2) { std::string msg = "GBWTGraph: " + std::to_string(this->real_nodes.size()) + " nodes according to real_nodes, " + std::to_string(potential_nodes / 2) + " according to sequences"; throw sdsl::simple_sds::InvalidData(msg); } if(this->node_to_segment.ones() != this->segments.size()) { std::string msg = "GBWTGraph: " + std::to_string(this->node_to_segment.ones()) + " segments, " + std::to_string(this->segments.size()) + " in node_to_segment"; throw sdsl::simple_sds::InvalidData(msg); } if(this->segments.size() > 0 && this->index != nullptr && this->node_to_segment.size() < this->index->sigma() / 2) { // Translation may have more nodes than the GBWT, if the last segments in the input GFA are not visited by any path. // TODO: We should avoid adding the unused segments at the end of node id space to the translation. std::string msg = "GBWTGraph: " + std::to_string(this->node_to_segment.size()) + " nodes in node_to_segment, " + std::to_string(this->index->sigma() / 2) + " in GBWT alphabet"; throw sdsl::simple_sds::InvalidData(msg); } } std::pair> GBWTGraph::copy_translation(const NamedNodeBackTranslation& translation) const { // This will hold an array of segment name strings, and a bit vector of // flags, where the flag is set at node IDs that // are the first in their segments. // The representation assumes that all nodes in a segment have successive IDs. std::pair> result; // NamedNodeBackTranslation copies out segment names, but to make a StringArray we need a place to store them. // We also need to know the first node ID in every segment, for the bit vector, but we need the total set bit count to start the builder. // So we store pairs of segment names and first node IDs, and end up re-numbering our segments by rank here. std::vector> segment_names_and_starts; // We need to track consistency with our contiguous node ID range segment model. nid_t prev_segment_number = std::numeric_limits::max(); bool prev_node_existed = false; size_t next_offset_along_segment = std::numeric_limits::max(); if(this->index->firstNode() > 1) { // Have a segment for the first run of nonexistent nodes segment_names_and_starts.emplace_back("", 1); } for(gbwt::node_type node = this->index->firstNode(); node < this->index->sigma(); node += 2) { // Get each node in the graph, in ID order. nid_t node_id = gbwt::Node::id(node); if(!(this->has_node(node_id))) { // This node doesn't exist. if(prev_node_existed) { // We're starting a nonexistent segment. // We need to remember its empty name and start ID. segment_names_and_starts.emplace_back("", node_id); // Remember we were in a nonexistent segment. prev_node_existed = false; } } else { // Translate it back to a segment range. size_t node_length = this->sequences.length(this->node_offset(node)); oriented_node_range_t range{node_id, false, 0, node_length}; auto translated_back = translation.translate_back(range); if(translated_back.size() != 1) { // This node didn't come from a segment, or spans multiple segments throw InvalidGBWT("GBWTGraph: Node " + std::to_string(node_id) + " did not come from exactly one segment"); } nid_t& segment_number = std::get<0>(translated_back[0]); bool& reverse_on_segment = std::get<1>(translated_back[0]); size_t& offset_along_segment = std::get<2>(translated_back[0]); if(reverse_on_segment) { // We can only deal with nodes on the forward strands of their segments. throw InvalidGBWT("GBWTGraph: Node " + std::to_string(node_id) + " came from the reverse strand of its segment"); } if(!prev_node_existed || prev_segment_number != segment_number) { // This is a new segment! prev_segment_number = segment_number; prev_node_existed = true; next_offset_along_segment = 0; // We need to remember its name and start ID. segment_names_and_starts.emplace_back(translation.get_back_graph_node_name(segment_number), node_id); } if(offset_along_segment != next_offset_along_segment) { // Actually we're not at the right place in the segment, so we can't store this translation. throw InvalidGBWT("GBWTGraph: Node " + std::to_string(node_id) + " not at expected position in segment"); } next_offset_along_segment += node_length; } } // Store the segment names. std::string empty; result.first = gbwt::StringArray(segment_names_and_starts.size(), [&](size_t offset) -> std::string_view { return std::string_view(segment_names_and_starts[offset].first); }); // Store the mapping. sdsl::sd_vector_builder builder(gbwt::Node::id(this->index->sigma()), segment_names_and_starts.size()); for(auto& record : segment_names_and_starts) { builder.set_unsafe(record.second); } result.second = sdsl::sd_vector<>(builder); return result; } //------------------------------------------------------------------------------ GBWTGraph::GBWTGraph(const gbwt::GBWT& gbwt_index, const NaiveGraph& graph) : index(nullptr) { // Set GBWT, cache named paths, and do sanity checks. this->set_gbwt(gbwt_index); if(this->index->empty()) { return; } // Build real_nodes to support has_node(). this->determine_real_nodes(); // Store the sequences. this->sequences = gbwt::StringArray(this->index->sigma() - this->index->firstNode(), [&](size_t offset) -> size_t { gbwt::node_type node = offset + this->index->firstNode(); nid_t id = gbwt::Node::id(node); if(!(this->has_node(id))) { return 0; } handle_t handle = NaiveGraph::node_to_handle(node); return graph.get_length(handle); }, [&](size_t offset) -> std::string { gbwt::node_type node = offset + this->index->firstNode(); nid_t id = gbwt::Node::id(node); if(!(this->has_node(id))) { return std::string(); } handle_t handle = NaiveGraph::node_to_handle(node); std::string result = graph.get_sequence(handle); return result; }); // Store the node to segment translation but leave the names of unused segments empty. if(graph.uses_translation()) { this->header.set(Header::FLAG_TRANSLATION); std::tie(this->segments, this->node_to_segment) = graph.invert_translation([&](std::pair interval) -> bool { return this->has_node(interval.first); }); } } GBWTGraph::GBWTGraph ( const gbwt::GBWT& gbwt_index, const HandleGraph& graph, const NamedNodeBackTranslation* segment_space ) : index(nullptr) { // Set GBWT, cache named paths, and do sanity checks. this->set_gbwt(gbwt_index); if(this->index->empty()) { return; } // Build real_nodes to support has_node(). this->determine_real_nodes(); // Store the sequences. this->sequences = gbwt::StringArray(this->index->sigma() - this->index->firstNode(), [&](size_t offset) -> size_t { gbwt::node_type node = offset + this->index->firstNode(); nid_t id = gbwt::Node::id(node); if(!(this->has_node(id))) { return 0; } handle_t handle = graph.get_handle(id, gbwt::Node::is_reverse(node)); return graph.get_length(handle); }, [&](size_t offset) -> std::string { gbwt::node_type node = offset + this->index->firstNode(); nid_t id = gbwt::Node::id(node); if(!(this->has_node(id))) { return std::string(); } handle_t handle = graph.get_handle(id, gbwt::Node::is_reverse(node)); return graph.get_sequence(handle); }); // Store the node to segment translation if(segment_space) { this->header.set(Header::FLAG_TRANSLATION); std::tie(this->segments, this->node_to_segment) = this->copy_translation(*segment_space); } } GBWTGraph GBWTGraph::subgraph(gbwt::GBWT& gbwt_index) const { GBWTGraph result; // Copy reference samples. if(this->index != nullptr) { auto reference_samples = parse_reference_samples_tag(*this->index); auto present_samples = present_sample_names(reference_samples, gbwt_index); if(!present_samples.empty()) { gbwt_index.tags.set(REFERENCE_SAMPLE_LIST_GBWT_TAG, compose_reference_samples_tag(present_samples)); } else { gbwt_index.tags.unset(REFERENCE_SAMPLE_LIST_GBWT_TAG); } } else { gbwt_index.tags.unset(REFERENCE_SAMPLE_LIST_GBWT_TAG); } // Set GBWT, cache named paths, and do sanity checks. result.set_gbwt(gbwt_index); if(result.index->empty()) { return result; } // Build real_nodes to support has_node(). result.determine_real_nodes(); // Store the sequences. result.sequences = gbwt::StringArray(result.index->sigma() - result.index->firstNode(), [&](size_t offset) -> std::string_view { gbwt::node_type node = offset + result.index->firstNode(); nid_t id = gbwt::Node::id(node); if(!(result.has_node(id))) { return std::string_view(nullptr, 0); } return this->get_sequence_view(node_to_handle(node)); }); return result; } //------------------------------------------------------------------------------ void GBWTGraph::determine_real_nodes() { // Sometimes (e.g. in `deserialize()`) we call this function after the header already // contains the correct number of nodes. this->header.nodes = 0; if(this->index->empty()) { this->real_nodes = sdsl::bit_vector(); return; } size_t potential_nodes = this->index->sigma() - this->index->firstNode(); this->real_nodes = sdsl::bit_vector(potential_nodes / 2, 0); for(gbwt::node_type node = this->index->firstNode(); node < this->index->sigma(); node += 2) { if(!(this->index->empty(node))) { this->real_nodes[this->node_offset(node) / 2] = 1; this->header.nodes++; } } } void GBWTGraph::cache_named_paths() { this->named_paths.clear(); this->name_to_path.clear(); this->id_to_path.clear(); // There cannot be named paths without sufficient metadata. if(this->index == nullptr || !(this->index->hasMetadata()) || !(this->index->metadata.hasSampleNames()) || !(this->index->metadata.hasContigNames()) || !(this->index->metadata.hasPathNames())) { return; } // Determine the named paths. for(gbwt::size_type sample = 0; sample < this->index->metadata.sample_names.size(); sample++) { PathSense sense = gbwtgraph::get_sample_sense(this->index->metadata, sample, this->reference_samples); if(sense == PathSense::GENERIC || sense == PathSense::REFERENCE) { // This is a named path sample. std::vector sample_ids = this->index->metadata.pathsForSample(sample); for(size_t i = 0; i < sample_ids.size(); i++) { // For each path in this reference sample const gbwt::PathName& path = this->index->metadata.path(sample_ids[i]); // Compose a name for it that matches the one we will present later. std::string composed_path_name = gbwtgraph::compose_path_name(this->index->metadata, path, sense); // Store a mapping from name to index this will appear at in named_paths this->name_to_path[composed_path_name] = named_paths.size(); // And from internal path ID to index this will appear at in named_paths; this->id_to_path[sample_ids[i]] = named_paths.size(); // Create the NamedPath this->named_paths.emplace_back(); // And store the ID this->named_paths.back().id = sample_ids[i]; // And whether we're reference sense this->named_paths.back().sense = sense; } } } if(this->name_to_path.size() != this->named_paths.size()) { throw InvalidGBWT("GBWTGraph: Named path names are not unique"); } // Cache named path information we get from traversing the paths. #pragma omp parallel for schedule(dynamic, 1) for(size_t i = 0; i < this->named_paths.size(); i++) { NamedPath& path = this->named_paths[i]; gbwt::edge_type curr = this->index->start(gbwt::Path::encode(path.id, false)); path.from = (curr.first == gbwt::ENDMARKER ? gbwt::invalid_edge() : curr); path.to = gbwt::invalid_edge(); path.length = 0; while(curr.first != gbwt::ENDMARKER) { path.to = curr; path.length++; curr = this->index->LF(curr); } } } //------------------------------------------------------------------------------ bool GBWTGraph::has_node(nid_t node_id) const { size_t offset = this->node_offset(gbwt::Node::encode(node_id, false)) / 2; return (offset < this->real_nodes.size() && this->real_nodes[offset]); } handle_t GBWTGraph::get_handle(const nid_t& node_id, bool is_reverse) const { return node_to_handle(gbwt::Node::encode(node_id, is_reverse)); } nid_t GBWTGraph::get_id(const handle_t& handle) const { return gbwt::Node::id(handle_to_node(handle)); } bool GBWTGraph::get_is_reverse(const handle_t& handle) const { return gbwt::Node::is_reverse(handle_to_node(handle)); } handle_t GBWTGraph::flip(const handle_t& handle) const { return node_to_handle(gbwt::Node::reverse(handle_to_node(handle))); } size_t GBWTGraph::get_length(const handle_t& handle) const { size_t offset = this->node_offset(handle); return this->sequences.length(offset); } std::string GBWTGraph::get_sequence(const handle_t& handle) const { size_t offset = this->node_offset(handle); return this->sequences.str(offset); } char GBWTGraph::get_base(const handle_t& handle, size_t index) const { size_t offset = this->node_offset(handle); std::string_view view = this->sequences.view(offset); return view[index]; } std::string GBWTGraph::get_subsequence(const handle_t& handle, size_t index, size_t size) const { size_t offset = this->node_offset(handle); std::string_view view = this->sequences.view(offset); index = std::min(index, view.size()); size = std::min(size, view.size() - index); return std::string(view.data() + index, view.data() + index + size); } size_t GBWTGraph::get_node_count() const { return this->header.nodes; } nid_t GBWTGraph::min_node_id() const { if(this->get_node_count() == 0) { return std::numeric_limits::max(); } return gbwt::Node::id(this->index->firstNode()); } nid_t GBWTGraph::max_node_id() const { if(this->get_node_count() == 0) { return 0; } nid_t next_id = gbwt::Node::id(this->index->sigma()); return next_id - 1; } bool GBWTGraph::follow_edges_impl(const handle_t& handle, bool go_left, const std::function& iteratee) const { return this->cached_follow_edges(this->get_single_cache(), handle, go_left, iteratee); } bool GBWTGraph::for_each_handle_impl(const std::function& iteratee, bool parallel) const { if(parallel) { #pragma omp parallel for schedule(dynamic, CHUNK_SIZE) for(gbwt::node_type node = this->index->firstNode(); node < this->index->sigma(); node += 2) { if(!(this->real_nodes[this->node_offset(node) / 2])) { continue; } if(!iteratee(node_to_handle(node))) { // We should stop early but it's not worth the effort. } } } else { for(gbwt::node_type node = this->index->firstNode(); node < this->index->sigma(); node += 2) { if(!(this->real_nodes[this->node_offset(node) / 2])) { continue; } if(!iteratee(node_to_handle(node))) { return false; } } } return true; } size_t GBWTGraph::get_degree(const handle_t& handle, bool go_left) const { // Cache the node. gbwt::node_type curr = handle_to_node(handle); if(go_left) { curr = gbwt::Node::reverse(curr); } gbwt::CachedGBWT cache = this->get_single_cache(); gbwt::size_type cache_index = cache.findRecord(curr); // The outdegree reported by GBWT might account for the endmarker, which is // always the first successor. size_t result = cache.outdegree(cache_index); if(result > 0 && cache.successor(cache_index, 0) == gbwt::ENDMARKER) { result--; } return result; } bool GBWTGraph::has_edge(const handle_t& left, const handle_t& right) const { // Cache the node. gbwt::node_type curr = handle_to_node(left); gbwt::CachedGBWT cache = this->get_single_cache(); gbwt::size_type cache_index = cache.findRecord(curr); for(gbwt::rank_type outrank = 0; outrank < cache.outdegree(cache_index); outrank++) { gbwt::node_type next = cache.successor(cache_index, outrank); if(node_to_handle(next) == right) { return true; } } return false; } //------------------------------------------------------------------------------ size_t GBWTGraph::get_path_count() const { return this->index->metadata.paths(); } bool GBWTGraph::has_path(const std::string& path_name) const { // Polling for path names that look like haplotypes can be kind of hard, so // we get the handle and look for the sentinel. return handlegraph::as_integer(this->get_path_handle(path_name)) != std::numeric_limits::max(); } path_handle_t GBWTGraph::get_path_handle(const std::string& path_name) const { // We define this to return a sentinel path handle when no path is found path_handle_t to_return = handlegraph::as_path_handle(std::numeric_limits::max()); auto found = this->name_to_path.find(path_name); if(found != this->name_to_path.end()) { // This is the name of a stored named path. // Pack up its cache offset as a handle. to_return = handlegraph::as_path_handle(found->second); } else { // Parse the path name. PathSense sense; std::string sample_name; std::string contig_name; size_t haplotype; size_t phase_block; subrange_t subrange; PathMetadata::parse_path_name(path_name, sense, sample_name, contig_name, haplotype, phase_block, subrange); if(gbwtgraph::get_sample_sense(sample_name, this->reference_samples) != PathSense::HAPLOTYPE) { // This is on a sample that's supposed to have named paths. // We aren't allowed to expose named paths through this mechanism. return to_return; } if(subrange != NO_SUBRANGE) { // We don't store haplotype subranges. return to_return; } if(phase_block == NO_PHASE_BLOCK) { // We need a phase block. return to_return; } if(haplotype == NO_HAPLOTYPE) { // We use the GBWT sentinel haplotype haplotype = GBWTGraph::NO_PHASE; } auto sample_number = this->index->metadata.sample(sample_name); if(sample_number == this->index->metadata.sample_names.size()) { // This sample doesn't exist. return to_return; } auto contig_number = this->index->metadata.contig(contig_name); if(contig_number == this->index->metadata.contig_names.size()) { // This contig doesn't exist. return to_return; } // Now we have to look up this path in the metadata and get the path number. for(auto& path_id : this->index->metadata.findPaths(sample_number, contig_number)) { // Paths are only indexed by sample and contig, so we have to scan for the // right haplotype and phase block. auto& structured_name = this->index->metadata.path(path_id); if(structured_name.phase == haplotype && structured_name.count == phase_block) { // This is the right path. Turn it into a haplotype path handle. to_return = handlegraph::as_path_handle(this->named_paths.size() + path_id); break; } } } // Now return what we found, or the sentinel if we found nothing. return to_return; } std::string GBWTGraph::get_path_name(const path_handle_t& path_handle) const { auto sense = this->get_sense(path_handle); gbwt::size_type path_id = this->handle_to_path(path_handle); // Get the name fields from the metadata. auto& structured_name = this->index->metadata.path(path_id); return gbwtgraph::compose_path_name(this->index->metadata, structured_name, sense); } bool GBWTGraph::get_is_circular(const path_handle_t&) const { // TODO: We don't track circular paths return false; } size_t GBWTGraph::get_step_count(const path_handle_t& path_handle) const { switch(this->get_sense(path_handle)) { case PathSense::GENERIC: // Fall-through case PathSense::REFERENCE: // This information is cached return this->named_paths[handlegraph::as_integer(path_handle)].length;; break; case PathSense::HAPLOTYPE: // This information is not cached. { size_t count = 0; for(step_handle_t here = this->path_begin(path_handle); here != this->path_end(path_handle); here = this->get_next_step(here)) { // Trace the path and count all the steps. count++; } return count; } break; default: throw std::runtime_error("Unimplemented sense!"); } } size_t GBWTGraph::get_step_count(const handle_t& handle) const { // Use the brute force approach where we total up the number of step handles // we iterate over. size_t count = 0; for_each_step_on_handle_impl(handle, [&](const step_handle_t&) { count++; return true; }); return count; } handle_t GBWTGraph::get_handle_of_step(const step_handle_t& step_handle) const { // The GBWT node number is the first field of the step handle, so grab that // and turn it into a graph handle. return node_to_handle(handlegraph::as_integers(step_handle)[0]); } path_handle_t GBWTGraph::get_path_handle_of_step(const step_handle_t& step_handle) const { // To find the thread number we will need to locate the selected visit. So // turn it into an edge. gbwt::edge_type here; here.first = handlegraph::as_integers(step_handle)[0]; here.second = handlegraph::as_integers(step_handle)[1]; gbwt::size_type path_id = gbwt::Path::id(index->locate(here)); // Convert path id number to path handle. return this->path_to_handle(path_id); } step_handle_t GBWTGraph::path_begin(const path_handle_t& path_handle) const { // Step handles correspond to GBWT edges. gbwt::edge_type from; switch(this->get_sense(path_handle)) { case PathSense::GENERIC: // Fall-through case PathSense::REFERENCE: // This information is cached from = this->named_paths[handlegraph::as_integer(path_handle)].from; break; case PathSense::HAPLOTYPE: { size_t path_number = this->handle_to_path(path_handle); // We can use the gbwt's start() to get the start of a GBWT sequence. // And we're always interested in the forward orientation of the path from = this->index->start(gbwt::Path::encode(path_number, false)); if(from.first == gbwt::ENDMARKER) { // Path must be empty. Use the past-end sentinel. from = gbwt::invalid_edge(); } } break; default: throw std::runtime_error("Unimplemented sense!"); } step_handle_t step; // Edges and steps have GBWT node numbers first as_integers(step)[0] = from.first; // And then offsets into the node's visits as_integers(step)[1] = from.second; return step; } step_handle_t GBWTGraph::path_end(const path_handle_t&) const { // path_end can just be invalid_edge() since it's a past-end. gbwt::edge_type past_last_edge = gbwt::invalid_edge(); step_handle_t step; // Edges and steps have GBWT node numbers first as_integers(step)[0] = past_last_edge.first; // And then offsets into the node's visits as_integers(step)[1] = past_last_edge.second; return step; } step_handle_t GBWTGraph::path_back(const path_handle_t& path_handle) const { gbwt::edge_type to; switch(this->get_sense(path_handle)) { case PathSense::GENERIC: // Fall-through case PathSense::REFERENCE: // This information is cached to = this->named_paths[handlegraph::as_integer(path_handle)].to; break; case PathSense::HAPLOTYPE: { // This information isn't cached. size_t path_number = this->handle_to_path(path_handle); // We need to get the final step along the path. // // inverseLF() deosnt' work *from* the end marker, because the end marker // edge values aren't actually necessarily unique. // // So we need to find the first step on the reverse version of the path, and // scan its node for the last step on the forward version of the path. gbwt::edge_type first_reverse_edge = this->index->start(gbwt::Path::encode(path_number, true)); if(first_reverse_edge.first == gbwt::ENDMARKER) { // Path is empty. return path_front_end(path_handle); } handlegraph::handle_t last_forward_handle = flip(node_to_handle(first_reverse_edge.first)); // Look up the GBWT node we're using, in the path-forward orientaton gbwt::SearchState node_state = get_state(last_forward_handle); // Fill in the node part of the to edge, which doesn't change to.first = node_state.node; // Get an edge for each single haplotype selected by the search state. // Remember that ranges are inclusive at both ends. for(to.second = node_state.range.first; to.second <= node_state.range.second; ++to.second) { gbwt::edge_type next_edge = this->index->LF(to); if(next_edge.first != gbwt::ENDMARKER) { // We're only interested in final visits, which we can ID before // locate() continue; } // This candidate is the last visit along some path, so it might be the // last visit along the path we're looking for. auto sequence_number = this->index->locate(to); if(gbwt::Path::is_reverse(sequence_number)) { // We're only interested in forward versions of paths continue; } auto path_number_here = gbwt::Path::id(sequence_number); if(path_number_here != path_number) { // We're only interested in the one path. continue; } // This is the visit! Stop scanning! break; } } break; default: throw std::runtime_error("Unimplemented sense!"); } step_handle_t step; // Edges and steps have GBWT node numbers first as_integers(step)[0] = to.first; // And then offsets into the node's visits as_integers(step)[1] = to.second; return step; } step_handle_t GBWTGraph::path_front_end(const path_handle_t&) const { // path_front_end can just be invalid_edge() since it's a past-end. gbwt::edge_type before_first_edge = gbwt::invalid_edge(); step_handle_t step; // Edges and steps have GBWT node numbers first as_integers(step)[0] = before_first_edge.first; // And then offsets into the node's visits as_integers(step)[1] = before_first_edge.second; return step; } bool GBWTGraph::has_next_step(const step_handle_t& step_handle) const { // Just look ahead and see if we get a past-end step_handle_t would_be_next = this->get_next_step(step_handle); gbwt::edge_type past_last_edge = gbwt::invalid_edge(); return as_integers(would_be_next)[0] != past_last_edge.first || as_integers(would_be_next)[1] != past_last_edge.second; } bool GBWTGraph::has_previous_step(const step_handle_t& step_handle) const { // Just look back and see if we get a sentinel step_handle_t would_be_prev = this->get_previous_step(step_handle); gbwt::edge_type before_first_edge = gbwt::invalid_edge(); return as_integers(would_be_prev)[0] != before_first_edge.first || as_integers(would_be_prev)[1] != before_first_edge.second; } step_handle_t GBWTGraph::get_next_step(const step_handle_t& step_handle) const { // Convert into a GBWT edge gbwt::edge_type here; here.first = as_integers(step_handle)[0]; here.second = as_integers(step_handle)[1]; // Follow it, and get either a real edge, or an edge where the node is // gbwt::ENDMARKER. here = this->index->LF(here); if(here.first == gbwt::ENDMARKER) { // We've hit the end marker. Use an invalid_edge() sentinel instead. here = gbwt::invalid_edge(); } // Convert back step_handle_t next; as_integers(next)[0] = here.first; as_integers(next)[1] = here.second; return next; } step_handle_t GBWTGraph::get_previous_step(const step_handle_t& step_handle) const { // Convert into a GBWT edge gbwt::edge_type here; here.first = as_integers(step_handle)[0]; here.second = as_integers(step_handle)[1]; // Follow it, backward and get either a real edge, or an edge where the // node is gbwt::ENDMARKER. here = this->index->inverseLF(here); if(here.first == gbwt::ENDMARKER) { // We've hit the end marker. Use an invalid_edge() sentinel instead. here = gbwt::invalid_edge(); } // Convert back step_handle_t prev; as_integers(prev)[0] = here.first; as_integers(prev)[1] = here.second; return prev; } bool GBWTGraph::for_each_path_handle_impl(const std::function& iteratee) const { return for_each_path_matching_impl(nullptr, nullptr, nullptr, iteratee); } bool GBWTGraph::for_each_step_on_handle_impl(const handle_t& handle, const std::function& iteratee) const { // Nothing to do without paths. if(this->get_path_count() == 0) { return true; } return this->for_each_edge_and_path_on_handle(handle, [&](const gbwt::edge_type& candidate_edge, const gbwt::size_type&) { // Prepare a step. step_handle_t step; handlegraph::as_integers(step)[0] = candidate_edge.first; handlegraph::as_integers(step)[1] = candidate_edge.second; // And show it to the iteratee return iteratee(step); }); } //------------------------------------------------------------------------------ PathSense GBWTGraph::get_sense(const path_handle_t& handle) const { if(handlegraph::as_integer(handle) < this->named_paths.size()) { // This is a cached named path. // To avoid pulling out the whole sample name every time we want to check // path sense, we cache the sense in the NamedPath. auto& named_path = this->named_paths[handlegraph::as_integer(handle)]; return named_path.sense; } // Otherwise it's a haolotype return PathSense::HAPLOTYPE; } std::string GBWTGraph::get_sample_name(const path_handle_t& handle) const { PathSense sense = this->get_sense(handle); auto& structured_name = this->index->metadata.path(this->handle_to_path(handle)); return gbwtgraph::get_path_sample_name(this->index->metadata, structured_name, sense); } std::string GBWTGraph::get_locus_name(const path_handle_t& handle) const { PathSense sense = this->get_sense(handle); auto& structured_name = this->index->metadata.path(this->handle_to_path(handle)); return gbwtgraph::get_path_locus_name(this->index->metadata, structured_name, sense); } size_t GBWTGraph::get_haplotype(const path_handle_t& handle) const { PathSense sense = this->get_sense(handle); auto& structured_name = this->index->metadata.path(this->handle_to_path(handle)); return gbwtgraph::get_path_haplotype(this->index->metadata, structured_name, sense); } size_t GBWTGraph::get_phase_block(const path_handle_t& handle) const { PathSense sense = this->get_sense(handle); auto& structured_name = this->index->metadata.path(this->handle_to_path(handle)); return gbwtgraph::get_path_phase_block(this->index->metadata, structured_name, sense); } subrange_t GBWTGraph::get_subrange(const path_handle_t& handle) const { PathSense sense = this->get_sense(handle); auto& structured_name = this->index->metadata.path(this->handle_to_path(handle)); return gbwtgraph::get_path_subrange(this->index->metadata, structured_name, sense); } gbwt::size_type GBWTGraph::sample_number_for_sample_name(const std::unordered_set* senses, const std::string& sample_name) const { if(sample_name == NO_SAMPLE_NAME) { // Don't try and look up the sentinel if(!senses || senses->count(PathSense::GENERIC)) { // But we might have the non-sample sample gbwt::size_type sample_number = this->index->metadata.sample(GENERIC_PATH_SAMPLE_NAME); if(sample_number < this->index->metadata.samples()) { return sample_number; } } return this->index->metadata.samples(); // Invalid sample number } // Otherwise we aren't working with NO_SAMPLE_NAME. if(!senses || senses->count(PathSense::HAPLOTYPE) || senses->count(PathSense::REFERENCE)) { gbwt::size_type sample_number = this->index->metadata.sample(sample_name); if(sample_number < this->index->metadata.samples()) { return sample_number; } } return this->index->metadata.samples(); // Invalid sample number } bool GBWTGraph::for_each_path_matching_impl(const std::unordered_set* senses, const std::unordered_set* samples, const std::unordered_set* loci, const std::function& iteratee) const { // First try to divert queries we have efficient implementations for. // TODO: Can we look up all the sample numbers and all the locus numbers once // first? That would be faster. if((senses && senses->empty()) || (samples && samples->empty()) || (loci && loci->empty())) { // Nothing to do! return true; } if(samples && samples->size() == 1 && loci && loci->size() == 1) { // We can look up one sample and locus, and we don't even need to filter ourselves. return this->for_each_path_matching_sample_and_locus(senses, *samples->begin(), *loci->begin(), iteratee); } if (senses && senses->size() == 1 && senses->count(PathSense::GENERIC)) { // We only want to look at generic paths, so we know the sample can only be one thing. if(!samples || samples->count(NO_SAMPLE_NAME)) { // We can use the only allowed sample. if(loci) { // And we have loci to restrict to. for(auto& locus_name : *loci) { if(!this->for_each_path_matching_sample_and_locus(senses, NO_SAMPLE_NAME, locus_name, iteratee)) { return false; } } return true; } else { // Any locus is acceptable in this sample. return this->for_each_path_matching_sample(senses, NO_SAMPLE_NAME, iteratee); } } else { // We banned the only allowed sample. return true; } } if(samples) { // We can look at just these particular samples. for(auto& sample_name : *samples) { if(loci && loci->size() == 1) { // We can look up the one locus for every sample if(!this->for_each_path_matching_sample_and_locus(senses, sample_name, *loci->begin(), iteratee)) { return false; } } else { // We need to scan and filter on locus for each sample bool keep_going = this->for_each_path_matching_sample(senses, sample_name, [&](const path_handle_t& path_handle) { if(loci && !loci->count(this->get_locus_name(path_handle))) { return true; } return iteratee(path_handle); }); if(!keep_going) { return false; } } } return true; } if(loci) { // We have no sample names but we have locus names for(auto& locus_name : *loci) { // We don't actually need to filter, because samples is unset, so all samples are allowed. if(!this->for_each_path_matching_locus(senses, locus_name, iteratee)) { return false; } } return true; } // If we get here we have no sample or locus names. if(senses && !senses->count(PathSense::HAPLOTYPE)) { // We just have to go through all the named paths for(size_t i = 0; i < this->named_paths.size(); i++) { auto& named_path = this->named_paths[i]; if(senses->count(named_path.sense)) { // This path is a sense we want if(!iteratee(handlegraph::as_path_handle(i))) { return false; } } } return true; } // We want haplotype paths at least, and we have no samples or loci to select // on, so we might as well go through everything. for(size_t i = 0; i < this->index->metadata.paths(); i++) { path_handle_t path_handle = this->path_to_handle(i); if(senses && !senses->count(this->get_sense(path_handle))) { // This sense is unwanted. continue; } if(!iteratee(path_handle)) { return false; } } return true; } bool GBWTGraph::for_each_path_matching_sample_and_locus(const std::unordered_set* senses, const std::string& sample_name, const std::string& locus_name, const std::function& iteratee) const { gbwt::size_type sample_number = this->sample_number_for_sample_name(senses, sample_name); gbwt::size_type contig_number = this->index->metadata.contig(locus_name); if(sample_number == this->index->metadata.samples() || contig_number == this->index->metadata.contigs()) { // Looking for a nonexistent path. return true; } for(auto& path_id : this->index->metadata.findPaths(sample_number, contig_number)) { // For each path in that sample-sense and locus, try it. if(!iteratee(this->path_to_handle(path_id))) { return false; } } return true; } bool GBWTGraph::for_each_path_matching_sample(const std::unordered_set* senses, const std::string& sample_name, const std::function& iteratee) const { gbwt::size_type sample_number = this->sample_number_for_sample_name(senses, sample_name); if(sample_number == this->index->metadata.samples()) { // Looking for a nonexistent sample return true; } for(auto& path_id : this->index->metadata.pathsForSample(sample_number)) { // For each path in that sample-sense, try it if(!iteratee(this->path_to_handle(path_id))) { return false; } } return true; } bool GBWTGraph::for_each_path_matching_locus(const std::unordered_set* senses, const std::string& locus_name, const std::function& iteratee) const { auto contig_number = this->index->metadata.contig(locus_name); if(contig_number == this->index->metadata.contig_names.size()) { // Looking for a nonexistent locus return true; } for(auto& path_id : this->index->metadata.pathsForContig(contig_number)) { // For each path in that locus, get the handle path_handle_t path_handle = this->path_to_handle(path_id); if(!senses || senses->count(this->get_sense(path_handle))) { // This is a sense we want. // We have to check because locus number doesn't determine sense like // sample number does. if(!iteratee(path_handle)) { return false; } } } return true; } bool GBWTGraph::for_each_step_of_sense_impl(const handle_t& visited, const PathSense& sense, const std::function& iteratee) const { return this->for_each_edge_and_path_on_handle(visited, [&](const gbwt::edge_type& candidate_edge, const gbwt::size_type& path_number) { auto found = this->id_to_path.find(path_number); if(found == this->id_to_path.end() && sense != PathSense::HAPLOTYPE) { // We found a haplotype we don't want return true; } else if(found != this->id_to_path.end() && this->named_paths[found->second].sense != sense) { // We are looking for reference paths but this is a generic path, or visa versa. return true; } // Make the step handle. step_handle_t step; handlegraph::as_integers(step)[0] = candidate_edge.first; handlegraph::as_integers(step)[1] = candidate_edge.second; // And show it to the iteratee return iteratee(step); }); } bool GBWTGraph::for_each_edge_and_path_on_handle(const handle_t& handle, const std::function& iteratee) const { for(const handle_t oriented_handle : {handle, flip(handle)}) { // We need to look at both orientations, because we only want steps on the // forward versions of their paths. And there's no good way to go from an // edge on a path's reverse sequence to an edge on the path's forward one. // Look up the GBWT node gbwt::SearchState node_state = get_state(oriented_handle); gbwt::edge_type candidate_edge; candidate_edge.first = node_state.node; for(candidate_edge.second = node_state.range.first; candidate_edge.second <= node_state.range.second; ++candidate_edge.second) { // Get the edge for each haplotype in the start-and-end-inclusive range // Get the sequence number the edge is on. // TODO: We could have a version of locate() that decompresses the entire DA for the node. auto sequence_number = this->index->locate(candidate_edge); if(gbwt::Path::is_reverse(sequence_number)) { // We're looking at the reverse version of the path, which doesn't // have a corresponding libhandlegraph step, because step handles don't // help you at all with path orientation. Skip this one and come up // with the forward version of the thread when we look at the other // orientation of the handle. continue; } auto path_number = gbwt::Path::id(sequence_number); bool should_continue = iteratee(candidate_edge, path_number); if(!should_continue) { // We are supposed to stop now. return false; } } } // We made it to the end. return true; } gbwt::size_type GBWTGraph::handle_to_path(const path_handle_t& handle) const { gbwt::size_type scratch = handlegraph::as_integer(handle); if(scratch < this->named_paths.size()) { // Look up the metadata object path number for this cache entry return this->named_paths[scratch].id; } else { // There's no cache entry; remove the offset and get the metadata object path number. return scratch - this->named_paths.size(); } } path_handle_t GBWTGraph::path_to_handle(gbwt::size_type path) const { // This might be a named path or a haplotype path. auto found = this->id_to_path.find(path); if (found == this->id_to_path.end()) { // This isn't referenced by a stored NamedPath. Must be a haplotype path. return handlegraph::as_path_handle(this->named_paths.size() + path); } // Otherwise just use the number of the stored NamedPath return handlegraph::as_path_handle(found->second); } //------------------------------------------------------------------------------ bool GBWTGraph::has_segment_names() const { return this->header.get(Header::FLAG_TRANSLATION); } std::pair> GBWTGraph::get_segment(const handle_t& handle) const { // If there is no translation, the predecessor is always at the end. nid_t id = this->get_id(handle); auto iter = this->node_to_segment.predecessor(id); if(!(this->has_node(id)) || iter == this->node_to_segment.one_end()) { return std::pair>(std::to_string(id), std::make_pair(id, id + 1)); } nid_t start = iter->second; std::string name = this->segments.str(iter->first); ++iter; nid_t limit = iter->second; return std::make_pair(name, std::make_pair(start, limit)); } std::pair GBWTGraph::get_segment_name_and_offset(const handle_t& handle) const { // If there is no translation, the predecessor is always at the end. nid_t id = this->get_id(handle); auto iter = this->node_to_segment.predecessor(id); if(!(this->has_node(id)) || iter == this->node_to_segment.one_end()) { return std::pair(std::to_string(id), 0); } // Determine the total length of nodes in this segment that precede `id` // in the given orientation. size_t start = 0, limit = 0; if(this->get_is_reverse(handle)) { auto successor = iter; ++successor; start = this->node_offset(gbwt::Node::encode(id + 1, false)); limit = this->node_offset(gbwt::Node::encode(successor->second, false)); } else { start = this->node_offset(gbwt::Node::encode(iter->second, false)); limit = this->node_offset(gbwt::Node::encode(id, false)); } size_t offset = this->sequences.length(start, limit) / 2; return std::pair(this->segments.str(iter->first), offset); } std::string GBWTGraph::get_segment_name(const handle_t& handle) const { // If there is no translation, the predecessor is always at the end. nid_t id = this->get_id(handle); auto iter = this->node_to_segment.predecessor(id); if(iter == this->node_to_segment.one_end()) { return std::to_string(id); } return this->segments.str(iter->first); } size_t GBWTGraph::get_segment_offset(const handle_t& handle) const { // If there is no translation, the predecessor is always at the end. nid_t id = this->get_id(handle); auto iter = this->node_to_segment.predecessor(id); if(!(this->has_node(id)) || iter == this->node_to_segment.one_end()) { return 0; } // Determine the total length of nodes in this segment that precede `id` // in the given orientation. size_t start = 0, limit = 0; if(this->get_is_reverse(handle)) { auto successor = iter; ++successor; start = this->node_offset(gbwt::Node::encode(id + 1, false)); limit = this->node_offset(gbwt::Node::encode(successor->second, false)); } else { start = this->node_offset(gbwt::Node::encode(iter->second, false)); limit = this->node_offset(gbwt::Node::encode(id, false)); } size_t offset = this->sequences.length(start, limit) / 2; return offset; } bool GBWTGraph::for_each_segment_impl(const std::function&)>& iteratee, bool parallel) const { if(!(this->has_segment_names())) { return true; } bool keep_going = true; if(parallel) { // The parallel version iterates over all handles, converts them to segments, and // calls the iteratee if the handle is the first node of the segment. keep_going = this->for_each_handle([&](const handle_t& handle) -> bool { std::pair> segment = this->get_segment(handle); if(this->get_id(handle) == segment.second.first) { return iteratee(segment.first, segment.second); } return true; }, parallel); } else { auto iter = this->node_to_segment.one_begin(); while(iter != this->node_to_segment.one_end()) { nid_t start = iter->second; std::string name = this->segments.str(iter->first); ++iter; nid_t limit = iter->second; // The translation may include segments that were not used on any path. // The corresponding nodes are missing from the graph. if(this->has_node(start)) { if(!iteratee(name, std::make_pair(start, limit))) { keep_going = false; break; } } } } return keep_going; } bool GBWTGraph::for_each_link_impl(const std::function& iteratee, bool parallel) const { if(!(this->has_segment_names())) { return true; } return this->for_each_segment([&](const std::string& from_segment, std::pair nodes) -> bool { bool keep_going = true; // Right edges from forward orientation are canonical if the destination node // has a greater id or if the edge is a self-loop. handle_t last = this->get_handle(nodes.second - 1, false); keep_going = this->follow_edges(last, false, [&](const handle_t& next) -> bool { nid_t next_id = this->get_id(next); if(next_id >= nodes.second - 1) { std::string to_segment = this->get_segment_name(next); if(!iteratee(edge_t(last, next), from_segment, to_segment)) { return false; } } return true; }); if(!keep_going) { return false; } // Right edges from reverse orientation are canonical if the destination node // has a greater id or if the edge is a self-loop to forward orientation of // this node. handle_t first = this->get_handle(nodes.first, true); keep_going = this->follow_edges(first, false, [&](const handle_t& next) -> bool { nid_t next_id = this->get_id(next); if(next_id > nodes.first || (next_id == nodes.first && !(this->get_is_reverse(next)))) { std::string to_segment = this->get_segment_name(next); if(!iteratee(edge_t(first, next), from_segment, to_segment)) { return false; } } return true; }); return keep_going; }, parallel); } //------------------------------------------------------------------------------ /// Translate a node range back to segment space std::vector GBWTGraph::translate_back(const oriented_node_range_t& range) const { // If there is no translation, the predecessor is always at the end. nid_t id = std::get<0>(range); auto iter = this->node_to_segment.predecessor(id); if(!(this->has_node(id)) || iter == this->node_to_segment.one_end()) { // No segments or nonexistent node. return {}; } // Determine the total length of nodes in this segment that precede `id` // in the given orientation. size_t start = 0, limit = 0; if(std::get<1>(range)) { auto successor = iter; ++successor; start = this->node_offset(gbwt::Node::encode(id + 1, false)); limit = this->node_offset(gbwt::Node::encode(successor->second, false)); } else { start = this->node_offset(gbwt::Node::encode(iter->second, false)); limit = this->node_offset(gbwt::Node::encode(id, false)); } size_t offset = this->sequences.length(start, limit) / 2; return {oriented_node_range_t(iter->first, std::get<1>(range), offset + std::get<2>(range), std::get<3>(range))}; } /// Get a segment name std::string GBWTGraph::get_back_graph_node_name(const nid_t& back_node_id) const { return this->segments.str(back_node_id); } //------------------------------------------------------------------------------ uint32_t GBWTGraph::get_magic_number() const { // Specify what it should look like on the wire const char* bytes = "GBG "; // Convert to a host byte order number return ntohl(*((const uint32_t*) bytes)); } void GBWTGraph::serialize_members(std::ostream& out) const { out.write(reinterpret_cast(&(this->header)), sizeof(Header)); this->sequences.serialize(out); this->real_nodes.serialize(out); if(this->header.get(Header::FLAG_TRANSLATION)) { this->segments.serialize(out); this->node_to_segment.serialize(out); } } void GBWTGraph::deserialize_members(std::istream& in) { // Read the header. Header h = sdsl::simple_sds::load_value
(in); h.check(); bool use_zstd = (h.version >= Header::ZSTD_VERSION); bool simple_sds = h.get(Header::FLAG_SIMPLE_SDS); h.unset(Header::FLAG_SIMPLE_SDS); // We only set this flag in the serialized header. h.set_version(); // Update to the current version. this->header = h; // Load the graph. if(simple_sds) { // Determine real nodes in the background. if(this->index == nullptr) { throw InvalidGBWT("GBWTGraph: A GBWT index is required for loading Simple-SDS format"); } auto call_determine_real_nodes = [this](void) { this->determine_real_nodes(); }; std::thread determine_real_nodes_thread(call_determine_real_nodes); // Load the sequences using the new fancy method. if(use_zstd) { this->sequences.simple_sds_decompress_duplicate(in, reverse_complement); } else { this->sequences.simple_sds_load_duplicate(in, reverse_complement); } determine_real_nodes_thread.join(); } else { this->sequences.load(in); this->real_nodes.load(in); } // Load the translation. if(simple_sds) { // The translation may be empty but not absent. this->segments.simple_sds_load(in); this->node_to_segment.simple_sds_load(in); if(this->header.get(Header::FLAG_TRANSLATION) != (this->segments.size() > 0)) { throw sdsl::simple_sds::InvalidData("GBWTGraph: Invalid translation flag in the header"); } } else if(this->header.get(Header::FLAG_TRANSLATION)) { this->segments.load(in); this->node_to_segment.load(in); } this->sanity_checks(); } void GBWTGraph::set_gbwt(const gbwt::GBWT& gbwt_index) { this->index = &gbwt_index; if(!(this->index->bidirectional())) { throw InvalidGBWT("GBWTGraph: The GBWT index must be bidirectional"); } this->reference_samples = parse_reference_samples_tag(*(this->index)); this->cache_named_paths(); } void GBWTGraph::set_gbwt_address(const gbwt::GBWT& gbwt_index) { this->index = &gbwt_index; } //------------------------------------------------------------------------------ void GBWTGraph::simple_sds_serialize_version(std::ostream& out, std::uint32_t version) const { if(version < Header::MIN_SERIALIZE_VERSION || version > Header::VERSION) { throw sdsl::simple_sds::UnsupportedVersion("GBWTGraph", version, Header::MIN_SERIALIZE_VERSION, Header::VERSION); } // Serialize the header. Header copy = this->header; copy.version = version; copy.set(Header::FLAG_SIMPLE_SDS); // We only set this flag in the serialized header. sdsl::simple_sds::serialize_value(copy, out); // Compress the sequences. `real_nodes` can be rebuilt from the GBWT. if(version >= Header::ZSTD_VERSION) { this->sequences.simple_sds_compress_even(out); } else { this->sequences.simple_sds_serialize_even(out); } // Compress the translation. this->segments.simple_sds_serialize(out); this->node_to_segment.simple_sds_serialize(out); } void GBWTGraph::simple_sds_load(std::istream& in, const gbwt::GBWT& gbwt_index) { // Set the GBWT so we can rebuild `real_nodes` later. this->set_gbwt(gbwt_index); // The same deserialize() function can handle the SDSL and Simple-SDS formats. this->deserialize_members(in); } size_t GBWTGraph::simple_sds_size() const { size_t result = sdsl::simple_sds::value_size(this->header); // Compress the sequences. { std::ostringstream out; this->sequences.simple_sds_compress_even(out); result += sdsl::simple_sds::data_size(out.tellp()); } // Compress the translation. result += this->segments.simple_sds_size(); result += this->node_to_segment.simple_sds_size(); return result; } //------------------------------------------------------------------------------ std::string_view GBWTGraph::get_sequence_view(const handle_t& handle) const { size_t offset = this->node_offset(handle); return this->sequences.view(offset); } bool GBWTGraph::starts_with(const handle_t& handle, char c) const { size_t offset = this->node_offset(handle); std::string_view view = this->sequences.view(offset); return (view.size() > 0 && view.front() == c); } bool GBWTGraph::ends_with(const handle_t& handle, char c) const { size_t offset = this->node_offset(handle); std::string_view view = this->sequences.view(offset); return (view.size() > 0 && view.back() == c); } gbwt::SearchState GBWTGraph::find(const std::vector& path) const { if(path.empty()) { return gbwt::SearchState(); } gbwt::SearchState result = this->get_state(path[0]); for(size_t i = 1; i < path.size() && !result.empty(); i++) { result = this->index->extend(result, handle_to_node(path[i])); } return result; } gbwt::BidirectionalState GBWTGraph::bd_find(const std::vector& path) const { if(path.empty()) { return gbwt::BidirectionalState(); } gbwt::BidirectionalState result = this->get_bd_state(path[0]); for(size_t i = 1; i < path.size() && !result.empty(); i++) { result = this->index->bdExtendForward(result, handle_to_node(path[i])); } return result; } //------------------------------------------------------------------------------ gbwt::SearchState GBWTGraph::find(const gbwt::CachedGBWT& cache, const std::vector& path) const { if(path.empty()) { return gbwt::SearchState(); } gbwt::SearchState result = this->get_state(cache, path[0]); for(size_t i = 1; i < path.size() && !result.empty(); i++) { result = cache.extend(result, handle_to_node(path[i])); } return result; } gbwt::BidirectionalState GBWTGraph::bd_find(const gbwt::CachedGBWT& cache, const std::vector& path) const { if(path.empty()) { return gbwt::BidirectionalState(); } gbwt::BidirectionalState result = this->get_bd_state(cache, path[0]); for(size_t i = 1; i < path.size() && !result.empty(); i++) { result = cache.bdExtendForward(result, handle_to_node(path[i])); } return result; } bool GBWTGraph::follow_paths(const gbwt::CachedGBWT& cache, gbwt::SearchState state, const std::function& iteratee) const { gbwt::size_type cache_index = cache.findRecord(state.node); for(gbwt::rank_type outrank = 0; outrank < cache.outdegree(cache_index); outrank++) { if(cache.successor(cache_index, outrank) == gbwt::ENDMARKER) { continue; } gbwt::SearchState next_state = cache.cachedExtend(state, cache_index, outrank); if(next_state.empty()) { continue; } if(!iteratee(next_state)) { return false; } } return true; } bool GBWTGraph::follow_paths(const gbwt::CachedGBWT& cache, gbwt::BidirectionalState state, bool backward, const std::function& iteratee) const { gbwt::size_type cache_index = cache.findRecord(backward ? state.backward.node : state.forward.node); for(gbwt::rank_type outrank = 0; outrank < cache.outdegree(cache_index); outrank++) { if(cache.successor(cache_index, outrank) == gbwt::ENDMARKER) { continue; } gbwt::BidirectionalState next_state = (backward ? cache.cachedExtendBackward(state, cache_index, outrank) : cache.cachedExtendForward(state, cache_index, outrank)); if(next_state.empty()) { continue; } if(!iteratee(next_state)) { return false; } } return true; } bool GBWTGraph::cached_follow_edges(const gbwt::CachedGBWT& cache, const handle_t& handle, bool go_left, const std::function& iteratee) const { // Incoming edges correspond to the outgoing edges of the reverse node. gbwt::node_type curr = handle_to_node(handle); if(go_left) { curr = gbwt::Node::reverse(curr); } // Cache the node. gbwt::size_type cache_index = cache.findRecord(curr); for(gbwt::rank_type outrank = 0; outrank < cache.outdegree(cache_index); outrank++) { gbwt::node_type next = cache.successor(cache_index, outrank); if(next == gbwt::ENDMARKER) { continue; } // If we started from the reverse node, we must reverse the successor nodes to get // the predecessor nodes of the original node. if(go_left) { next = gbwt::Node::reverse(next); } if(!iteratee(node_to_handle(next))) { return false; } } return true; } //------------------------------------------------------------------------------ /* Haplotype-consistent traversal of the graph and the corresponding sequence. The traversal may start and end in the middle of a node. */ struct GBWTTraversal { std::vector traversal; size_t length; size_t offset; // In the initial node. gbwt::SearchState state; // GBWT search state at the end of the traversal. std::string get_sequence(const HandleGraph& graph) const { std::string result; result.reserve(this->length); for(size_t i = 0; i < this->traversal.size(); i++) { handle_t handle = this->traversal[i]; std::string str = graph.get_sequence(handle); if(i == 0) { result.append(str, this->offset, this->length - result.length()); } else { result.append(str, 0, this->length - result.length()); } } return result; } std::string get_sequence(const GBWTGraph& graph) const { std::string result; result.reserve(this->length); for(size_t i = 0; i < this->traversal.size(); i++) { handle_t handle = this->traversal[i]; std::string_view view = graph.get_sequence_view(handle); if(i == 0) { result.append(view, this->offset, this->length - result.length()); } else { result.append(view, 0, this->length - result.length()); } } return result; } }; void for_each_haplotype_window_impl( const GBWTGraph& graph, size_t window_size, const std::function&, const std::string&)>& lambda, bool parallel, bool nonredundant) { if(window_size == 0) { return; } // Traverse all starting nodes in parallel. graph.for_each_handle([&](const handle_t& h) -> bool { // Get a GBWT cache. gbwt::CachedGBWT cache = graph.get_cache(); // Initialize the stack with both orientations. std::stack windows; size_t node_length = graph.get_length(h); bool long_node = nonredundant & (node_length >= window_size); for(bool is_reverse : { false, true }) { handle_t handle = (is_reverse ? graph.flip(h) : h); gbwt::SearchState state = graph.get_state(cache, handle); if(state.empty()) { continue; } GBWTTraversal window { { handle }, node_length, 0, state }; if(long_node) { lambda(window.traversal, window.get_sequence(graph)); window.length = window_size - 1; window.offset = node_length - (window_size - 1); } windows.push(window); } // Extend the windows. size_t target_length = (long_node ? 2 * (window_size - 1) : node_length + window_size - 1); if(target_length == 0) { return true; } while(!windows.empty()) { GBWTTraversal window = windows.top(); windows.pop(); // Report the full window. if(window.length >= target_length) { lambda(window.traversal, window.get_sequence(graph)); continue; } // Try to extend the window to all successor nodes. bool extend_success = false; graph.follow_paths(cache, window.state, [&](const gbwt::SearchState& next_state) -> bool { handle_t next_handle = GBWTGraph::node_to_handle(next_state.node); GBWTTraversal next_window = window; next_window.traversal.push_back(next_handle); next_window.length += std::min(graph.get_length(next_handle), target_length - window.length); next_window.state = next_state; windows.push(next_window); extend_success = true; return true; }); // Report sufficiently long kmers that cannot be extended. if(!extend_success && window.length >= window_size) { lambda(window.traversal, window.get_sequence(graph)); } } return true; }, parallel); } void for_each_haplotype_window(const GBWTGraph& graph, size_t window_size, const std::function&, const std::string&)>& lambda, bool parallel) { for_each_haplotype_window_impl(graph, window_size, lambda, parallel, false); } void for_each_nonredundant_window( const GBWTGraph& graph, size_t window_size, const std::function&, const std::string&)>& lambda, bool parallel) { for_each_haplotype_window_impl(graph, window_size, lambda, parallel, true); } //------------------------------------------------------------------------------ } // namespace gbwtgraph jltsiren-gbwtgraph-e27bc43/src/gbz.cpp000066400000000000000000000263601524136466400200410ustar00rootroot00000000000000#include #include namespace gbwtgraph { //------------------------------------------------------------------------------ // Numerical class constants. constexpr std::uint32_t GBZ::Header::TAG; constexpr std::uint32_t GBZ::Header::VERSION; constexpr std::uint64_t GBZ::Header::FLAG_MASK; constexpr std::uint32_t GBZ::Header::ZSTD_BWT_VERSION; constexpr std::uint32_t GBZ::Header::ZSTD_SEQUENCES_VERSION; constexpr std::uint64_t GBZ::Header::ZSTD_SEQUENCES_FLAG_MASK; constexpr std::uint32_t GBZ::Header::OLD_VERSION; constexpr std::uint64_t GBZ::Header::OLD_FLAG_MASK; constexpr std::uint32_t GBZ::Header::MIN_VERSION; constexpr std::uint32_t GBZ::Header::MIN_SERIALIZE_VERSION; constexpr std::uint32_t GBZ::Header::DEFAULT_VERSION; //------------------------------------------------------------------------------ // Other class variables. const std::string GBZ::EXTENSION = ".gbz"; //------------------------------------------------------------------------------ GBZ::Header::Header() : tag(TAG), version(VERSION), flags(0) { } void GBZ::Header::check() const { if(this->tag != TAG) { throw sdsl::simple_sds::InvalidData("GBZ: Invalid tag"); } if(this->version > VERSION || this->version < OLD_VERSION) { std::string msg = "GBZ: Expected version " + std::to_string(OLD_VERSION) + " to " + std::to_string(VERSION) + ", got version " + std::to_string(this->version); throw sdsl::simple_sds::InvalidData(msg); } std::uint64_t mask = 0; switch(this->version) { case VERSION: mask = FLAG_MASK; break; case OLD_VERSION: mask = OLD_FLAG_MASK; break; } if((this->flags & mask) != this->flags) { throw sdsl::simple_sds::InvalidData("GBZ: Invalid flags"); } } bool GBZ::Header::operator==(const Header& another) const { return (this->tag == another.tag && this->version == another.version && this->flags == another.flags); } //------------------------------------------------------------------------------ size_t GBZ::set_reference_samples(const sample_name_set& samples) { sample_name_set present_samples = present_sample_names(samples, this->index); std::string tag_value = compose_reference_samples_tag(present_samples); this->index.tags.set(REFERENCE_SAMPLE_LIST_GBWT_TAG, tag_value); // By resetting the pointer to the GBWT index, we tell the GBWTGraph object // to pull the tag and recache named paths. this->set_gbwt(); return present_samples.size(); } //------------------------------------------------------------------------------ GBZ::GBZ() { this->add_source(); this->set_gbwt(); this->compute_pggname(nullptr); } GBZ::GBZ(const GBZ& source) { this->copy(source); } GBZ::GBZ(GBZ&& source) noexcept { *this = std::move(source); } GBZ::~GBZ() { } void GBZ::swap(GBZ& another) noexcept { if(&another == this) { return; } std::swap(this->header, another.header); this->tags.swap(another.tags); this->index.swap(another.index); this->graph.swap(another.graph); // GBWTGraph did not know that we also swapped the GBWTs. this->set_gbwt_address(); another.set_gbwt_address(); } GBZ& GBZ::operator=(const GBZ& source) { if(&source != this) { this->copy(source); } return *this; } GBZ& GBZ::operator=(GBZ&& source) noexcept { if(&source != this) { this->header = std::move(source.header); this->tags = std::move(source.tags); this->index = std::move(source.index); this->graph = std::move(source.graph); // GBWTGraph did not know that we also moved the GBWT. this->set_gbwt_address(); } return *this; } void GBZ::copy(const GBZ& source) { this->header = source.header; this->tags = source.tags; this->index = source.index; this->graph = source.graph; // Use the local copy of the GBWT. this->set_gbwt_address(); } void GBZ::reset_tags() { this->tags.clear(); this->add_source(); } void GBZ::add_source() { this->tags.set(Version::SOURCE_KEY, Version::SOURCE_VALUE); } //------------------------------------------------------------------------------ GBZ::GBZ(std::unique_ptr& index, std::unique_ptr& graph) { if(index == nullptr || graph == nullptr) { throw std::runtime_error("GBZ: Index and graph must be non-null"); } this->add_source(); this->index = std::move(*index); index.reset(); GraphName parent = graph->graph_name(); this->graph = GBWTGraph(this->index, *graph); graph.reset(); this->compute_pggname(&parent); } GBZ::GBZ(const gbwt::GBWT& index, const NaiveGraph& graph) : index(index), graph(this->index, graph) { this->add_source(); GraphName parent = graph.graph_name(); this->compute_pggname(&parent); } GBZ::GBZ(std::vector&& subgraphs) { if(subgraphs.empty()) { this->add_source(); this->set_gbwt(); this->compute_pggname(nullptr); return; } // Move the GBWTs into a vector for the GBWT merge constructor. std::vector indexes; indexes.reserve(subgraphs.size()); for(GBZ& subgraph : subgraphs) { indexes.push_back(std::move(subgraph.index)); subgraph.graph.set_gbwt_address(indexes.back()); } // Determine the reference samples as a union over the subgraphs. sample_name_set reference_samples; for(const gbwt::GBWT& index : indexes) { auto current_ref_samples = parse_reference_samples_tag(index); reference_samples.insert(current_ref_samples.begin(), current_ref_samples.end()); } // Merge the GBWTs and set reference samples. // This may fail if the subgraphs are not disjoint. this->index = gbwt::GBWT(indexes); if(!reference_samples.empty()) { std::string reference_samples_tag = compose_reference_samples_tag(reference_samples); this->index.tags.set(REFERENCE_SAMPLE_LIST_GBWT_TAG, reference_samples_tag); } // Create the union of the graphs. NaiveGraph sequence_source; for(const GBZ& subgraph : subgraphs) { subgraph.graph.for_each_handle([&](const handle_t& handle) { std::string_view sequence = subgraph.graph.get_sequence_view(handle); sequence_source.create_node(subgraph.graph.get_id(handle), sequence); }); } indexes.clear(); subgraphs.clear(); // Free memory before building the GBWTGraph. this->graph = GBWTGraph(this->index, sequence_source); // Determine GBZ tags. this->add_source(); this->compute_pggname(nullptr); } GBZ::GBZ(gbwt::GBWT&& index, const GBZ& supergraph) : index(std::move(index)), graph(supergraph.graph.subgraph(this->index)) { this->add_source(); GraphName parent = supergraph.graph_name(); this->compute_pggname(&parent, ParentGraphType::SUPERGRAPH); } GBZ::GBZ(gbwt::GBWT&& index, const HandleGraph& graph, const NamedNodeBackTranslation* segment_space) : index(index), graph(this->index, graph, segment_space) { this->add_source(); } void GBZ::set_gbwt() { this->graph.set_gbwt(this->index); } void GBZ::set_gbwt_address() { this->graph.set_gbwt_address(this->index); } //------------------------------------------------------------------------------ bool GBZ::compute_pggname(const GraphName* parent, ParentGraphType relationship) { // Compute the name. DigestStream digest_stream(EVP_sha256()); gbwt_to_canonical_gfa(this->graph, digest_stream); std::string digest = digest_stream.finish(); if(digest.empty()) { return false; } // Set the name and copy existing relationships. GraphName name(digest); name.add_relationships(this->graph_name()); // Determine the relationship to the parent graph, if given, // and copy relationships from it. if(parent != nullptr && parent->has_name()) { if(relationship == ParentGraphType::HEURISTIC) { relationship = (this->graph.has_segment_names() ? ParentGraphType::TRANSLATION_TARGET : ParentGraphType::SUPERGRAPH); } if(relationship == ParentGraphType::TRANSLATION_TARGET) { if(!name.same(*parent)) { name.add_translation(name.name(), parent->name()); this->tags.set(GraphName::GBZ_TRANSLATION_TARGET_TAG, parent->name()); } } else { // This does nothing if the names are the same. name.add_subgraph(name.name(), parent->name()); } name.add_relationships(*parent); } // Store the information back into the tags. name.set_tags(this->tags); return true; } //------------------------------------------------------------------------------ void GBZ::simple_sds_serialize_version(std::ostream& out, std::uint32_t version) const { if(version < Header::MIN_SERIALIZE_VERSION || version > Header::VERSION) { throw sdsl::simple_sds::UnsupportedVersion("GBZ", version, Header::MIN_SERIALIZE_VERSION, Header::VERSION); } // Serialize the header. Header copy = this->header; copy.version = version; sdsl::simple_sds::serialize_value(copy, out); this->tags.simple_sds_serialize(out); if(version >= Header::ZSTD_BWT_VERSION) { this->index.simple_sds_serialize_version(out, gbwt::GBWTHeader::ZSTD_VERSION); } else { this->index.simple_sds_serialize_version(out, gbwt::GBWTHeader::TAGS_VERSION); } if(version >= Header::ZSTD_SEQUENCES_VERSION) { this->graph.simple_sds_serialize_version(out, GBWTGraph::Header::ZSTD_VERSION); } else { this->graph.simple_sds_serialize_version(out, GBWTGraph::Header::SIMPLE_SDS_VERSION); } } void GBZ::simple_sds_serialize(const gbwt::GBWT& index, const GBWTGraph& graph, std::ostream& out) { GBZ empty; sdsl::simple_sds::serialize_value(empty.header, out); empty.tags.simple_sds_serialize(out); index.simple_sds_serialize(out); graph.simple_sds_serialize(out); } void GBZ::simple_sds_load(std::istream& in) { this->header = sdsl::simple_sds::load_value
(in); this->header.check(); this->header.set_version(); // Load the tags and update the source to this library. // We could also check if the source was already this library, but we have no // uses for that information at the moment. this->tags.simple_sds_load(in); this->add_source(); this->index.simple_sds_load(in); this->graph.simple_sds_load(in, this->index); } size_t GBZ::simple_sds_size() const { size_t result = sdsl::simple_sds::value_size(this->header); result += this->tags.simple_sds_size(); result += this->index.simple_sds_size(); result += this->graph.simple_sds_size(); return result; } gbwt::Tags GBZ::simple_sds_load_tags(const std::string& filename) { std::ifstream in(filename, std::ios_base::binary); if(!in) { throw sdsl::simple_sds::CannotOpenFile(filename, false); } in.exceptions(std::ios::eofbit | std::ios::badbit | std::ios::failbit); Header header = sdsl::simple_sds::load_value
(in); header.check(); gbwt::Tags tags; tags.simple_sds_load(in); in.close(); return tags; } void GBZ::serialize_to_files(const std::string& gbwt_name, const std::string& graph_name, bool simple_sds_graph) const { sdsl::simple_sds::serialize_to(this->index, gbwt_name); if(simple_sds_graph) { sdsl::simple_sds::serialize_to(this->graph, graph_name); } else { this->graph.serialize(graph_name); } } void GBZ::load_from_files(const std::string& gbwt_name, const std::string& graph_name) { this->header.set_version(); this->tags.clear(); this->add_source(); sdsl::simple_sds::load_from(this->index, gbwt_name); this->set_gbwt(); this->graph.deserialize(graph_name); } //------------------------------------------------------------------------------ } // namespace gbwtgraph jltsiren-gbwtgraph-e27bc43/src/gbz_extract.cpp000066400000000000000000000244411524136466400215710ustar00rootroot00000000000000#include #include #include #include #include #include #include #include #include using namespace gbwtgraph; //------------------------------------------------------------------------------ /* This tool extracts sequences from a GBZ graph, one sequence per selected path. The sequences are written in the same order as the paths in the graph and terminated by a newline character. The sequences are extracted either in the forward orientation or in both orientations. In the latter case, the reverse complement of each sequence follows the original sequence. */ const std::string tool_name = "GBZ Sequence Extractor"; struct Config { Config(int argc, char** argv); std::string graph_name; std::string contig_name; // Extract sequences in both orientations. bool both_orientations = false; // TODO: Block size to assign multiple paths to a thread. bool progress = false; size_t threads; }; // Returns the path identifiers for the selected contig, or for all paths // if the name is empty. std::vector select_paths(const GBZ& graph, const std::string& contig_name, bool progress); // Extracts the nucleotide sequence corresponding to the given path id to the // provided buffer. The buffer is cleared before the extraction. void extract_path(const GBZ& graph, gbwt::size_type path_id, std::string& buffer); //------------------------------------------------------------------------------ int main(int argc, char** argv) { Config config(argc, argv); GBZ graph; double start = gbwt::readTimer(); if(config.progress) { std::cerr << "Loading the graph from " << config.graph_name << std::endl; } sdsl::simple_sds::load_from(graph, config.graph_name); std::vector paths = select_paths(graph, config.contig_name, config.progress); // Start the extraction. if(config.progress) { std::cerr << "Extracting " << paths.size() << " sequences"; if(config.both_orientations) { std::cerr << " in both orientations"; } std::cerr << std::endl; } std::vector threads; threads.reserve(config.threads); std::vector thread_buffers(config.threads); for(size_t to_read = 0; to_read < config.threads && to_read < paths.size(); to_read++) { threads.emplace_back(extract_path, std::ref(graph), paths[to_read], std::ref(thread_buffers[to_read])); } // Write the extracted sequences. size_t total_length = 0; for(size_t to_write = 0; to_write < paths.size(); to_write++) { // Get the buffer from the thread that was responsible for exracting this path. size_t thread_id = to_write % config.threads; std::string write_buffer; threads[thread_id].join(); write_buffer.swap(thread_buffers[thread_id]); // Relaunch the thread if there are still more paths to extract. if(to_write + threads.size() < paths.size()) { threads[thread_id] = std::thread(extract_path, std::ref(graph), paths[to_write + threads.size()], std::ref(thread_buffers[thread_id])); } // And write the extracted sequence. std::cout.write(write_buffer.data(), write_buffer.length()); std::cout.put('\n'); total_length += write_buffer.length(); if(config.both_orientations) { reverse_complement_in_place(write_buffer); std::cout.write(write_buffer.data(), write_buffer.length()); std::cout.put('\n'); total_length += write_buffer.length(); } } if(config.progress) { std::cerr << "Total sequence length: " << total_length << " bp" << std::endl; } if(config.progress) { double seconds = gbwt::readTimer() - start; std::cerr << "Used " << seconds << " seconds, " << gbwt::inGigabytes(gbwt::memoryUsage()) << " GiB" << std::endl; } return 0; } //------------------------------------------------------------------------------ size_t default_threads() { return std::thread::hardware_concurrency(); } size_t max_threads() { return 2 * default_threads(); } void printUsage(int exit_code) { Version::print(std::cerr, tool_name); std::cerr << "Usage: gbz_extract [options] graph.gbz > output" << std::endl; std::cerr << std::endl; std::cerr << "Computational parameters:" << std::endl; std::cerr << " -c, --contig NAME select paths corresponding to contig NAME" << std::endl; std::cerr << " -b, --both-orientations extract sequences in both orientations" << std::endl; std::cerr << std::endl; std::cerr << "Other options:" << std::endl; std::cerr << " -t, --threads N use N threads (default: " << default_threads() << ", max: " << max_threads() << ")" << std::endl; std::cerr << " -p, --progress print progress information" << std::endl; std::cerr << " -h, --help print this help message" << std::endl; std::cerr << std::endl; std::exit(exit_code); } Config::Config(int argc, char** argv) : threads(default_threads()) { if(argc < 2) { printUsage(EXIT_SUCCESS); } // Data for `getopt_long()`. int c = 0, option_index = 0; option long_options[] = { { "contig", required_argument, 0, 'c' }, { "both-orientations", no_argument, 0, 'b' }, { "threads", required_argument, 0, 't' }, { "progress", no_argument, 0, 'p' }, { "help", no_argument, 0, 'h' }, { 0, 0, 0, 0 } }; // Process options. while((c = getopt_long(argc, argv, "c:bt:ph", long_options, &option_index)) != -1) { switch(c) { case 'c': this->contig_name = optarg; break; case 'b': this->both_orientations = true; break; case 't': try { this->threads = std::stoul(optarg); } catch(std::exception& e) { std::cerr << "Cannot parse --threads " << optarg << ": " << e.what() << std::endl; std::exit(EXIT_FAILURE); } break; case 'p': this->progress = true; break; case 'h': printUsage(EXIT_SUCCESS); break; case '?': std::exit(EXIT_FAILURE); default: std::exit(EXIT_FAILURE); } } // Positional arguments. if(optind +1 != argc) { printUsage(EXIT_FAILURE); } this->graph_name = argv[optind]; optind++; // Sanity checks. if(this->threads < 1 || this->threads > max_threads()) { std::cerr << "Invalid number of threads: " << this->threads << " (must be between 1 and " << max_threads() << ")" << std::endl; std::exit(EXIT_FAILURE); } } //------------------------------------------------------------------------------ size_t component_for_path(const GBZ& graph, gbwt::size_type path_id, const std::unordered_map& node_to_component) { gbwt::size_type seq_id = gbwt::Path::encode(path_id, false); nid_t node = gbwt::Node::id(graph.index.start(seq_id).first); auto iter = node_to_component.find(node); return (iter == node_to_component.end() ? std::numeric_limits::max() : iter->second); } std::vector select_paths(const GBZ& graph, const std::string& contig_name, bool progress) { std::vector result; if(contig_name.empty()) { if(progress) { std::cerr << "Selecting all " << (graph.index.sequences() / 2) << " paths" << std::endl; } result.reserve(graph.index.sequences() / 2); for(gbwt::size_type i = 0; i < graph.index.sequences() / 2; i++) { result.push_back(i); } return result; } // Find reference paths for the contig. if(progress) { std::cerr << "Selecting reference paths for contig " << contig_name << std::endl; } const gbwt::Metadata& metadata = graph.index.metadata; if(!(graph.index.hasMetadata()) || !(metadata.hasContigNames()) || !(metadata.hasPathNames())) { std::cerr << "Error: The graph does not have metadata with contig names and path names" << std::endl; std::exit(EXIT_FAILURE); } gbwt::size_type contig_id = metadata.contig(contig_name); if(contig_id >= metadata.contigs()) { std::cerr << "Error: Contig " << contig_name << " not found in the graph" << std::endl; std::exit(EXIT_FAILURE); } std::vector ref_paths = metadata.pathsForContig(contig_id); if(ref_paths.empty()) { std::cerr << "Warning: Contig " << contig_name << " does not have any paths" << std::endl; return result; } if(progress) { std::cerr << "Found " << ref_paths.size() << " reference paths" << std::endl; } // Find the graph components for the selected reference paths. if(progress) { std::cerr << "Finding graph components for contig " << contig_name << std::endl; } auto components = weakly_connected_components(graph.graph); std::unordered_map node_to_component; for(size_t i = 0; i < components.size(); i++) { for(nid_t node : components[i]) { node_to_component[node] = i; } } components = std::vector>(); std::unordered_set selected_components; for(gbwt::size_type ref_path : ref_paths) { size_t component = component_for_path(graph, ref_path, node_to_component); if(component != std::numeric_limits::max()) { selected_components.insert(component); } } if(progress) { std::cerr << "Found " << selected_components.size() << " components" << std::endl; } // Now select the paths. if(progress) { std::cerr << "Selecting paths for contig " << contig_name << std::endl; } for(gbwt::size_type path_id = 0; path_id < metadata.paths(); path_id++) { size_t component = component_for_path(graph, path_id, node_to_component); if(selected_components.find(component) != selected_components.end()) { result.push_back(path_id); } } if(progress) { std::cerr << "Found " << result.size() << " paths" << std::endl; } return result; } //------------------------------------------------------------------------------ void extract_path(const GBZ& graph, gbwt::size_type path_id, std::string& buffer) { buffer.clear(); gbwt::size_type seq_id = gbwt::Path::encode(path_id, false); gbwt::edge_type curr = graph.index.start(seq_id); while(curr.first != gbwt::ENDMARKER) { handle_t handle = GBWTGraph::node_to_handle(curr.first); std::string_view view = graph.graph.get_sequence_view(handle); buffer.append(view.data(), view.size()); curr = graph.index.LF(curr); } } //------------------------------------------------------------------------------ jltsiren-gbwtgraph-e27bc43/src/gbz_find.cpp000066400000000000000000000247401524136466400210410ustar00rootroot00000000000000#include #include #include #include #include #include #include #include using namespace gbwtgraph; //------------------------------------------------------------------------------ /* This tool finds all exact occurrences of the pattern in the haplotypes. It iterates over non-redundant haplotype windows. If a window contains the pattern, the tool outputs the following tab-separated fields: 1. The path using GFA walk notation. 2. The sequence as comma-separated node labels, excluding left/right flanks. 3. Number of haplotypes containing the path. 4. Length of the left flank. 5. Length of the right flank. 6. Reference coordinates for each node, if requested. Left (right) flank is the prefix (suffix) of the first (last) node not included in the window. If the path is a single node, the flanks are always empty. TODO: Make this a library function. TODO: How to avoid reporting the same window in both orientations if the pattern is palindromic? */ const std::string tool_name = "GBZ Haplotype Search"; struct Config { Config(int argc, char** argv); std::string graph_name; std::string pattern; std::string reference_sample; bool progress; size_t threads; constexpr static size_t BUFFER_SIZE = 1024; // Lines. }; // Returns a mapping GBWT node id -> (GBWT sequence id, sequence offset). std::unordered_map> reference_labels(const GBZ& graph, const Config& config); //------------------------------------------------------------------------------ int main(int argc, char** argv) { double start = gbwt::readTimer(); Config config(argc, argv); omp_set_num_threads(config.threads); GBZ graph; if(config.progress) { std::cerr << "Loading the graph from " << config.graph_name << std::endl; } sdsl::simple_sds::load_from(graph, config.graph_name); std::unordered_map> ref_labels = reference_labels(graph, config); auto create_label = [&](gbwt::size_type seq_id, gbwt::size_type offset, bool is_reverse) -> std::string { const gbwt::Metadata& metadata = graph.index.metadata; gbwt::FullPathName path_name = metadata.fullPath(seq_id); std::string result; result.push_back(is_reverse ? '<' : '>'); result.append(path_name.sample_name); result.push_back('#'); result.append(std::to_string(path_name.haplotype)); result.push_back('#'); result.append(path_name.contig_name); result.push_back('#'); result.append(std::to_string(path_name.offset + offset)); return result; }; auto get_label = [&](handle_t handle) -> std::string { gbwt::node_type node = GBWTGraph::handle_to_node(handle); auto it = ref_labels.find(node); if(it != ref_labels.end()) { return create_label(it->second.first, it->second.second, false); } it = ref_labels.find(gbwt::Node::reverse(node)); if(it != ref_labels.end()) { return create_label(it->second.first, it->second.second, true); } return ">-"; }; if(config.progress) { std::cerr << "Searching the haplotypes for " << config.pattern << std::endl; } std::vector> buffers(config.threads); size_t windows = 0; auto flush = [&](size_t thread_id) { std::vector& buffer = buffers[thread_id]; for(const std::string& line : buffer) { std::cout.write(line.data(), line.length()); std::cout.put('\n'); } windows += buffer.size(); buffer.clear(); }; // FIXME: update for_each_nonredundant_window ( graph.graph, config.pattern.length(), [&](const std::vector& path, const std::string& sequence) { if(sequence.find(config.pattern) == std::string::npos) { return; } // Path in the GFA walk notation. std::string line; for(handle_t handle : path) { gbwt::node_type node = GBWTGraph::handle_to_node(handle); line.push_back(gbwt::Node::is_reverse(node) ? '<' : '>'); line += std::to_string(gbwt::Node::id(node)); } // Path as a comma-separated list of oriented node labels. // Also determine the lengths of the left and right flanks. line.push_back('\t'); size_t left_flank = 0, right_flank = 0; if(path.size() > 1) { size_t first_node_length = graph.graph.get_length(path.front()); if(first_node_length >= config.pattern.length()) { left_flank = first_node_length - config.pattern.length() + 1; } size_t offset = first_node_length - left_flank; line += std::string_view(sequence).substr(0, offset); for(size_t i = 1; i < path.size(); i++) { size_t node_length = graph.graph.get_length(path[i]); line.push_back(','); line += std::string_view(sequence).substr(offset, node_length); offset += node_length; } right_flank = offset - sequence.length(); } else { line += sequence; } // Number of haplotypes containing the path. // TODO: for_each_haplotype_window should expose the search state. line.push_back('\t'); gbwt::SearchState state = graph.index.find(GBWTGraph::handle_to_node(path.front())); for(size_t i = 1; i < path.size(); i++) { state = graph.index.extend(state, GBWTGraph::handle_to_node(path[i])); } line += std::to_string(state.size()); // Length of the left and right flanks. line.push_back('\t'); line += std::to_string(left_flank); line.push_back('\t'); line += std::to_string(right_flank); // Reference coordinates for each node, if requested. // If the orientation of the final node is the opposite to the reference, // the coordinates are for the other end of the node. To determine the // true coordinates, we need to use the previous field. if(!ref_labels.empty()) { line.push_back('\t'); for(handle_t handle : path) { line += get_label(handle); } } size_t thread_id = omp_get_thread_num(); buffers[thread_id].push_back(std::move(line)); if(buffers[thread_id].size() >= config.BUFFER_SIZE) { #pragma omp critical flush(thread_id); } }, true ); for(size_t thread_id = 0; thread_id < config.threads; thread_id++) { flush(thread_id); } std::cout.flush(); if(config.progress) { double seconds = gbwt::readTimer() - start; std::cerr << "Found a match in " << windows << " windows in " << seconds << " seconds, " << gbwt::inGigabytes(gbwt::memoryUsage()) << " GiB" << std::endl; } return 0; } //------------------------------------------------------------------------------ size_t default_threads() { return std::thread::hardware_concurrency(); } void printUsage(int exit_code) { Version::print(std::cerr, tool_name); std::cerr << "Usage: gbz_find [options] graph.gbz pattern > output" << std::endl; std::cerr << std::endl; std::cerr << "Options:" << std::endl; std::cerr << " -r, --reference NAME output coordinates in this reference sample" << std::endl; std::cerr << " -t, --threads N use N threads (default and max: " << default_threads() << ")" << std::endl; std::cerr << " -p, --progress print progress information" << std::endl; std::cerr << " -h, --help print this help message" << std::endl; std::cerr << std::endl; std::exit(exit_code); } Config::Config(int argc, char** argv) : progress(false), threads(default_threads()) { if(argc < 2) { printUsage(EXIT_SUCCESS); } // Data for `getopt_long()`. int c = 0, option_index = 0; option long_options[] = { { "reference", required_argument, 0, 'r' }, { "threads", required_argument, 0, 't' }, { "progress", no_argument, 0, 'p' }, { "help", no_argument, 0, 'h' }, { 0, 0, 0, 0 } }; // Process options. while((c = getopt_long(argc, argv, "r:t:ph", long_options, &option_index)) != -1) { switch(c) { case 'r': this->reference_sample = optarg; break; case 't': try { this->threads = std::stoul(optarg); } catch(std::exception& e) { std::cerr << "Cannot parse --threads " << optarg << ": " << e.what() << std::endl; std::exit(EXIT_FAILURE); } break; case 'p': this->progress = true; break; case 'h': printUsage(EXIT_SUCCESS); break; case '?': std::exit(EXIT_FAILURE); default: std::exit(EXIT_FAILURE); } } // Positional arguments. if(optind + 2 != argc) { printUsage(EXIT_FAILURE); } this->graph_name = argv[optind]; optind++; this->pattern = argv[optind]; optind++; // Sanity checks. if(this->threads < 1 || this->threads > default_threads()) { std::cerr << "Invalid number of threads: " << this->threads << " (must be between 1 and " << default_threads() << ")" << std::endl; std::exit(EXIT_FAILURE); } if(this->pattern.empty()) { std::cerr << "Pattern cannot be empty" << std::endl; std::exit(EXIT_FAILURE); } } //------------------------------------------------------------------------------ std::unordered_map> reference_labels(const GBZ& graph, const Config& config) { std::unordered_map> result; if(config.reference_sample.empty()) { return result; } if(config.progress) { std::cerr << "Determining coordinates in reference sample " << config.reference_sample << std::endl; } const gbwt::Metadata& metadata = graph.index.metadata; gbwt::size_type sample = metadata.sample(config.reference_sample); std::vector paths = metadata.pathsForSample(sample); if(paths.empty()) { if(config.progress) { std::cerr << "No paths found for the reference sample" << std::endl; } return result; } for(gbwt::size_type path_id : paths) { gbwt::vector_type path = graph.index.extract(gbwt::Path::encode(path_id, false)); std::uint32_t offset = 0; for(gbwt::node_type node : path) { // TODO: Should we print a warning? if(result.find(node) != result.end()) { continue; } result[node] = std::pair(path_id, offset); offset += graph.graph.get_length(GBWTGraph::node_to_handle(node)); } } return result; } //------------------------------------------------------------------------------ jltsiren-gbwtgraph-e27bc43/src/gbz_stats.cpp000066400000000000000000000161401524136466400212520ustar00rootroot00000000000000#include #include #include #include #include #include #include using namespace gbwtgraph; //------------------------------------------------------------------------------ const std::string tool_name = "GBZ Statistics"; struct Config { Config(int argc, char** argv); bool graph = false; bool gbwt = false; bool path_length = false; size_t window_length = 0; bool nonredundant = false; bool node_degrees = false; bool node_visits = false; bool record_bytes = false; bool record_runs = false; std::string filename; }; void print_distribution(const std::map& distribution, const std::string& header1, const std::string& header2); //------------------------------------------------------------------------------ int main(int argc, char** argv) { Config config(argc, argv); GBZ gbz; sdsl::simple_sds::load_from(gbz, config.filename); if(config.graph) { std::cout << "Nodes\t" << gbz.graph.get_node_count() << std::endl; std::cout << "Edges\t" << gbz.graph.get_edge_count() << std::endl; size_t total_length = 0; gbz.graph.for_each_handle([&](const handle_t& handle) { total_length += gbz.graph.get_length(handle); }); std::cout << "Sequence\t" << total_length << std::endl; } if(config.gbwt) { gbwt::printStatistics(gbz.index, config.filename, std::cout); } if(config.path_length) { size_t total_length = 0; gbz.graph.for_each_handle([&](const handle_t& handle) { size_t node_length = gbz.graph.get_length(handle); gbwt::size_type node = GBWTGraph::handle_to_node(handle); size_t visits = gbz.index.nodeSize(node); total_length += node_length * visits; }); std::cout << "Path length\t" << total_length << std::endl; } if(config.window_length > 0) { std::atomic fragments(0), total_length(0); auto count = [&](const std::vector&, const std::string& sequence) -> bool { fragments++; total_length += sequence.length(); return true; }; if(config.nonredundant) { for_each_nonredundant_window(gbz.graph, config.window_length, count, true); } else { for_each_haplotype_window(gbz.graph, config.window_length, count, true); } std::cout << "Fragments\t" << fragments << std::endl; std::cout << "Total length\t" << total_length << std::endl; } if(config.node_degrees) { std::map distribution; gbz.graph.for_each_handle([&](const handle_t& handle) { size_t degree = gbz.graph.get_degree(handle, false) + gbz.graph.get_degree(handle, true); distribution[degree]++; }); print_distribution(distribution, "Degree", "Nodes"); } if(config.node_visits) { std::map distribution; for(gbwt::node_type node = gbz.index.firstNode(); node < gbz.index.sigma(); node += 2) { distribution[gbz.index.nodeSize(node)]++; } print_distribution(distribution, "Visits", "Nodes"); } if(config.record_bytes) { std::map distribution; for(gbwt::node_type node = gbz.index.firstNode(); node < gbz.index.sigma(); node++) { std::pair range = gbz.index.bwt.getRange(gbz.index.toComp(node)); distribution[range.second - range.first]++; } print_distribution(distribution, "Bytes", "Records"); } if(config.record_runs) { std::map distribution; for(gbwt::node_type node = gbz.index.firstNode(); node < gbz.index.sigma(); node++) { distribution[gbz.index.record(node).runs().first]++; } print_distribution(distribution, "Runs", "Records"); } return 0; } //------------------------------------------------------------------------------ void printUsage(int exit_code) { Version::print(std::cerr, tool_name); std::cerr << "Usage: gbz_stats [options] graph.gbz" << std::endl; std::cerr << std::endl; std::cerr << "Overall statistics:" << std::endl; std::cerr << " -g, --graph graph statistics" << std::endl; std::cerr << " -i, --gbwt GBWT index statistics" << std::endl; std::cerr << " -p, --path-length total path length (in bp; one orientation)" << std::endl; std::cerr << " -w, --windows N N bp haplotype windows" << std::endl; std::cerr << " --nonredundant nonredundant haplotype windows" << std::endl; std::cerr << std::endl; std::cerr << "Nodes:" << std::endl; std::cerr << " -d, --node-degrees node degree distribution" << std::endl; std::cerr << " -v, --node-visits node visit distribution" << std::endl; std::cerr << std::endl; std::cerr << "GBWT records:" << std::endl; std::cerr << " -b, --record-bytes record size distribution" << std::endl; std::cerr << " -r, --record-runs run count distribution" << std::endl; std::cerr << std::endl; std::exit(exit_code); } //------------------------------------------------------------------------------ Config::Config(int argc, char** argv) { if(argc < 2) { printUsage(EXIT_SUCCESS); } constexpr int OPT_NONREDUNDANT = 1000; // Data for `getopt_long()`. int c = 0, option_index = 0; option long_options[] = { { "graph", no_argument, 0, 'g' }, { "gbwt", no_argument, 0, 'i' }, { "path-length", no_argument, 0, 'p' }, { "windows", required_argument, 0, 'w' }, { "nonredundant", no_argument, 0, OPT_NONREDUNDANT }, { "node-degrees", no_argument, 0, 'd' }, { "node-visits", no_argument, 0, 'v' }, { "record-bytes", no_argument, 0, 'b' }, { "record-runs", no_argument, 0, 'r' }, { 0, 0, 0, 0 } }; // Process options. while((c = getopt_long(argc, argv, "gipw:dvbr", long_options, &option_index)) != -1) { switch(c) { case 'g': this->graph = true; break; case 'i': this->gbwt = true; break; case 'p': this->path_length = true; break; case 'w': try { this->window_length = std::stoul(optarg); } catch(std::exception& e) { std::cerr << "Cannot parse --window " << optarg << ": " << e.what() << std::endl; std::exit(EXIT_FAILURE); } break; case OPT_NONREDUNDANT: this->nonredundant = true; break; case 'd': this->node_degrees = true; break; case 'v': this->node_visits = true; break; case 'b': this->record_bytes = true; break; case 'r': this->record_runs = true; break; case '?': std::exit(EXIT_FAILURE); default: std::exit(EXIT_FAILURE); } } // Sanity checks. if(optind >= argc) { printUsage(EXIT_FAILURE); } this->filename = argv[optind]; optind++; } //------------------------------------------------------------------------------ void print_distribution(const std::map& distribution, const std::string& header1, const std::string& header2) { std::cout << header1 << "\t" << header2 << std::endl; for(auto iter = distribution.begin(); iter != distribution.end(); ++iter) { std::cout << iter->first << "\t" << iter->second << std::endl; } } //------------------------------------------------------------------------------ jltsiren-gbwtgraph-e27bc43/src/gfa.cpp000066400000000000000000002112471524136466400200140ustar00rootroot00000000000000#include #include #include #include #include #include #include #include #include #include #include #include namespace gbwtgraph { //------------------------------------------------------------------------------ // Global variables. const std::string GFA_EXTENSION = ".gfa"; // Class constants. constexpr size_t GFAParsingParameters::APPROXIMATE_NUM_JOBS; const std::string GFAParsingParameters::DEFAULT_REGEX = ".*"; const std::string GFAParsingParameters::DEFAULT_FIELDS = "C"; const PathSense GFAParsingParameters::DEFAULT_SENSE = PathSense::GENERIC; const std::string GFAParsingParameters::PAN_SN_REGEX = "(.*)#([0-9]+)#(.*)"; const std::string GFAParsingParameters::PAN_SN_FIELDS = "XSHC"; const PathSense GFAParsingParameters::PAN_SN_SENSE = PathSense::HAPLOTYPE; const std::vector GFAParsingParameters::SUPPORTED_VERSIONS { "1.0", "1.1", "1.2" }; constexpr size_t GFAExtractionParameters::LARGE_RECORD_BYTES; //------------------------------------------------------------------------------ GFAParsingParameters::PathNameParsingParameters::PathNameParsingParameters(const std::string& regex, const std::string& fields, PathSense sense) : regex(regex), fields(fields), sense(sense) { } //------------------------------------------------------------------------------ // Parse a nonnegative integer, assuming that the string is valid. std::uint64_t stoul_unsafe(const std::string& str) { std::uint64_t result = 0; for(char c : str) { result = 10 * result + (c - '0'); } return result; } //------------------------------------------------------------------------------ struct GFAFile { // Memory mapped file. int fd; size_t file_size; char* ptr; // GFA information. bool translate_segment_ids; size_t max_segment_length, max_path_length; // Bit masks for field separators. size_t field_end[4]; size_t subfield_end[4]; size_t walk_subfield_end[4]; // Headers: stored header lines and pointers to tags. std::vector h_lines; std::vector h_tags; // Pointers to line starts. std::vector s_lines; std::vector l_lines; std::vector p_lines; std::vector w_lines; std::vector q_lines; struct field_type { const char* begin; const char* end; size_t line_num; char type; bool has_next; size_t size() const { return this->end - this->begin; } bool empty() const { return (this->size() == 0); } std::string str() const { return std::string(this->begin, this->end); } std::string_view view() const { return std::string_view(this->begin, this->end - this->begin); } char front() const { return *(this->begin); } char back() const { return *(this->end - 1); } // For segment orientations in links. bool valid_orientation() const { return (this->size() == 1 && (this->back() == '-' || this->back() == '+')); } bool is_reverse_orientation() const { return (this->back() == '-'); } // For path segment subfields. bool valid_path_segment() const { return (this->size() >= 2 && (this->back() == '-' || this->back() == '+')); } std::string path_segment() const { return std::string(this->begin, this->end - 1); } std::string_view path_segment_view() const { return std::string_view(this->begin, (this->end - 1) - this->begin); } bool is_reverse_path_segment() const { return (this->back() == '-'); } // Usually the next field/subfield starts at `end + 1`, because `end` points // to the separator. Walk subfields include the separator as a part of the field, // so they start at `end` instead. Before we go to the first subfield, we must // increment `end` (which points to the preceding '\t' before the call). void start_walk() { this->end++; } // For walk segment subfields. bool valid_walk_segment() const { return (this->size() >= 2 && (this->front() == '<' || this->front() == '>')); } std::string walk_segment() const { return std::string(this->begin + 1, this->end); } std::string_view walk_segment_view() const { return std::string_view(this->begin + 1, this->end - (this->begin + 1)); } bool is_reverse_walk_segment() const { return (this->front() == '<'); } // For tags (always 2 character name, one character type, colon separators) bool valid_tag() const { return this->size() >= 5 && this->begin[2] == ':' && this->begin[4] == ':'; } std::string tag_name() const { return std::string(this->begin, this->begin + 2); } std::string_view tag_name_view() const { return std::string_view(this->begin, 2); } char tag_type() const { return this->begin[3]; } std::string tag_value() const { return std::string(this->begin + 5, this->end); } std::string_view tag_value_view() const { return std::string_view(this->begin + 5, this->end - (this->begin + 5)); } }; // Memory map and validate a GFA file. The constructor checks that all mandatory // fields used for GBWTGraph construction exist and are nonempty. // There are no checks for duplicates. // Throws `std::runtime_error` on failure. GFAFile(const std::string& filename, bool show_progress); ~GFAFile(); size_t size() const { return this->file_size; } size_t header_lines() const { return this->h_lines.size(); } size_t header_tags() const { return this->h_tags.size(); } size_t segments() const { return this->s_lines.size(); } size_t links() const { return this->l_lines.size(); } size_t paths() const { return this->p_lines.size(); } size_t walks() const { return this->w_lines.size(); } size_t rules() const { return this->q_lines.size(); } private: // Preprocess a new H-line. Returns an iterator at the start of the next line or // throws `std::runtime_error` if the parse failed. const char* add_h_line(const char* iter, size_t line_num); // Preprocess a new S-line. Returns an iterator at the start of the next line or // throws `std::runtime_error` if the parse failed. const char* add_s_line(const char* iter, size_t line_num); // Preprocess a new S-line. Returns an iterator at the start of the next line or // throws `std::runtime_error` if the parse failed. const char* add_l_line(const char* iter, size_t line_num); // Preprocess a new P-line. Returns an iterator at the start of the next line or // throws `std::runtime_error` if the parse failed. const char* add_p_line(const char* iter, size_t line_num); // Preprocess a new W-line (or a Z-line). Returns an iterator at the start of the // next line or throws `std::runtime_error` if the parse failed. const char* add_w_line(const char* iter, size_t line_num); // Preprocess a new Q-line. Returns an iterator at the start of the next line or // throws `std::runtime_error` if the parse failed. const char* add_q_line(const char* iter, size_t line_num); // Throws `std::runtime_error` if the field is invalid. void check_field(const field_type& field, const std::string& field_name, bool should_have_next); const char* begin() const { return this->ptr; } const char* end() const { return this->ptr + this->size(); } // Return an iterator to the beginning of the next line. const char* next_line(const char* iter) const { while(iter != this->end() && *iter != '\n') { ++iter; } if(iter != this->end()) { ++iter; } return iter; } // Return the first tab-separated field at or after the given position. field_type first_field(const char* start, size_t line_num = 0) const { const char* limit = start; while(limit != this->end() && !(this->is_field_end(limit))) { ++limit; } return { start, limit, line_num, *start, (limit != this->end() && *limit == '\t') }; } // Return the next tab-separated field, assuming there is one. field_type next_field(const field_type& field) const { const char* limit = field.end + 1; while(limit != this->end() && !(this->is_field_end(limit))) { ++limit; } return { field.end + 1, limit, field.line_num, field.type, (limit != this->end() && *limit == '\t') }; } // Return the next comma-separated subfield, assuming there is one. field_type next_subfield(const field_type& field) const { const char* limit = field.end + 1; while(limit != this->end() && !(this->is_subfield_end(limit))) { ++limit; } return { field.end + 1, limit, field.line_num, field.type, (limit != this->end() && *limit == ',') }; } // Return the next walk subfield, assuming there is one. // The orientation symbol at the start of the segment is also used as subfield separator. field_type next_walk_subfield(const field_type& field) const { const char* limit = field.end; if(limit != this->end() && (*limit == '<' || *limit == '>')) { do { limit++; } while(limit != this->end() && !(this->is_walk_subfield_end(limit))); } return { field.end, limit, field.line_num, field.type, (limit != this->end() && (*limit == '<' || *limit == '>')) }; } bool is_field_end(const char* iter) const { unsigned char c = *iter; return (this->field_end[c / 64] & (size_t(1) << (c & 0x3F))); } bool is_subfield_end(const char* iter) const { unsigned char c = *iter; return (this->subfield_end[c / 64] & (size_t(1) << (c & 0x3F))); } bool is_walk_subfield_end(const char* iter) const { unsigned char c = *iter; return (this->walk_subfield_end[c / 64] & (size_t(1) << (c & 0x3F))); } void add_field_end(unsigned char c) { this->field_end[c / 64] |= size_t(1) << (c & 0x3F); } void add_subfield_end(unsigned char c) { this->subfield_end[c / 64] |= size_t(1) << (c & 0x3F); } void add_walk_subfield_end(unsigned char c) { this->walk_subfield_end[c / 64] |= size_t(1) << (c & 0x3F); } public: /* Iterate over the H-line tags, calling header_tag() for all tags. */ void for_each_header_tag(const std::function& header_tag) const; /* Iterate over the S-lines, calling segment() for all segments. */ void for_each_segment(const std::function& segment) const; /* Iterate over the Q-lines, calling rule() for all grammar rules, expansion_symbol() for all symbols in the expansions, and finish_rule() after parsing each rule. */ void for_each_grammar_rule ( const std::function& rule, const std::function& expansion_symbol, const std::function& finish_rule ) const; /* Iterate over the L-lines, calling link() for all segments. */ void for_each_link(const std::function& link) const; /* Iterate over the file, calling path_start() for each path. */ void for_each_path_start(const std::function& path_start) const; /* Iterate over the file, calling path() for the selected paths. */ void for_these_path_names(const std::vector& selected_paths, const std::function& path) const; /* Iterate over the file, calling path() for the selected paths, path_segment() for each path segment, and finish_path() after parsing each path. */ void for_these_paths(const std::vector& selected_paths, const std::function& path, const std::function& path_segment, const std::function& finish_path) const; /* Iterate over the file, calling walk_start() for each walk. The first symbol may be a segment or a rule. */ void for_each_walk_start(const std::function& walk_start) const; /* Iterate over the file, calling walk() for the selected walks. */ void for_these_walk_names(const std::vector& selected_walks, const std::function& walk) const; /* Iterate over the file, calling walk() for the selected walks, walk_segment() for each walk segment, and finish_walk() after parsing each walk. */ void for_these_walks(const std::vector& selected_walks, const std::function& walk, const std::function& walk_segment, const std::function& finish_walk) const; }; //------------------------------------------------------------------------------ GFAFile::GFAFile(const std::string& filename, bool show_progress) : fd(-1), file_size(0), ptr(nullptr), translate_segment_ids(false), max_segment_length(0), max_path_length(0) { if(show_progress) { std::cerr << "Opening GFA file " << filename << std::endl; } // Open the file. this->fd = ::open(filename.c_str(), O_RDONLY); if(this->fd < 0) { throw std::runtime_error("GFAFile: Cannot open file " + filename); } // Memory map the file. struct stat st; if(::fstat(this->fd, &st) < 0) { throw std::runtime_error("GFAFile: Cannot stat file " + filename); } this->file_size = st.st_size; void* temp_ptr = ::mmap(nullptr, file_size, PROT_READ, MAP_FILE | MAP_SHARED, this->fd, 0); if(temp_ptr == MAP_FAILED) { throw std::runtime_error("GFAFile: Cannot memory map file " + filename); } ::madvise(temp_ptr, file_size, MADV_SEQUENTIAL); // We will be making sequential passes over the data. this->ptr = static_cast(temp_ptr); // Mark characters indicating field/subfield end. This could depend on the GFA version. // TODO: If that happens, we need variables for field/subfield separators. this->field_end[0] = 0; this->field_end[1] = 0; this->field_end[2] = 0; this->field_end[3] = 0; this->add_field_end('\n'); this->add_field_end('\t'); this->subfield_end[0] = 0; this->subfield_end[1] = 0; this->subfield_end[2] = 0; this->subfield_end[3] = 0; this->add_subfield_end('\n'); this->add_subfield_end('\t'); this->add_subfield_end(','); this->walk_subfield_end[0] = 0; this->walk_subfield_end[1] = 0; this->walk_subfield_end[2] = 0; this->walk_subfield_end[3] = 0; this->add_walk_subfield_end('\n'); this->add_walk_subfield_end('\t'); this->add_walk_subfield_end('<'); this->add_walk_subfield_end('>'); // Preprocess and validate the file. double start = gbwt::readTimer(); if(show_progress) { std::cerr << "Validating GFA file " << filename << std::endl; } const char* iter = this->begin(); size_t line_num = 0, unknown_lines = 0; while(iter != this->end()) { switch(*iter) { case 'H': iter = this->add_h_line(iter, line_num); break; case 'S': iter = this->add_s_line(iter, line_num); break; case 'L': iter = this->add_l_line(iter, line_num); break; case 'P': iter = this->add_p_line(iter, line_num); break; case 'Z': // Fall through; deprecated compressed walks. case 'W': iter = this->add_w_line(iter, line_num); break; case 'Q': iter = this->add_q_line(iter, line_num); break; default: iter = this->next_line(iter); unknown_lines++; break; } if(iter == nullptr) { return; } line_num++; } if(show_progress) { double seconds = gbwt::readTimer() - start; std::cerr << "Found " << this->segments() << " segments, " << this->links() << " links, " << this->paths() << " paths, " << this->walks() << " walks, and " << this->rules() << " rules in " << seconds << " seconds" << std::endl; if(unknown_lines > 0) { std::cerr << "Skipped " << unknown_lines << " lines of unknown type" << std::endl; } } } GFAFile::~GFAFile() { if(this->ptr != nullptr) { ::munmap(static_cast(this->ptr), this->file_size); this->file_size = 0; this->ptr = nullptr; } if(this->fd >= 0) { ::close(this->fd); this->fd = -1; } } const char* GFAFile::add_h_line(const char* iter, size_t line_num) { const char* line_start = iter; // Skip the record type field. field_type field = this->first_field(iter, line_num); this->check_field(field, "record type", true); while(field.has_next) { // Look at the tags. field = this->next_field(field); if(!field.valid_tag()) { throw std::runtime_error("GFAFile: Invalid header tag " + field.str() + " on line " + std::to_string(line_num)); } // Save them all. h_tags.push_back(field.begin); } // Store the header line for GraphName parsing. const char* next = this->next_line(field.end); std::string line(line_start, next - line_start); if(line.back() == '\n') { line.pop_back(); } this->h_lines.push_back(line); return next; } const char* GFAFile::add_s_line(const char* iter, size_t line_num) { this->s_lines.push_back(iter); // Skip the record type field. field_type field = this->first_field(iter, line_num); this->check_field(field, "record type", true); // Segment name field. field = this->next_field(field); this->check_field(field, "segment name", true); std::string name = field.str(); if(!(this->translate_segment_ids)) { auto parse = parse_unsigned(name); if(!parse.second || parse.first == 0) { this->translate_segment_ids = true; } } // Sequence field. field = this->next_field(field); this->check_field(field, "sequence", false); this->max_segment_length = std::max(this->max_segment_length, field.size()); return this->next_line(field.end); } const char* GFAFile::add_l_line(const char* iter, size_t line_num) { this->l_lines.push_back(iter); // Skip the record type field. field_type field = this->first_field(iter, line_num); this->check_field(field, "record type", true); // Source segment field. field = this->next_field(field); this->check_field(field, "source segment", true); // Source orientation field. field = this->next_field(field); this->check_field(field, "source orientation", true); if(!(field.valid_orientation())) { throw std::runtime_error("GFAFile: Invalid source orientation " + field.str() + " on line " + std::to_string(line_num)); } // Destination segment field. field = this->next_field(field); this->check_field(field, "destination segment", true); // Destination orientation field. field = this->next_field(field); this->check_field(field, "destination orientation", false); if(!(field.valid_orientation())) { throw std::runtime_error("GFAFile: Invalid destination orientation " + field.str() + " on line " + std::to_string(line_num)); } return this->next_line(field.end); } const char* GFAFile::add_p_line(const char* iter, size_t line_num) { this->p_lines.push_back(iter); // Skip the record type field. field_type field = this->first_field(iter, line_num); this->check_field(field, "record type", true); // Path name field. field = this->next_field(field); this->check_field(field, "path name", true); // Segment names field. size_t path_length = 0; do { field = this->next_subfield(field); if(!(field.valid_path_segment())) { throw std::runtime_error("GFAFile: Invalid path segment " + field.str() + " on line " + std::to_string(line_num)); } path_length++; } while(field.has_next); if(path_length == 0) { throw std::runtime_error("GFAFile: The path on line " + std::to_string(line_num) + " is empty"); } this->max_path_length = std::max(this->max_path_length, path_length); return this->next_line(field.end); } const char* GFAFile::add_w_line(const char* iter, size_t line_num) { this->w_lines.push_back(iter); // Skip the record type field. field_type field = this->first_field(iter, line_num); this->check_field(field, "record type", true); // Sample name field. field = this->next_field(field); this->check_field(field, "sample name", true); // Haplotype index field. field = this->next_field(field); this->check_field(field, "haplotype index", true); // Contig name field. field = this->next_field(field); this->check_field(field, "contig name", true); // Start position field. field = this->next_field(field); this->check_field(field, "start position", true); // Skip the end position field. field = this->next_field(field); this->check_field(field, "end position", true); // TODO: The estimated length is wrong for compressed walks. But we cannot determine // the true length, as GFA provides no guarantees that grammar rules are defined // before their use. // Segment names field. size_t path_length = 0; field.start_walk(); do { field = this->next_walk_subfield(field); if(!(field.valid_walk_segment())) { throw std::runtime_error("GFAFile: Invalid walk segment " + field.str() + " on line " + std::to_string(line_num)); } path_length++; } while(field.has_next); if(path_length == 0) { throw std::runtime_error("GFAFile: The walk on line " + std::to_string(line_num) + " is empty"); } this->max_path_length = std::max(this->max_path_length, path_length); return this->next_line(field.end); } const char* GFAFile::add_q_line(const char* iter, size_t line_num) { this->q_lines.push_back(iter); // Skip the record type field. field_type field = this->first_field(iter, line_num); this->check_field(field, "record type", true); // Rule name field. field = this->next_field(field); this->check_field(field, "rule name", true); // Expansion field. size_t expansion_length = 0; field.start_walk(); do { field = this->next_walk_subfield(field); if(!(field.valid_walk_segment())) { throw std::runtime_error("GFAFile: Invalid expansion segment " + field.str() + " on line " + std::to_string(line_num)); } expansion_length++; } while(field.has_next); if(expansion_length == 0) { throw std::runtime_error("GFAFile: The expansion on line " + std::to_string(line_num) + " is empty"); } // TODO: Do we need to keep track of expansion lengths? return this->next_line(iter); } void GFAFile::check_field(const field_type& field, const std::string& field_name, bool should_have_next) { if(field.empty()) { throw std::runtime_error("GFAFile: " + std::string(field.type, 1) + "-line " + std::to_string(field.line_num) + " has no " + field_name); } if(should_have_next && !(field.has_next)) { throw std::runtime_error("GFAFile: " + std::string(field.type, 1) + "-line " + std::to_string(field.line_num) + " ended after " + field_name); } } //------------------------------------------------------------------------------ void GFAFile::for_each_header_tag(const std::function& header_tag) const { for(const char* iter : this->h_tags) { // The tag is a field field_type field = this->first_field(iter); // Show its pieces to the function header_tag(field.tag_name(), field.tag_type(), field.tag_value_view()); } } void GFAFile::for_each_segment(const std::function& segment) const { for(const char* iter : this->s_lines) { // Skip the record type field. field_type field = this->first_field(iter); // Segment name field. field = this->next_field(field); std::string name = field.str(); // Sequence field. field = this->next_field(field); std::string_view sequence = field.view(); segment(name, sequence); } } void GFAFile::for_each_grammar_rule ( const std::function& rule, const std::function& expansion_symbol, const std::function& finish_rule ) const { for(const char* iter : this->q_lines) { // Skip the record type field. field_type field = this->first_field(iter); // Rule name field. field = this->next_field(field); std::string name = field.str(); rule(name); // Expansion field. field.start_walk(); do { field = this->next_walk_subfield(field); std::string symbol = field.walk_segment(); expansion_symbol(symbol, field.is_reverse_walk_segment()); } while(field.has_next); finish_rule(); } } void GFAFile::for_each_link(const std::function& link) const { for(const char* iter : this->l_lines) { // Skip the record type field. field_type field = this->first_field(iter); // Source segment field. field = this->next_field(field); std::string from = field.str(); // Source orientation field. field = this->next_field(field); bool from_is_reverse = field.is_reverse_orientation(); // Destination segment field. field = this->next_field(field); std::string to = field.str(); // Destination orientation field. field = this->next_field(field); bool to_is_reverse = field.is_reverse_orientation(); link(from, from_is_reverse, to, to_is_reverse); } } //------------------------------------------------------------------------------ void GFAFile::for_each_path_start(const std::function& path_start) const { for(const char* iter : this->p_lines) { const char* line_start = iter; // Skip the record type field and the path name field. field_type field = this->first_field(iter); field = this->next_field(field); // First segment. field = this->next_subfield(field); std::string first_segment = field.path_segment(); path_start(line_start, first_segment); } } void GFAFile::for_these_path_names(const std::vector& selected_paths, const std::function& path) const { for(const char* iter : selected_paths) { // Skip the record type field. field_type field = this->first_field(iter); // Path name field. field = this->next_field(field); std::string path_name = field.str(); path(path_name); } } void GFAFile::for_these_paths(const std::vector& selected_paths, const std::function& path, const std::function& path_segment, const std::function& finish_path) const { for(const char* iter : selected_paths) { // Skip the record type field. field_type field = this->first_field(iter); // Path name field. field = this->next_field(field); path(field.str()); // Segment names field. do { field = this->next_subfield(field); std::string segment_name = field.path_segment(); path_segment(segment_name, field.is_reverse_path_segment()); } while(field.has_next); finish_path(); } } //------------------------------------------------------------------------------ void GFAFile::for_each_walk_start(const std::function& walk_start) const { for(const char* iter : this->w_lines) { const char* line_start = iter; // Skip the record type field and metadata fields. field_type field = this->first_field(iter); field = this->next_field(field); // Sample. field = this->next_field(field); // Haplotype. field = this->next_field(field); // Contig. field = this->next_field(field); // Start. field = this->next_field(field); // End. // First segment. field.start_walk(); field = this->next_walk_subfield(field); std::string first_segment = field.walk_segment(); walk_start(line_start, first_segment); } } void GFAFile::for_these_walk_names(const std::vector& selected_walks, const std::function& walk) const { for(const char* iter : selected_walks) { // Skip the record type field. field_type field = this->first_field(iter); // Sample field. field = this->next_field(field); std::string sample = field.str(); // Haplotype field. field = this->next_field(field); std::string haplotype = field.str(); // Contig field. field = this->next_field(field); std::string contig = field.str(); // Start field. field = this->next_field(field); std::string start = field.str(); walk(sample, haplotype, contig, start); } } void GFAFile::for_these_walks(const std::vector& selected_walks, const std::function& walk, const std::function& walk_segment, const std::function& finish_walk) const { for(const char* iter : selected_walks) { // Skip the record type field. field_type field = this->first_field(iter); // Sample field. field = this->next_field(field); std::string sample = field.str(); // Haplotype field. field = this->next_field(field); std::string haplotype = field.str(); // Contig field. field = this->next_field(field); std::string contig = field.str(); // Start field. field = this->next_field(field); std::string start = field.str(); walk(sample, haplotype, contig, start); // Skip the end field. field = this->next_field(field); // Segment names field. field.start_walk(); do { field = this->next_walk_subfield(field); std::string segment_name = field.walk_segment(); walk_segment(segment_name, field.is_reverse_walk_segment()); } while(field.has_next); finish_walk(); } } //------------------------------------------------------------------------------ void check_gfa_file(const GFAFile& gfa_file, const GFAParsingParameters& parameters) { std::unordered_set supported_versions ( GFAParsingParameters::SUPPORTED_VERSIONS.begin(), GFAParsingParameters::SUPPORTED_VERSIONS.end() ); gfa_file.for_each_header_tag([&](const std::string& name, char type, std::string_view value) { if(name == "VN" && type == 'Z') { std::string gfa_version = std::string(value); if(supported_versions.find(gfa_version) == supported_versions.end()) { std::cerr << "Warning: Unfamiliar GFA version: " << gfa_version << std::endl; } } }); if(gfa_file.segments() == 0) { throw std::runtime_error("No segments in the GFA file"); } if(gfa_file.paths() > 0) { if(parameters.show_progress) { std::cerr << "Storing generic paths as sample " << GENERIC_PATH_SAMPLE_NAME << std::endl; } } if(gfa_file.paths() == 0 && gfa_file.walks() == 0) { throw std::runtime_error("No paths or walks in the GFA file"); } } gbwt::size_type determine_batch_size(const GFAFile& gfa_file, const GFAParsingParameters& parameters) { gbwt::size_type batch_size = parameters.batch_size; if(parameters.automatic_batch_size) { gbwt::size_type min_size = gbwt::DynamicGBWT::MIN_SEQUENCES_PER_BATCH * (gfa_file.max_path_length + 1); batch_size = std::max(min_size, batch_size); batch_size = std::min(static_cast(gfa_file.size()), batch_size); } if(parameters.show_progress) { std::cerr << "GBWT insertion batch size: " << batch_size << " nodes" << std::endl; } return batch_size; } struct ConstructionJob { size_t num_nodes; size_t id; std::vector p_lines; std::vector w_lines; // Largest jobs first. bool operator<(const ConstructionJob& another) const { return (this->num_nodes > another.num_nodes); } }; //------------------------------------------------------------------------------ // NaiveGraph construction may throw std::runtime_error on error. std::unique_ptr parse_segments(const GFAFile& gfa_file, const GFAParsingParameters& parameters) { double start = gbwt::readTimer(); if(parameters.show_progress) { std::cerr << "Parsing segments" << std::endl; } // Determine if we need translation. bool translate = false; size_t max_node_length = (parameters.max_node_length == 0 ? std::numeric_limits::max() : parameters.max_node_length); if(gfa_file.max_segment_length > max_node_length) { translate = true; if(parameters.show_progress) { std::cerr << "Breaking segments into " << max_node_length << " bp nodes" << std::endl; } } else if(gfa_file.translate_segment_ids) { translate = true; if(parameters.show_progress) { std::cerr << "Translating segment ids into valid node ids" << std::endl; } } std::unique_ptr result(new NaiveGraph()); gfa_file.for_each_segment([&](const std::string& name, std::string_view sequence) { if(translate) { result->translate_segment(name, sequence, max_node_length); } else { nid_t id = stoul_unsafe(name); result->create_node(id, sequence); } }); if(parameters.show_progress) { double seconds = gbwt::readTimer() - start; std::cerr << "Parsed " << result->get_node_count() << " nodes in " << seconds << " seconds" << std::endl; } return result; } GFAGrammar parse_grammar_rules(const GFAFile& gfa_file, const NaiveGraph& graph, const GFAParsingParameters& parameters) { double start = gbwt::readTimer(); if(parameters.show_progress) { std::cerr << "Parsing grammar rules" << std::endl; } GFAGrammar result; std::string rule_name; GFAGrammar::expansion_type expansion; gfa_file.for_each_grammar_rule ( [&](const std::string& name) { rule_name = name; }, [&](const std::string& symbol, bool is_reverse) { expansion.emplace_back(symbol, is_reverse); }, [&]() { bool success = result.insert(std::move(rule_name), std::move(expansion)); if(!success) { throw std::runtime_error("Duplicate grammar rule " + rule_name); } expansion.clear(); // The insertion should have moved the expansion. } ); result.validate(graph); // Throws on error. if(parameters.show_progress) { double seconds = gbwt::readTimer() - start; std::cerr << "Parsed " << result.size() << " rules in " << seconds << " seconds" << std::endl; } return result; } // NaiveGraph construction may throw std::runtime_error on error. void parse_links(const GFAFile& gfa_file, NaiveGraph& graph, const GFAParsingParameters& parameters) { double start = gbwt::readTimer(); if(parameters.show_progress) { std::cerr << "Parsing links" << std::endl; } size_t edge_count = 0; gfa_file.for_each_link([&](const std::string& from, bool from_is_reverse, const std::string& to, bool to_is_reverse) { std::pair from_nodes = graph.translate(from); if(from_nodes == NaiveGraph::no_translation()) { throw std::runtime_error("Invalid source segment " + from); } std::pair to_nodes = graph.translate(to); if(to_nodes == NaiveGraph::no_translation()) { throw std::runtime_error("Invalid destination segment " + to); } nid_t from_node = (from_is_reverse ? from_nodes.first : from_nodes.second - 1); nid_t to_node = (to_is_reverse ? to_nodes.second - 1 : to_nodes.first); graph.create_edge(gbwt::Node::encode(from_node, from_is_reverse), gbwt::Node::encode(to_node, to_is_reverse)); edge_count++; }); // Add edges inside segments if necessary. if(graph.uses_translation()) { for(auto iter = graph.translation_begin(); iter != graph.translation_end(); ++iter) { for(nid_t id = iter->second.first; id + 1 < iter->second.second; id++) { graph.create_edge(gbwt::Node::encode(id, false), gbwt::Node::encode(id + 1, false)); edge_count++; } } } // Get rid of duplicate edges if the GFA graph had them. graph.remove_duplicate_edges(); if(parameters.show_progress) { double seconds = gbwt::readTimer() - start; std::cerr << "Parsed " << edge_count << " edges in " << seconds << " seconds" << std::endl; } } std::unordered_map parse_header_tags(const GFAFile& gfa_file, const GFAParsingParameters& parameters) { double start = gbwt::readTimer(); if(parameters.show_progress) { std::cerr << "Parsing GFA header tags" << std::endl; } // This will map from GBWT tag name to tag value std::unordered_map result; // This happens just once, no jobs. gfa_file.for_each_header_tag([&](const std::string& name, char type, std::string_view value) { if(name == REFERENCE_SAMPLE_LIST_GFA_TAG) { // We need this list of reference samples to be part of the GBWT too. if(type != 'Z') { // It's not the type we want. Bail out. // TODO: Maybe tolerate other people using this tag name? throw std::runtime_error("Expected GFA header tag " + name + " to have type Z, not type " + std::string(1, type)); } // Convert value to set to value to get the canonical order for the reference samples. auto ref_samples = parse_reference_samples_tag(value); std::string canonical_value = compose_reference_samples_tag(ref_samples); result[REFERENCE_SAMPLE_LIST_GBWT_TAG] = canonical_value; } }); if(parameters.show_progress) { double seconds = gbwt::readTimer() - start; std::cerr << "Parsed header tags in " << seconds << " seconds" << std::endl; } return result; } gbwt::Metadata parse_metadata(const GFAFile& gfa_file, const std::vector& jobs, MetadataBuilder& metadata, const GFAParsingParameters& parameters) { double start = gbwt::readTimer(); if(parameters.show_progress) { std::cerr << "Parsing metadata" << std::endl; } for(size_t i = 0; i < jobs.size(); i++) { // Parse paths. gfa_file.for_these_path_names(jobs[i].p_lines, [&](const std::string& name) { metadata.add_path(name, jobs[i].id); }); // Parse walks. gfa_file.for_these_walk_names(jobs[i].w_lines, [&](const std::string& sample, const std::string& haplotype, const std::string& contig, const std::string& start) { metadata.add_walk(sample, haplotype, contig, start, jobs[i].id); }); } gbwt::Metadata result = metadata.get_metadata(); if(parameters.show_progress) { double seconds = gbwt::readTimer() - start; std::cerr << "Metadata: "; gbwt::operator<<(std::cerr, result) << std::endl; std::cerr << "Parsed metadata in " << seconds << " seconds" << std::endl; } return result; } std::unique_ptr parse_paths(const GFAFile& gfa_file, const std::vector& jobs, const NaiveGraph& graph, const GFAGrammar& grammar, const GFAParsingParameters& parameters, gbwt::size_type node_width, gbwt::size_type batch_size) { double start = gbwt::readTimer(); if(parameters.show_progress) { std::cerr << "Indexing paths/walks" << std::endl; } // Prepare for GBWT construction. gbwt::Verbosity::set(gbwt::Verbosity::SILENT); size_t parallel_jobs = std::max(parameters.parallel_jobs, size_t(1)); omp_set_num_threads(parallel_jobs); std::vector partial_indexes(jobs.size()); std::vector current_paths(parallel_jobs); auto add_segment = [&](const std::string& name, bool is_reverse) { std::pair range = graph.translate(name); if(range == NaiveGraph::no_translation()) { throw std::runtime_error("Invalid segment " + name); } gbwt::vector_type& current_path = current_paths[omp_get_thread_num()]; if(is_reverse) { for(nid_t id = range.second; id > range.first; id--) { current_path.push_back(gbwt::Node::encode(id - 1, is_reverse)); } } else { for(nid_t id = range.first; id < range.second; id++) { current_path.push_back(gbwt::Node::encode(id, is_reverse)); } } }; auto add_expansion = [&](const std::string& name, bool is_reverse) { GFAGrammarIterator iter = grammar.iter(name, is_reverse); if(iter.empty()) { add_segment(name, is_reverse); } else { for(auto segment = iter.next(); !segment.first.empty(); segment = iter.next()) { add_segment(std::string(segment.first), segment.second); } } }; // Build the partial indexes in parallel. #pragma omp parallel for schedule(dynamic, 1) for(size_t i = 0; i < jobs.size(); i++) { double job_start = gbwt::readTimer(); if(parameters.show_progress) { #pragma omp critical { std::cerr << "Starting job " << i << " (" << jobs[i].num_nodes << " nodes, " << jobs[i].p_lines.size() << " paths, " << jobs[i].w_lines.size() << " walks)" << std::endl; } } gbwt::GBWTBuilder builder(node_width, batch_size, parameters.sample_interval); size_t thread_num = omp_get_thread_num(); try { gfa_file.for_these_paths(jobs[i].p_lines, [&](const std::string&) {}, add_segment, [&]() { builder.insert(current_paths[thread_num], true); current_paths[thread_num].clear(); }); gfa_file.for_these_walks(jobs[i].w_lines, [&](const std::string&, const std::string&, const std::string&, const std::string&) {}, add_expansion, [&]() { builder.insert(current_paths[thread_num], true); current_paths[thread_num].clear(); }); } catch(const std::runtime_error& e) { std::cerr << "Error: " << e.what() << std::endl; std::exit(EXIT_FAILURE); } builder.finish(); // Order the partial indexes by original ids (minimum node ids) to make the construction deterministic. partial_indexes[jobs[i].id] = gbwt::GBWT(builder.index); // Deleting a dynamic GBWT is a bit expensive, so we do it manually to include it in the measured time. builder.index = gbwt::DynamicGBWT(); if(parameters.show_progress) { double seconds = gbwt::readTimer() - job_start; #pragma omp critical { std::cerr << "Finished job " << i << " in " << seconds << " seconds" << std::endl; } } } // Merge the indexes. if(parameters.show_progress) { std::cerr << "Merging partial indexes" << std::endl; } std::unique_ptr result(new gbwt::GBWT(partial_indexes)); if(parameters.show_progress) { double seconds = gbwt::readTimer() - start; std::cerr << "Indexed " << gfa_file.paths() << " paths and " << gfa_file.walks() << " walks in " << seconds << " seconds" << std::endl; } return result; } //------------------------------------------------------------------------------ std::vector determine_jobs(const GFAFile& gfa_file, const NaiveGraph& graph, const GFAGrammar& grammar, const GFAParsingParameters& parameters) { double start = gbwt::readTimer(); if(parameters.show_progress) { std::cerr << "Creating jobs" << std::endl; } // Determine the jobs. size_t target_size = graph.get_node_count() / std::max(size_t(1), parameters.approximate_num_jobs); ConstructionJobs jobs = gbwt_construction_jobs(graph, target_size); std::vector result; for(size_t i = 0; i < jobs.size(); i++) { result.push_back({ jobs.job_size(i), i, {}, {} }); } // Assign P-lines, W-lines, and Z-lines to jobs. gfa_file.for_each_path_start([&](const char* line_start, const std::string& first_segment) { nid_t node_id = graph.translate(first_segment).first; // 0 on failure. size_t job_id = jobs.job(node_id); if(job_id < jobs.size()) { result[job_id].p_lines.push_back(line_start); } else { throw std::runtime_error("Invalid path segment " + first_segment); } }); gfa_file.for_each_walk_start([&](const char* line_start, const std::string& first_symbol) { // Use the grammar to determine if the symbol is a rule or a segment. std::string first_segment = grammar.first_segment(first_symbol); nid_t node_id = graph.translate(first_segment).first; // 0 on failure. size_t job_id = jobs.job(node_id); if(job_id < jobs.size()) { result[job_id].w_lines.push_back(line_start); } else { throw std::runtime_error("Invalid walk segment " + first_segment); } }); // Sort the jobs to process largest ones first. std::sort(result.begin(), result.end()); // Delete temporary structures before reporting time, as some structures are a bit complex. size_t num_components = jobs.components(); jobs.clear(); if(parameters.show_progress) { double seconds = gbwt::readTimer() - start; std::cerr << "Created " << result.size() << " jobs for " << num_components << " components in " << seconds << " seconds" << std::endl; } return result; } //------------------------------------------------------------------------------ std::pair, std::unique_ptr> gfa_to_gbwt(const std::string& gfa_filename, const GFAParsingParameters& parameters) { // Metadata handling. MetadataBuilder metadata; for(auto& format : parameters.path_name_formats) { metadata.add_path_name_format(format.regex, format.fields, format.sense); } // GFA parsing. GFAFile gfa_file(gfa_filename, parameters.show_progress); check_gfa_file(gfa_file, parameters); // Adjust batch size by GFA size and maximum path length. gbwt::size_type batch_size = determine_batch_size(gfa_file, parameters); // Parse segments and determine node width for buffers. std::unique_ptr graph = parse_segments(gfa_file, parameters); gbwt::size_type node_width = sdsl::bits::length(gbwt::Node::encode(graph->max_node_id(), true)); // Parse grammar rules. GFAGrammar grammar = parse_grammar_rules(gfa_file, *graph, parameters); // Add possible GraphName data from the headers to NaiveGraph. GraphName name(gfa_file.h_lines); graph->set_graph_name(name); // Parse links and create jobs. parse_links(gfa_file, *graph, parameters); std::vector jobs = determine_jobs(gfa_file, *graph, grammar, parameters); // Build the GBWT index. gbwt::Metadata final_metadata = parse_metadata(gfa_file, jobs, metadata, parameters); std::unique_ptr gbwt_index = parse_paths(gfa_file, jobs, *graph, grammar, parameters, node_width, batch_size); gbwt_index->addMetadata(); gbwt_index->metadata = final_metadata; // We can't quite parse the GFA tags to a gbwt::Tags because we would mess up // the source tagging. for(auto& kv : parse_header_tags(gfa_file, parameters)) { // Apply all the tags to our GBWT ourselves. gbwt_index->tags.set(kv.first, kv.second); } return std::make_pair(std::move(gbwt_index), std::move(graph)); } //------------------------------------------------------------------------------ std::string GFAExtractionParameters::mode_name(path_mode mode) { switch(mode) { case mode_default: return "default"; case mode_pan_sn: return "pan-sn"; case mode_ref_only: return "ref-only"; } return "unknown"; // This should not happen. } GFAExtractionParameters::path_mode GFAExtractionParameters::get_mode(const std::string& name) { if(name == "default") { return mode_default; } if(name == "pan-sn") { return mode_pan_sn; } if(name == "ref-only") { return mode_ref_only; } throw std::runtime_error(std::string("Invalid path extraction mode: ") + name); } //------------------------------------------------------------------------------ // Cache segment names and lengths (in nodes). Assume that segment names are short enough // that small string optimization avoids unnecessary memory allocations. struct SegmentCache { SegmentCache(const GBWTGraph& graph, bool use_translation) : graph(graph), segments((graph.index->sigma() > graph.index->firstNode() ? (graph.index->sigma() - graph.index->firstNode()) / 2 : 0)) { if(use_translation) { // We assume that the translation exists if we are told to use it. graph.for_each_segment([&](const std::string& name, std::pair nodes) -> bool { size_t relative = (gbwt::Node::encode(nodes.first, false) - graph.index->firstNode()) / 2; size_t length = nodes.second - nodes.first; for(size_t i = relative; i < relative + length; i++) { this->segments[i] = std::pair(this->names.size(), length); } this->names.emplace_back(name); return true; }); } else { graph.for_each_handle([&](const handle_t& handle) { size_t relative = (GBWTGraph::handle_to_node(handle) - graph.index->firstNode()) / 2; this->segments[relative] = std::pair(this->names.size(), 1); this->names.emplace_back(std::to_string(graph.get_id(handle))); }); } } size_t size() const { return this->names.size(); } std::pair get(const handle_t& handle) const { return this->get(GBWTGraph::handle_to_node(handle)); } std::pair get(gbwt::node_type node) const { size_t relative = (node - this->graph.index->firstNode()) / 2; size_t offset = this->segments[relative].first; return std::make_pair(std::string_view(this->names[offset]), this->segments[relative].second); } const GBWTGraph& graph; // This vector goes over the same range as `graph.real_nodes`. The first component // is offset in `names` and the second is the length of the segment in nodes. std::vector> segments; std::vector names; }; //------------------------------------------------------------------------------ void write_segments(const GBWTGraph& graph, const SegmentCache& cache, TSVWriter& writer, bool show_progress) { double start = gbwt::readTimer(); size_t segments = 0; if(show_progress) { std::cerr << "Writing segments" << std::endl; } std::string_view prev(nullptr, 0); graph.for_each_handle([&](const handle_t& handle) { auto segment = cache.get(handle); if(segment.first != prev) { if(!prev.empty()) { writer.newline(); } prev = segment.first; writer.put('S'); writer.newfield(); writer.write(segment.first); writer.newfield(); segments++; } writer.write(graph.get_sequence_view(handle)); }); if(!prev.empty()) { writer.newline(); } if(show_progress) { double seconds = gbwt::readTimer() - start; std::cerr << "Wrote " << segments << " segments in " << seconds << " seconds" << std::endl; } } void write_links(const GBWTGraph& graph, const SegmentCache& cache, std::ostream& out, const GFAExtractionParameters& parameters) { double start = gbwt::readTimer(); if(parameters.show_progress) { std::cerr << "Writing links" << std::endl; } size_t threads = parameters.threads(); std::vector writers(threads, ManualTSVWriter(out)); std::vector links(threads, 0); if(parameters.use_translation && graph.has_segment_names()) { graph.for_each_link([&](const edge_t& edge, const std::string& from, const std::string& to) -> bool { size_t thread = omp_get_thread_num(); ManualTSVWriter& writer = writers[thread]; writer.put('L'); writer.newfield(); writer.write(from); writer.newfield(); writer.put((graph.get_is_reverse(edge.first) ? '-' : '+')); writer.newfield(); writer.write(to); writer.newfield(); writer.put((graph.get_is_reverse(edge.second) ? '-' : '+')); writer.newfield(); writer.write("0M"); writer.newline(); links[thread]++; if(writer.full()) { #pragma omp critical { writer.flush(); } } return true; }, (threads > 1)); } else { graph.for_each_edge([&](const edge_t& edge) { size_t thread = omp_get_thread_num(); ManualTSVWriter& writer = writers[thread]; writer.put('L'); writer.newfield(); writer.write(cache.get(edge.first).first); writer.newfield(); writer.put((graph.get_is_reverse(edge.first) ? '-' : '+')); writer.newfield(); writer.write(cache.get(edge.second).first); writer.newfield(); writer.put((graph.get_is_reverse(edge.second) ? '-' : '+')); writer.newfield(); writer.write("0M"); writer.newline(); links[thread]++; if(writer.full()) { #pragma omp critical { writer.flush(); } } }, (threads > 1)); } // Flush the writers and count the total number of links. size_t total_links = 0; for(size_t i = 0; i < threads; i++) { writers[i].flush(); total_links += links[i]; } if(parameters.show_progress) { double seconds = gbwt::readTimer() - start; std::cerr << "Wrote " << total_links << " links in " << seconds << " seconds" << std::endl; } } // Write one path in panSN P line format (including optional bracketed offset) and synthesizing phase field if empty. // Use the given sample name instead of computing it ourselves. void write_pan_sn_path(const gbwt::GBWT& index, const SegmentCache& segment_cache, const LargeRecordCache& record_cache, ManualTSVWriter& writer, gbwt::size_type path_id, const std::string& sample_name) { const gbwt::PathName& path_name = index.metadata.path(path_id); writer.put('P'); writer.newfield(); writer.write(sample_name); writer.put('#'); writer.write(path_name.phase == GBWTGraph::NO_PHASE ? PathMetadata::NO_HAPLOTYPE : path_name.phase); writer.put('#'); if(index.metadata.hasContigNames()) { writer.write(index.metadata.contig(path_name.contig)); } else { writer.write(path_name.contig); } if(path_name.count != 0) { // TODO: We're going to assume we're a named path and we mean to have an // offset here. If we're actually a haplotype path, vg would expect a // not-quite-panSN '#' separator. writer.put('['); writer.write(path_name.count); writer.put(']'); } writer.newfield(); gbwt::vector_type path = record_cache.extract(gbwt::Path::encode(path_id, false)); size_t offset = 0; while(offset < path.size()) { auto segment = segment_cache.get(path[offset]); writer.write(segment.first); writer.put((gbwt::Node::is_reverse(path[offset]) ? '-' : '+')); offset += segment.second; if(offset < path.size()) { writer.put(','); } } writer.newfield(); writer.put('*'); writer.newline(); #pragma omp critical { writer.flush(); } } void write_generic_paths(const GBWTGraph& graph, const SegmentCache& segment_cache, const LargeRecordCache& record_cache, std::ostream& out, gbwt::size_type ref_sample, const GFAExtractionParameters& parameters) { double start = gbwt::readTimer(); if(parameters.show_progress) { std::cerr << "Writing generic paths" << std::endl; } std::atomic paths(0); const gbwt::GBWT& index = *(graph.index); { std::vector generic_paths = index.metadata.pathsForSample(ref_sample); std::vector writers(parameters.threads(), ManualTSVWriter(out)); // Some compilers default to older versions of OpenMP that do not support range-based for loops. #pragma omp parallel for schedule(dynamic, 1) for(size_t i = 0; i < generic_paths.size(); i++) { gbwt::size_type path_id = generic_paths[i]; ManualTSVWriter& writer = writers[omp_get_thread_num()]; const gbwt::PathName& path_name = index.metadata.path(path_id); gbwt::vector_type path = record_cache.extract(gbwt::Path::encode(path_id, false)); writer.put('P'); writer.newfield(); writer.write(index.metadata.contig(path_name.contig)); if(path_name.count != 0) { // We have a subrange on this generic path writer.put('['); writer.write(path_name.count); writer.put(']'); } writer.newfield(); size_t offset = 0; while(offset < path.size()) { auto segment = segment_cache.get(path[offset]); writer.write(segment.first); writer.put((gbwt::Node::is_reverse(path[offset]) ? '-' : '+')); offset += segment.second; if(offset < path.size()) { writer.put(','); } } writer.newfield(); writer.put('*'); writer.newline(); #pragma omp critical { writer.flush(); } } paths += generic_paths.size(); } // We don't write reference paths as P lines, just generic paths. if(parameters.show_progress && paths > 0) { double seconds = gbwt::readTimer() - start; std::cerr << "Wrote " << paths << " paths in " << seconds << " seconds" << std::endl; } } void write_pan_sn(const GBWTGraph& graph, const SegmentCache& segment_cache, const LargeRecordCache& record_cache, std::ostream& out, const GFAExtractionParameters& parameters) { double start = gbwt::readTimer(); if(parameters.show_progress) { std::cerr << "Writing PanSN paths" << std::endl; } const gbwt::GBWT& index = *(graph.index); for(gbwt::size_type path_id = 0; path_id < index.metadata.paths(); path_id++) { const gbwt::PathName& path_name = index.metadata.path(path_id); if(path_name.count > 0) { std::cerr << "Warning: Fragment/count field is in use, there may be duplicate path names" << std::endl; break; } } std::vector writers(parameters.threads(), ManualTSVWriter(out)); // Some compilers default to older versions of OpenMP that do not support range-based for loops. #pragma omp parallel for schedule(dynamic, 1) for(gbwt::size_type path_id = 0; path_id < index.metadata.paths(); path_id++) { ManualTSVWriter& writer = writers[omp_get_thread_num()]; const gbwt::PathName& path_name = index.metadata.path(path_id); std::string sample_name; if(index.metadata.hasSampleNames()) { sample_name = index.metadata.sample(path_name.sample); } else { sample_name = std::to_string(path_name.sample); } write_pan_sn_path(index, segment_cache, record_cache, writer, path_id, sample_name); } if(parameters.show_progress && index.metadata.paths() > 0) { double seconds = gbwt::readTimer() - start; std::cerr << "Wrote " << index.metadata.paths() << " paths in " << seconds << " seconds" << std::endl; } } // Writes the given path as a GFA walk to the writer but does not flush the writer. void write_walk(const GBWTGraph& graph, const SegmentCache& segment_cache, const LargeRecordCache& record_cache, ManualTSVWriter& writer, gbwt::size_type path_id) { const gbwt::PathName& path_name = graph.index->metadata.path(path_id); gbwt::vector_type path = record_cache.extract(gbwt::Path::encode(path_id, false)); size_t length = 0; for(auto node : path) { length += graph.get_length(GBWTGraph::node_to_handle(node)); } writer.put('W'); writer.newfield(); if(graph.index->metadata.hasSampleNames()) { writer.write(graph.index->metadata.sample(path_name.sample)); } else { writer.write(path_name.sample); } writer.newfield(); writer.write(path_name.phase == GBWTGraph::NO_PHASE ? PathMetadata::NO_HAPLOTYPE : path_name.phase); writer.newfield(); if(graph.index->metadata.hasContigNames()) { writer.write(graph.index->metadata.contig(path_name.contig)); } else { writer.write(path_name.contig); } writer.newfield(); writer.write(path_name.count); writer.newfield(); writer.write(path_name.count + length); writer.newfield(); size_t offset = 0; while(offset < path.size()) { auto segment = segment_cache.get(path[offset]); writer.put((gbwt::Node::is_reverse(path[offset]) ? '<' : '>')); writer.write(segment.first); offset += segment.second; } writer.newline(); } // Write haplotype paths (including reference paths) as walks. void write_walks(const GBWTGraph& graph, const SegmentCache& segment_cache, const LargeRecordCache& record_cache, std::ostream& out, gbwt::size_type ref_sample, const GFAExtractionParameters& parameters) { double start = gbwt::readTimer(); std::atomic walks(0); if(parameters.show_progress) { std::cerr << "Writing walks" << std::endl; } const gbwt::GBWT& index = *(graph.index); std::vector writers(parameters.threads(), ManualTSVWriter(out)); #pragma omp parallel for schedule(dynamic, 1) for(gbwt::size_type path_id = 0; path_id < index.metadata.paths(); path_id++) { const gbwt::PathName& path_name = index.metadata.path(path_id); if(path_name.sample == ref_sample) { continue; } walks++; write_walk(graph, segment_cache, record_cache, writers[omp_get_thread_num()], path_id); #pragma omp critical { writers[omp_get_thread_num()].flush(); } } if(parameters.show_progress && walks > 0) { double seconds = gbwt::readTimer() - start; std::cerr << "Wrote " << walks << " walks in " << seconds << " seconds" << std::endl; } } // Write reference paths as walks. void write_ref_walks(const GBWTGraph& graph, const SegmentCache& segment_cache, const LargeRecordCache& record_cache, std::ostream& out, const GFAExtractionParameters& parameters) { double start = gbwt::readTimer(); std::atomic walks(0); if(parameters.show_progress) { std::cerr << "Writing reference walks" << std::endl; } // Figure out sample ids for reference paths. std::string ref_samples_tag = graph.index->tags.get(REFERENCE_SAMPLE_LIST_GBWT_TAG); sample_name_set ref_samples = parse_reference_samples_tag(ref_samples_tag); std::set ref_sample_ids; for(const std::string& ref_sample : ref_samples) { ref_sample_ids.insert(graph.index->metadata.sample(ref_sample)); } const gbwt::GBWT& index = *(graph.index); std::vector writers(parameters.threads(), ManualTSVWriter(out)); #pragma omp parallel for schedule(dynamic, 1) for(gbwt::size_type path_id = 0; path_id < index.metadata.paths(); path_id++) { const gbwt::PathName& path_name = index.metadata.path(path_id); if(ref_sample_ids.find(path_name.sample) == ref_sample_ids.end()) { continue; } walks++; write_walk(graph, segment_cache, record_cache, writers[omp_get_thread_num()], path_id); #pragma omp critical { writers[omp_get_thread_num()].flush(); } } if(parameters.show_progress && walks > 0) { double seconds = gbwt::readTimer() - start; std::cerr << "Wrote " << walks << " walks in " << seconds << " seconds" << std::endl; } } void write_all_paths(const GBWTGraph& graph, const SegmentCache& segment_cache, const LargeRecordCache& record_cache, std::ostream& out, const GFAExtractionParameters& parameters) { double start = gbwt::readTimer(); if(parameters.show_progress) { std::cerr << "Writing paths" << std::endl; } const gbwt::GBWT& index = *(graph.index); std::vector writers(parameters.threads(), ManualTSVWriter(out)); #pragma omp parallel for schedule(dynamic, 1) for(gbwt::size_type seq_id = 0; seq_id < index.sequences(); seq_id += 2) { ManualTSVWriter& writer = writers[omp_get_thread_num()]; gbwt::size_type path_id = seq_id / 2; gbwt::vector_type path = record_cache.extract(seq_id); writer.put('P'); writer.newfield(); writer.write(path_id); writer.newfield(); size_t offset = 0; while(offset < path.size()) { auto segment = segment_cache.get(path[offset]); writer.write(segment.first); writer.put((gbwt::Node::is_reverse(path[offset]) ? '-' : '+')); offset += segment.second; if(offset < path.size()) { writer.put(','); } } writer.newfield(); writer.put('*'); writer.newline(); #pragma omp critical { writer.flush(); } } if(parameters.show_progress) { double seconds = gbwt::readTimer() - start; std::cerr << "Wrote " << (index.sequences() / 2) << " paths in " << seconds << " seconds" << std::endl; } } //------------------------------------------------------------------------------ void gbwt_to_gfa(const GBZ& gbz, std::ostream& out, const GFAExtractionParameters& parameters) { const GBWTGraph& graph = gbz.graph; bool sufficient_metadata = graph.index->hasMetadata() && graph.index->metadata.hasPathNames(); // Cache segment names. double start = gbwt::readTimer(); if(parameters.show_progress) { std::cerr << "Caching segments" << std::endl; } bool use_translation = parameters.use_translation & graph.has_segment_names(); SegmentCache segment_cache(graph, use_translation); if(parameters.show_progress) { double seconds = gbwt::readTimer() - start; std::cerr << "Cached " << segment_cache.size() << " segments in " << seconds << " seconds" << std::endl; } // Cache large GBWT records. start = gbwt::readTimer(); if(parameters.show_progress) { std::cerr << "Caching large GBWT records" << std::endl; } LargeRecordCache record_cache(*(graph.index), parameters.large_record_bytes); if(parameters.show_progress) { double seconds = gbwt::readTimer() - start; std::cerr << "Cached " << record_cache.size() << " GBWT records larger than " << parameters.large_record_bytes << " bytes in " << seconds << " seconds" << std::endl; } // Cache and write the header and the segments using a single writer. TSVWriter writer(out); writer.put('H'); writer.newfield(); writer.write(std::string("VN:Z:1.1")); if(graph.index->tags.contains(REFERENCE_SAMPLE_LIST_GBWT_TAG)) { // We need to save the reference sample list writer.newfield(); writer.write(REFERENCE_SAMPLE_LIST_GFA_TAG); writer.put(':'); writer.put('Z'); writer.put(':'); writer.write(graph.index->tags.get(REFERENCE_SAMPLE_LIST_GBWT_TAG)); } writer.newline(); if(!use_translation) { // If translation is in use, we may output the translation target or its subgraph. GraphName graph_name = gbz.graph_name(); std::vector header_lines = graph_name.gfa_header_lines(); for(const std::string& line : header_lines) { writer.write(line); writer.newline(); } } write_segments(graph, segment_cache, writer, parameters.show_progress); writer.flush(); // Write the links and paths using multiple threads. int old_threads = omp_get_max_threads(); omp_set_num_threads(parameters.threads()); write_links(graph, segment_cache, out, parameters); if(sufficient_metadata) { gbwt::size_type generic_ref_sample = graph.index->metadata.sample(GENERIC_PATH_SAMPLE_NAME); switch(parameters.mode) { case GFAExtractionParameters::mode_default: write_generic_paths(graph, segment_cache, record_cache, out, generic_ref_sample, parameters); write_walks(graph, segment_cache, record_cache, out, generic_ref_sample, parameters); break; case GFAExtractionParameters::mode_pan_sn: write_pan_sn(graph, segment_cache, record_cache, out, parameters); break; case GFAExtractionParameters::mode_ref_only: write_generic_paths(graph, segment_cache, record_cache, out, generic_ref_sample, parameters); write_ref_walks(graph, segment_cache, record_cache, out, parameters); break; } } else { std::cerr << "Warning: No metadata available, writing all paths as P-lines" << std::endl; write_all_paths(graph, segment_cache, record_cache, out, parameters); } // Restore the number of threads. omp_set_num_threads(old_threads); } //------------------------------------------------------------------------------ void canonical_edges_from(const GBWTGraph& graph, handle_t from_handle, TSVWriter& writer) { // Because we iterate over the right (exit side) edges of from_handle, // they are sorted by (to_id, to_is_reverse). gbwt::node_type from = GBWTGraph::handle_to_node(from_handle); graph.follow_edges(from_handle, false, [&](const handle_t& to_handle) { gbwt::node_type to = GBWTGraph::handle_to_node(to_handle); if(edge_is_canonical(from, to)) { writer.put('L'); writer.newfield(); writer.write(std::to_string(graph.get_id(from_handle))); writer.newfield(); writer.put((gbwt::Node::is_reverse(from) ? '-' : '+')); writer.newfield(); writer.write(std::to_string(graph.get_id(to_handle))); writer.newfield(); writer.put((gbwt::Node::is_reverse(to) ? '-' : '+')); writer.newline(); } }); } void gbwt_to_canonical_gfa(const GBWTGraph& graph, std::ostream& out) { TSVWriter writer(out); // Cache segment names. SegmentCache segment_cache(graph, false); // Single-threaded for_each_handle visits the nodes in order. graph.for_each_handle([&](const handle_t& handle) { writer.put('S'); writer.newfield(); writer.write(segment_cache.get(handle).first); writer.newfield(); writer.write(graph.get_sequence_view(handle)); writer.newline(); canonical_edges_from(graph, handle, writer); canonical_edges_from(graph, graph.flip(handle), writer); }); writer.flush(); } //------------------------------------------------------------------------------ } // namespace gbwtgraph jltsiren-gbwtgraph-e27bc43/src/gfa2gbwt.cpp000066400000000000000000000466161524136466400207700ustar00rootroot00000000000000#include #include #include #include #include #include #include #include using namespace gbwtgraph; //------------------------------------------------------------------------------ enum input_type { input_gfa, input_gbz, input_graph }; enum output_type { output_gfa, output_gbz, output_graph, output_none }; struct Config { Config(int argc, char** argv); GFAParsingParameters parameters; GFAExtractionParameters output_parameters; std::string basename; input_type input = input_gfa; output_type output = output_gbz; std::uint32_t gbz_version = 0; // Default version. bool translation = false; bool show_progress = false; bool simple_sds_graph = false; bool bitvectors = false; }; const std::string tool_name = "GFA to GBWTGraph"; void parse_gfa(GBZ& gbz, const Config& config); // May throw `std::runtime_error`. void load_gbz(GBZ& gbz, const Config& config); void load_graph(GBZ& gbz, const Config& config); void write_gfa(const GBZ& gbz, const Config& config); void write_gbz(const GBZ& gbz, const Config& config); void write_graph(const GBZ& gbz, const Config& config); void extract_translation(const GBZ& gbz, const Config& config); void extract_bitvectors(const GBZ& gbz, const Config& config); //------------------------------------------------------------------------------ int main(int argc, char** argv) { double start = gbwt::readTimer(); Config config(argc, argv); // Initial output. if(config.show_progress) { Version::print(std::cerr, tool_name); gbwt::printHeader("Base name", std::cerr) << config.basename << std::endl; if(config.input == input_gfa) { gbwt::printHeader("--approx-jobs", std::cerr) << config.parameters.approximate_num_jobs << std::endl; gbwt::printHeader("--parallel-jobs", std::cerr) << config.parameters.parallel_jobs << std::endl; gbwt::printHeader("--max-node", std::cerr) << config.parameters.max_node_length << std::endl; gbwt::printHeader("--path-regex", std::cerr) << config.parameters.path_name_formats.front().regex << std::endl; gbwt::printHeader("--path-fields", std::cerr) << config.parameters.path_name_formats.front().fields << std::endl; gbwt::printHeader("--path-sense", std::cerr) << (int)config.parameters.path_name_formats.front().sense << std::endl; } if(config.output == output_gfa) { gbwt::printHeader("--parallel-jobs", std::cerr) << config.output_parameters.num_threads << std::endl; gbwt::printHeader("--cache-records", std::cerr) << config.output_parameters.large_record_bytes << std::endl; gbwt::printHeader("--paths", std::cerr) << GFAExtractionParameters::mode_name(config.output_parameters.mode) << std::endl; if(!(config.output_parameters.use_translation)) { std::cerr << "--no-translation" << std::endl; } } std::cerr << std::endl; } // This is the data we are using. GBZ gbz; std::unordered_map> translation; try { // Handle the input. if(config.input == input_gfa) { parse_gfa(gbz, config); } else if(config.input == input_gbz) { load_gbz(gbz, config); } else if(config.input == input_graph) { load_graph(gbz, config); } // Handle the output. if(config.output == output_gfa) { write_gfa(gbz, config); } else if(config.output == output_gbz) { write_gbz(gbz, config); } else if(config.output == output_graph) { write_graph(gbz, config); } // Extract parts of the GBZ. if(config.translation) { extract_translation(gbz, config); } if(config.bitvectors) { extract_bitvectors(gbz, config); } } catch(const std::exception& e) { std::cerr << "Error: " << e.what() << std::endl; std::exit(EXIT_FAILURE); } if(config.show_progress) { std::cerr << std::endl; if(config.input == input_gfa) { gbwt::printStatistics(gbz.index, config.basename, std::cerr); } double seconds = gbwt::readTimer() - start; std::cerr << "Used " << seconds << " seconds, " << gbwt::inGigabytes(gbwt::memoryUsage()) << " GiB" << std::endl; std::cerr << std::endl; } return 0; } //------------------------------------------------------------------------------ void printUsage(int exit_code) { Version::print(std::cerr, tool_name); std::cerr << "Usage: gfa2gbwt [mode] [options] basename" << std::endl; std::cerr << std::endl; std::cerr << "Modes:" << std::endl; std::cerr << " -c, --compress-gfa read " << GFA_EXTENSION << ", write " << GBZ::EXTENSION << " (default)" << std::endl; std::cerr << " -d, --decompress-gfa read " << GBZ::EXTENSION << ", write " << GFA_EXTENSION << std::endl; std::cerr << " -b, --build-graph read " << GFA_EXTENSION << ", write " << gbwt::GBWT::EXTENSION << " and " << GBWTGraph::EXTENSION << std::endl; std::cerr << " -e, --extract-gfa read " << gbwt::GBWT::EXTENSION << " and " << GBWTGraph::EXTENSION << ", write " << GFA_EXTENSION << std::endl; std::cerr << " -C, --compress-graph read " << gbwt::GBWT::EXTENSION << " and " << GBWTGraph::EXTENSION << ", write " << GBZ::EXTENSION << std::endl; std::cerr << " -D, --decompress-graph read " << GBZ::EXTENSION << ", write " << gbwt::GBWT::EXTENSION << " and " << GBWTGraph::EXTENSION << std::endl; std::cerr << " --rewrite-gbz read " << GBZ::EXTENSION << ", write " << GBZ::EXTENSION << std::endl; std::cerr << " --gbz-version N write GBZ version N instead of the default version" << std::endl; std::cerr << std::endl; std::cerr << "General options:" << std::endl; std::cerr << " -p, --progress show progress information" << std::endl; std::cerr << " -t, --translation write translation table into a " << NaiveGraph::TRANSLATION_EXTENSION << " file" << std::endl; std::cerr << std::endl; std::cerr << "Parallel options:" << std::endl; std::cerr << " -j, --approx-jobs N create approximately N GBWT construction jobs (default " << GFAParsingParameters::APPROXIMATE_NUM_JOBS << ")" << std::endl; std::cerr << " -P, --parallel-jobs N run N construction / extraction jobs in parallel (default 1)" << std::endl; std::cerr << std::endl; std::cerr << "Output options:" << std::endl; std::cerr << " -R, --cache-records N cache > N-byte GBWT records for " << GFA_EXTENSION << " output (default " << GFAExtractionParameters::LARGE_RECORD_BYTES << ")" << std::endl; std::cerr << " -s, --simple-sds-graph serialize " << GBWTGraph::EXTENSION << " in simple-sds format instead of libhandlegraph format" << std::endl; std::cerr << " (this tool cannot read simple-sds graphs)" << std::endl; std::cerr << " --paths STR extract paths as STR (default, pan-sn, ref-only)" << std::endl; std::cerr << " --pan-sn extract paths as P-lines with PanSN names" << std::endl; std::cerr << " --ref-only extract generic paths as P-lines and reference paths as W-lines" << std::endl; std::cerr << " --no-translation do not use the node-to-segment translation" << std::endl; std::cerr << std::endl; std::cerr << "GFA parsing parameters:" << std::endl; std::cerr << " -m, --max-node N break > N bp segments into multiple nodes (default " << MAX_NODE_LENGTH << ")" << std::endl; std::cerr << " (minimizer index requires nodes of length <= 1024 bp)" << std::endl; std::cerr << " -r, --path-regex STR parse path names using regex STR (default " << GFAParsingParameters::DEFAULT_REGEX << ")" << std::endl; std::cerr << " -f, --path-fields STR map the submatches to fields STR (default " << GFAParsingParameters::DEFAULT_FIELDS << ")" << std::endl; std::cerr << " (the first submatch is the entire path name)" << std::endl; std::cerr << " --path-sense INT assign paths the sense INT (default " << (int)GFAParsingParameters::DEFAULT_SENSE << ")" << std::endl; std::cerr << " --pan-sn parse PanSN path names (sets --path-regex, --path-fields, and --path-sense)" << std::endl; std::cerr << std::endl; std::cerr << "Fields (case insensitive):" << std::endl; std::cerr << " S sample name" << std::endl; std::cerr << " C contig name" << std::endl; std::cerr << " H numerical haplotype identifier" << std::endl; std::cerr << " F numerical fragment identifier" << std::endl; std::cerr << " other ignore this field" << std::endl; std::cerr << std::endl; std::exit(exit_code); } //------------------------------------------------------------------------------ Config::Config(int argc, char** argv) { if(argc < 2) { printUsage(EXIT_SUCCESS); } constexpr int OPT_REWRITE_GBZ = 1000; constexpr int OPT_GBZ_VERSION = 1001; constexpr int OPT_PATHS = 1100; constexpr int OPT_PAN_SN = 1101; constexpr int OPT_REF_ONLY = 1102; constexpr int OPT_NO_TRANSLATION = 1103; constexpr int OPT_PATH_SENSE = 1200; // Data for `getopt_long()`. int c = 0, option_index = 0; option long_options[] = { { "compress-gfa", no_argument, 0, 'c' }, { "decompress-gfa", no_argument, 0, 'd' }, { "build-graph", no_argument, 0, 'b' }, { "extract-gfa", no_argument, 0, 'e' }, { "compress-graph", no_argument, 0, 'C' }, { "decompress-graph", no_argument, 0, 'D' }, { "rewrite-gbz", no_argument, 0, OPT_REWRITE_GBZ }, { "gbz-version", required_argument, 0, OPT_GBZ_VERSION }, { "load-gbz", no_argument, 0, 'l' }, // Hidden. { "bitvectors", no_argument, 0, 'B' }, // Hidden. { "progress", no_argument, 0, 'p' }, { "translation", no_argument, 0, 't' }, { "approx-jobs", required_argument, 0, 'j' }, { "parallel-jobs", required_argument, 0, 'P' }, { "cache-records", required_argument, 0, 'R' }, { "simple-sds-graph", no_argument, 0, 's' }, { "paths", required_argument, 0, OPT_PATHS }, { "pan-sn", no_argument, 0, OPT_PAN_SN }, { "ref-only", no_argument, 0, OPT_REF_ONLY }, { "no-translation", no_argument, 0, OPT_NO_TRANSLATION }, { "max-node", required_argument, 0, 'm' }, { "path-regex", required_argument, 0, 'r' }, { "path-fields", required_argument, 0, 'f' }, { "path-sense", required_argument, 0, OPT_PATH_SENSE }, { 0, 0, 0, 0 } }; // Process options. while((c = getopt_long(argc, argv, "cdbeCDlBptj:P:R:sm:r:f:", long_options, &option_index)) != -1) { switch(c) { case 'c': this->input = input_gfa; this->output = output_gbz; break; case 'd': this->input = input_gbz; this->output = output_gfa; break; case 'b': this->input = input_gfa; this->output = output_graph; break; case 'e': this->input = input_graph; this->output = output_gfa; break; case 'C': this->input = input_graph; this->output = output_gbz; break; case 'D': this->input = input_gbz; this->output = output_graph; break; case OPT_REWRITE_GBZ: this->input = input_gbz; this->output = output_gbz; break; case OPT_GBZ_VERSION: try { this->gbz_version = std::stoul(optarg); } catch(const std::invalid_argument&) { std::cerr << "gfa2gbwt: Invalid GBZ version: " << optarg << std::endl; std::exit(EXIT_FAILURE); } break; case 'l': this->input = input_gbz; this->output = output_none; break; case 'B': this->bitvectors = true; break; case 'p': this->show_progress = true; this->parameters.show_progress = true; this->output_parameters.show_progress = true; break; case 't': this->translation = true; break; case 'j': try { this->parameters.approximate_num_jobs = std::stoul(optarg); } catch(const std::invalid_argument&) { std::cerr << "gfa2gbwt: Invalid number of jobs: " << optarg << std::endl; std::exit(EXIT_FAILURE); } break; case 'P': try { this->parameters.parallel_jobs = std::stoul(optarg); } catch(const std::invalid_argument&) { std::cerr << "gfa2gbwt: Invalid number of parallel jobs: " << optarg << std::endl; std::exit(EXIT_FAILURE); } this->output_parameters.num_threads = this->parameters.parallel_jobs; break; case 'R': try { this->output_parameters.large_record_bytes = std::stoul(optarg); } catch(const std::invalid_argument&) { std::cerr << "gfa2gbwt: Invalid record caching threshold: " << optarg << std::endl; std::exit(EXIT_FAILURE); } break; case 's': this->simple_sds_graph = true; break; case OPT_PATHS: try { this->output_parameters.mode = GFAExtractionParameters::get_mode(optarg); } catch(const std::exception& e) { std::cerr << "Error: " << e.what() << std::endl; std::exit(EXIT_FAILURE); } break; case OPT_PAN_SN: this->output_parameters.mode = GFAExtractionParameters::mode_pan_sn; this->parameters.path_name_formats.clear(); this->parameters.path_name_formats.emplace_back( GFAParsingParameters::PAN_SN_REGEX, GFAParsingParameters::PAN_SN_FIELDS, GFAParsingParameters::PAN_SN_SENSE ); break; case OPT_REF_ONLY: this->output_parameters.mode = GFAExtractionParameters::mode_ref_only; break; case OPT_NO_TRANSLATION: this->output_parameters.use_translation = false; break; case 'm': try { this->parameters.max_node_length = std::stoul(optarg); } catch(const std::invalid_argument&) { std::cerr << "gfa2gbwt: Invalid maximum node length: " << optarg << std::endl; std::exit(EXIT_FAILURE); } break; case 'r': this->parameters.path_name_formats.front().regex = optarg; break; case 'f': this->parameters.path_name_formats.front().fields = optarg; break; case OPT_PATH_SENSE: // TODO: Use by-name sense parsing when available in libhandlegraph try { this->parameters.path_name_formats.front().sense = (PathSense) std::stoul(optarg); } catch(const std::invalid_argument&) { std::cerr << "gfa2gbwt: Invalid path sense: " << optarg << std::endl; std::exit(EXIT_FAILURE); } if (this->parameters.path_name_formats.front().sense > PathSense::HAPLOTYPE) { std::cerr << "gfa2gbwt: Invalid path sense: " << optarg << std::endl; std::exit(EXIT_FAILURE); } break; case '?': std::exit(EXIT_FAILURE); default: std::exit(EXIT_FAILURE); } } // Sanity checks. if(optind >= argc) { printUsage(EXIT_FAILURE); } this->basename = argv[optind]; optind++; if(this->gbz_version != 0 && (this->gbz_version < GBZ::Header::MIN_SERIALIZE_VERSION || this->gbz_version > GBZ::Header::VERSION)) { std::cerr << "gfa2gbwt: Unsupported GBZ version: " << this->gbz_version << std::endl; std::exit(EXIT_FAILURE); } } //------------------------------------------------------------------------------ void parse_gfa(GBZ& gbz, const Config& config) { std::string gfa_name = config.basename + GFA_EXTENSION; if(config.show_progress) { std::cerr << "Parsing GFA from " << gfa_name << " and building GBWT" << std::endl; } // This may throw an exception. auto result = gfa_to_gbwt(gfa_name, config.parameters); if(config.show_progress) { std::cerr << "Building GBZ" << std::endl; } gbz = GBZ(result.first, result.second); } void load_gbz(GBZ& gbz, const Config& config) { std::string gbz_name = config.basename + GBZ::EXTENSION; if(config.show_progress) { std::cerr << "Decompressing GBWT and GBWTGraph from " << gbz_name << std::endl; } sdsl::simple_sds::load_from(gbz, gbz_name); } void load_graph(GBZ& gbz, const Config& config) { std::string gbwt_name = config.basename + gbwt::GBWT::EXTENSION; std::string graph_name = config.basename + GBWTGraph::EXTENSION; if(config.show_progress) { std::cerr << "Loading GBWT and GBWTGraph from " << gbwt_name << " and " << graph_name << std::endl; } gbz.load_from_files(gbwt_name, graph_name); } //------------------------------------------------------------------------------ void write_gfa(const GBZ& gbz, const Config& config) { std::string gfa_name = config.basename + GFA_EXTENSION; if(config.show_progress) { std::cerr << "Writing the GFA to " << gfa_name << std::endl; } std::ofstream out; out.exceptions(std::ofstream::failbit | std::ofstream::badbit); out.open(gfa_name, std::ios_base::binary); gbwt_to_gfa(gbz, out, config.output_parameters); out.close(); } void write_gbz(const GBZ& gbz, const Config& config) { std::string gbz_name = config.basename + GBZ::EXTENSION; if(config.show_progress) { std::cerr << "Compressing GBWT and GBWTGraph to " << gbz_name; if(config.gbz_version != 0) { std::cerr << " (version " << config.gbz_version << ")"; } std::cerr << std::endl; } if(config.gbz_version != 0) { sdsl::simple_sds::serialize_to(gbz, gbz_name, config.gbz_version); } else { sdsl::simple_sds::serialize_to(gbz, gbz_name); } } void write_graph(const GBZ& gbz, const Config& config) { std::string gbwt_name = config.basename + gbwt::GBWT::EXTENSION; std::string graph_name = config.basename + GBWTGraph::EXTENSION; if(config.show_progress) { std::cerr << "Writing GBWT and GBWTGraph to " << gbwt_name << " and " << graph_name << std::endl; } gbz.serialize_to_files(gbwt_name, graph_name, config.simple_sds_graph); } //------------------------------------------------------------------------------ void extract_translation(const GBZ& gbz, const Config& config) { std::string translation_name = config.basename + NaiveGraph::TRANSLATION_EXTENSION; if(config.show_progress) { std::cerr << "Writing the translation table to " << translation_name << std::endl; } std::ofstream out(translation_name, std::ios_base::binary); TSVWriter writer(out); gbz.graph.for_each_segment([&](const std::string& name, std::pair nodes) -> bool { writer.put('T'); writer.newfield(); writer.write(name); writer.newfield(); writer.write(nodes.first); for(nid_t i = nodes.first + 1; i < nodes.second; i++) { writer.put(','); writer.write(i); } writer.newline(); return true; }); } sdsl::bit_vector convert(const sdsl::sd_vector<>& source) { sdsl::bit_vector result(source.size(), 0); for(auto iter = source.one_begin(); iter != source.one_end(); ++iter) { result[iter->second] = 1; } return result; } sdsl::bit_vector convert(const sdsl::int_vector<0>& source, size_t n) { // The sequences are stored in both orientations, so we only take every other value and divide by two. sdsl::bit_vector result(n / 2, 0); for(size_t i = 0; i < source.size(); i += 2) { result[source[i] / 2] = 1; } return result; } void extract_bitvectors(const GBZ& gbz, const Config& config) { std::string bv_gbwt = config.basename + ".bv_gbwt"; std::string bv_seq = config.basename + ".bv_seq"; if(config.show_progress) { std::cerr << "Writing bitvectors to " << bv_gbwt << " and " << bv_seq << std::endl; } sdsl::bit_vector bv = convert(gbz.index.bwt.index); sdsl::simple_sds::serialize_to(bv, bv_gbwt); bv = convert(gbz.graph.sequences.index, gbz.graph.sequences.length() + 1); sdsl::simple_sds::serialize_to(bv, bv_seq); } //------------------------------------------------------------------------------ jltsiren-gbwtgraph-e27bc43/src/index.cpp000066400000000000000000000467571524136466400204020ustar00rootroot00000000000000#include #include #include #include #include #include namespace gbwtgraph { //------------------------------------------------------------------------------ const std::string PATH_NAME_FIELDS_TAG = "path_name_fields"; // We need this because PathIdMap is not meant to be part of the library // interface. const size_t MAX_PATH_IDS = PathIdMap::MAX_HAPLOTYPES; //------------------------------------------------------------------------------ template void index_haplotypes ( const GBZ& gbz, MinimizerIndex& index, const std::function& get_payload ) { using minimizer_type = typename MinimizerIndex::minimizer_type; using code_type = KmerEncoding::code_type; using value_type = KmerEncoding::value_type; struct alignas(CACHE_LINE_SIZE) Cache { std::vector> data; }; // Take the graph reference for convenience. // Also copy graph name and relationships from GBZ tags to MinimizerIndex tags. const GBWTGraph& graph = gbz.graph; GraphName graph_name = gbz.graph_name(); index.set_graph_name(graph_name); // Minimizer caching. We only generate the payloads after we have removed duplicate positions. int threads = omp_get_max_threads(); constexpr size_t MINIMIZER_CACHE_SIZE = 1024; std::vector cache(threads); auto flush_cache = [&](int thread_id) { auto& current_cache = cache[thread_id].data; gbwt::removeDuplicates(current_cache, false); std::vector payloads; size_t payload_size = index.payload_size(); payloads.reserve(current_cache.size() * payload_size); for (auto& minimizer : current_cache) { const code_type* payload = get_payload(minimizer.second); for(size_t i = 0; i < payload_size; i++) { payloads.push_back(payload[i]); } } #pragma omp critical (minimizer_index) { for(size_t i = 0, payload_offset = 0; i < current_cache.size(); i++, payload_offset += payload_size) { value_type value(Position(current_cache[i].second), &payloads[payload_offset]); index.insert(current_cache[i].first, value); } } current_cache.clear(); }; // Minimizer finding. auto find_minimizers = [&](const std::vector& traversal, const std::string& seq) { std::vector minimizers = index.minimizers(seq); // Calls syncmers() when appropriate. auto iter = traversal.begin(); size_t node_start = 0; int thread_id = omp_get_thread_num(); for(minimizer_type& minimizer : minimizers) { if(minimizer.empty()) { continue; } // Find the node covering minimizer starting position. size_t node_length = graph.get_length(*iter); while(node_start + node_length <= minimizer.offset) { node_start += node_length; ++iter; node_length = graph.get_length(*iter); } pos_t pos { graph.get_id(*iter), graph.get_is_reverse(*iter), minimizer.offset - node_start }; if(minimizer.is_reverse) { pos = reverse_base_pos(pos, node_length); } if(!Position::valid_offset(pos)) { #pragma omp critical (cerr) { std::cerr << "index_haplotypes(): Node offset " << offset(pos) << " is too large" << std::endl; } std::exit(EXIT_FAILURE); } cache[thread_id].data.emplace_back(minimizer, pos); } if(cache[thread_id].data.size() >= MINIMIZER_CACHE_SIZE) { flush_cache(thread_id); } }; // Traverse all haplotype windows for_each_haplotype_window(graph, index.window_bp(), find_minimizers, (threads > 1)); for(int thread_id = 0; thread_id < threads; thread_id++) { flush_cache(thread_id); } } // Instantiate the template, as we did not define the function in the header. template void index_haplotypes ( const GBZ& gbz, MinimizerIndex& index, const std::function& get_payload ); template void index_haplotypes ( const GBZ& gbz, MinimizerIndex& index, const std::function& get_payload ); //------------------------------------------------------------------------------ template void index_haplotypes_with_paths ( const GBZ& gbz, MinimizerIndex& index, const std::function& get_payload ) { if(index.payload_size() == 0) { std::string msg = "index_haplotypes_with_paths(): Payload size must be greater than zero"; throw std::runtime_error(msg); } using minimizer_type = typename MinimizerIndex::minimizer_type; using code_type = KmerEncoding::code_type; using value_type = KmerEncoding::value_type; // This version needs the path corresponding to the minimizer. struct alignas(CACHE_LINE_SIZE) Cache { std::vector>> data; }; // Take the graph reference for convenience. // Also copy graph name and relationships from GBZ tags to MinimizerIndex tags. const GBWTGraph& graph = gbz.graph; GraphName graph_name = gbz.graph_name(); index.set_graph_name(graph_name); int threads = omp_get_max_threads(); constexpr size_t MINIMIZER_CACHE_SIZE = 1024; std::vector cache(threads); const gbwt::Metadata& metadata = graph.index->metadata; PathIdMap path_ids_map(metadata); index.set_tag(PATH_NAME_FIELDS_TAG, PathIdMap::key_type_str(path_ids_map.key_type())); // Build the r-index once. Locate queries against the per-thread state_cache // miss frequently (the cache is local to each flush_cache invocation), so the // O(occ) FastLocate amortises far below the cost of CachedGBWT::locate's // walk-back through the BWT. // start timer auto start_time = std::chrono::high_resolution_clock::now(); std::cerr << "Building r-index for locate queries..." << std::endl; gbwt::FastLocate r_index(*graph.index); auto end_time = std::chrono::high_resolution_clock::now(); auto duration = std::chrono::duration_cast(end_time - start_time).count(); std::cerr << "Done building r-index in " << duration << " seconds" << std::endl; // Minimizer caching. We only generate the payloads after we have removed duplicate positions. auto flush_cache = [&](int thread_id) { // Haplotypes for search states we have already seen. std::unordered_map state_cache; auto cached_gbwt = graph.get_cache(); auto& current_cache = cache[thread_id].data; gbwt::removeDuplicates(current_cache, false); // Determine the payloads. std::vector payloads; size_t payload_size = index.payload_size(); payloads.reserve(current_cache.size() * payload_size); for(auto& minimizer : current_cache) { // Fill in the payload except for the last word. const code_type* payload = get_payload(std::get<1>(minimizer)); for(size_t i = 0; i < payload_size - 1; i++) { payloads.push_back(payload[i]); } // Determine the haplotypes that contain the minimizer. auto& traversed_nodes = std::get<2>(minimizer); gbwt::SearchState state = cached_gbwt.find(traversed_nodes.begin(), traversed_nodes.end()); auto iter = state_cache.find(state); if(iter != state_cache.end()) { payloads.push_back(iter->second); } else { code_type haps = 0; for(gbwt::size_type seq_id : r_index.locate(state)) { const gbwt::PathName& path = metadata.path(gbwt::Path::id(seq_id)); auto id = path_ids_map.id(path); haps |= (code_type(1) << id); } payloads.push_back(haps); state_cache[state] = haps; } } // And now flush the cache. We use insert_with_last_payload_word_or so that // collisions on (key, Position) coming from different kmer-paths (typically // at node boundaries with alternative successors sharing their first k-1 bp) // OR-merge the haplotype bitmap stored in the last payload word, instead of // silently dropping the second inserter's bits. #pragma omp critical (minimizer_index) { for(size_t i = 0, payload_offset = 0; i < current_cache.size(); i++, payload_offset += payload_size) { value_type value(Position(std::get<1>(current_cache[i])), &payloads[payload_offset]); index.insert_with_last_payload_word_or(std::get<0>(current_cache[i]), value); } } current_cache.clear(); }; // Minimizer finding logic. auto find_minimizers = [&](const std::vector& traversal, const std::string& seq) { std::vector minimizers = index.minimizers(seq); // Calls syncmers() when appropriate. auto iter = traversal.begin(); size_t path_offset = 0; size_t node_start = 0; int thread_id = omp_get_thread_num(); for(minimizer_type& minimizer : minimizers) { if (minimizer.empty()) { continue; } // Find the node covering minimizer starting position and the path corresponding to the minimizer. size_t node_length = graph.get_length(*iter); while(node_start + node_length <= minimizer.offset) { node_start += node_length; ++iter; path_offset++; node_length = graph.get_length(*iter); } pos_t pos { graph.get_id(*iter), graph.get_is_reverse(*iter), minimizer.offset - node_start }; std::vector traversed_nodes = extract_kmer_path(graph, traversal, path_offset, offset(pos), index.k(), minimizer.is_reverse); if(minimizer.is_reverse) { pos = reverse_base_pos(pos, node_length); } if(!Position::valid_offset(pos)) { #pragma omp critical (cerr) { std::cerr << "index_haplotypes(): Node offset " << offset(pos) << " is too large" << std::endl; } std::exit(EXIT_FAILURE); } cache[thread_id].data.emplace_back(minimizer, pos, traversed_nodes); } if(cache[thread_id].data.size() >= MINIMIZER_CACHE_SIZE) { flush_cache(thread_id); } }; // Traverse all haplotype windows for_each_haplotype_window(graph, index.window_bp(), find_minimizers, (threads > 1)); for(int thread_id = 0; thread_id < threads; thread_id++) { flush_cache(thread_id); } } // Instantiate the template, as we did not define the function in the header. template void index_haplotypes_with_paths ( const GBZ& gbz, MinimizerIndex& index, const std::function& get_payload ); template void index_haplotypes_with_paths ( const GBZ& gbz, MinimizerIndex& index, const std::function& get_payload ); //------------------------------------------------------------------------------ template std::vector> canonical_kmers(std::string::const_iterator begin, std::string::const_iterator end, size_t k) { std::vector> result; if(k == 0 || k > KeyType::KMER_MAX_LENGTH) { std::cerr << "canonical_kmers(): k must be between 1 and " << KeyType::KMER_MAX_LENGTH << std::endl; return result; } size_t valid_chars = 0, offset = 0; KeyType forward_key, reverse_key; std::string::const_iterator iter = begin; while(iter != end) { forward_key.forward(k, *iter, valid_chars); reverse_key.reverse(k, *iter); if(valid_chars >= k) { if(forward_key < reverse_key) { result.push_back({ forward_key, forward_key.hash(), offset_type(offset - (k - 1)), false }); } else { result.push_back({ reverse_key, reverse_key.hash(), offset_type(offset), true }); } } ++iter; offset++; } std::sort(result.begin(), result.end()); return result; } // Instantiate the template, as we did not define the function in the header. template std::vector> canonical_kmers(std::string::const_iterator begin, std::string::const_iterator end, size_t k); template std::vector> canonical_kmers(std::string::const_iterator begin, std::string::const_iterator end, size_t k); //------------------------------------------------------------------------------ template void build_kmer_index(const GBWTGraph& graph, KmerIndex& index, size_t k, const std::function& include) { using kmer_type = Kmer; using code_type = KmerEncoding::code_type; using value_type = KmerEncoding::value_type; struct alignas(CACHE_LINE_SIZE) Cache { std::vector> data; }; // Kmer caching. int threads = omp_get_max_threads(); std::vector cache(threads); constexpr size_t KMER_CACHE_SIZE = 1024; std::vector null_payload(index.payload_size(), 0); auto flush_cache = [&](int thread_id) { auto& current_cache = cache[thread_id].data; gbwt::removeDuplicates(current_cache, false); #pragma omp critical (kmer_index) { for(auto& kmer : current_cache) { value_type value(Position(kmer.second), null_payload.data()); index.insert(kmer.first.key, value, kmer.first.hash); } } current_cache.clear(); }; // Kmer finding. auto find_kmers = [&](const std::vector& traversal, const std::string& seq) { std::vector kmers = canonical_kmers(seq, k); auto iter = traversal.begin(); size_t node_start = 0; int thread_id = omp_get_thread_num(); for(auto kmer : kmers) { if(kmer.empty()) { continue; } // Find the node covering kmer starting position. size_t node_length = graph.get_length(*iter); while(node_start + node_length <= kmer.offset) { node_start += node_length; ++iter; node_length = graph.get_length(*iter); } pos_t pos { graph.get_id(*iter), graph.get_is_reverse(*iter), kmer.offset - node_start }; if(kmer.is_reverse) { pos = reverse_base_pos(pos, node_length); } if(include(kmer.key)) { cache[thread_id].data.emplace_back(kmer, Position(pos)); } } if(cache[thread_id].data.size() >= KMER_CACHE_SIZE) { flush_cache(thread_id); } }; // Count all kmers. for_each_haplotype_window(graph, k, find_kmers, (threads > 1)); for(int thread_id = 0; thread_id < threads; thread_id++) { flush_cache(thread_id); } } // Instantiate the template, as we did not define the function in the header. template void build_kmer_index(const GBWTGraph& graph, KmerIndex& index, size_t k, const std::function& include); template void build_kmer_index(const GBWTGraph& graph, KmerIndex& index, size_t k, const std::function& include); //------------------------------------------------------------------------------ template void build_kmer_indexes(const GBWTGraph& graph, std::array, 4>& indexes, size_t k) { size_t payload_size = indexes[0].payload_size(); for(size_t i = 1; i < indexes.size(); i++) { if(indexes[i].payload_size() != payload_size) { throw std::runtime_error("build_kmer_indexes(): All indexes must have the same payload size"); } } using kmer_type = Kmer; using code_type = KmerEncoding::code_type; using value_type = KmerEncoding::value_type; struct alignas(CACHE_LINE_SIZE) Cache { std::array>, 4> data; }; // Kmer caching. int threads = omp_get_max_threads(); std::array mutexes; std::vector cache(threads); constexpr size_t KMER_CACHE_SIZE = 1024; std::vector null_payload(payload_size, 0); auto flush_cache = [&](int thread_id, size_t index) { auto& current_cache = cache[thread_id].data[index]; gbwt::removeDuplicates(current_cache, false); { std::lock_guard lock(mutexes[index]); for(auto& kmer : current_cache) { value_type value(Position(kmer.second), null_payload.data()); indexes[index].insert(kmer.first.key, value, kmer.first.hash); } } current_cache.clear(); }; // Kmer finding. auto find_kmers = [&](const std::vector& traversal, const std::string& seq) { std::vector kmers = canonical_kmers(seq, k); auto iter = traversal.begin(); size_t node_start = 0; int thread_id = omp_get_thread_num(); for(auto kmer : kmers) { if(kmer.empty()) { continue; } // Find the node covering kmer starting position. size_t node_length = graph.get_length(*iter); while(node_start + node_length <= kmer.offset) { node_start += node_length; ++iter; node_length = graph.get_length(*iter); } pos_t pos { graph.get_id(*iter), graph.get_is_reverse(*iter), kmer.offset - node_start }; if(kmer.is_reverse) { pos = reverse_base_pos(pos, node_length); } // Insert the kmer into the right index. size_t index = kmer.key.access_raw(k, k / 2); if(index < indexes.size()) { cache[thread_id].data[index].emplace_back(kmer, Position(pos)); } } for(size_t i = 0; i < indexes.size(); i++) { if(cache[thread_id].data[i].size() >= KMER_CACHE_SIZE) { flush_cache(thread_id, i); } } }; // Count all kmers. for_each_haplotype_window(graph, k, find_kmers, (threads > 1)); for(int thread_id = 0; thread_id < threads; thread_id++) { for(size_t index = 0; index < indexes.size(); index++) { flush_cache(thread_id, index); } } } // Instantiate the template, as we did not define the function in the header. template void build_kmer_indexes(const GBWTGraph& graph, std::array, 4>& indexes, size_t k); template void build_kmer_indexes(const GBWTGraph& graph, std::array, 4>& indexes, size_t k); //------------------------------------------------------------------------------ template std::vector frequent_kmers(const GBWTGraph& graph, size_t k, size_t threshold, bool space_efficient, size_t hash_table_size) { using index_type = KmerIndex; using multi_value_type = KmerEncoding::multi_value_type; std::vector result; auto select_frequent = [&](const index_type& index) { index.for_each_kmer([&](KeyType key, multi_value_type values) { if(threshold == 0 || values.second > threshold) { result.push_back(key); result.back().clear_pointer(); } }); }; if(space_efficient) { for(char base : { 'A', 'C', 'G', 'T' }) { index_type index(hash_table_size, 0); std::function include = [&](KeyType key) -> bool { return (key.access(k, k / 2) == base); }; build_kmer_index(graph, index, k, include); select_frequent(index); } } else { std::array indexes { index_type(hash_table_size, 0), index_type(hash_table_size, 0), index_type(hash_table_size, 0), index_type(hash_table_size, 0), }; build_kmer_indexes(graph, indexes, k); for(auto& index : indexes) { select_frequent(index); } } gbwt::parallelQuickSort(result.begin(), result.end()); return result; } // Instantiate the template, as we did not define the function in the header. template std::vector frequent_kmers(const GBWTGraph& graph, size_t k, size_t threshold, bool space_efficient, size_t hash_table_size); template std::vector frequent_kmers(const GBWTGraph& graph, size_t k, size_t threshold, bool space_efficient, size_t hash_table_size); //------------------------------------------------------------------------------ } // namespace gbwtgraph jltsiren-gbwtgraph-e27bc43/src/internal.cpp000066400000000000000000000250501524136466400210660ustar00rootroot00000000000000#include #include #include #include #include #include namespace gbwtgraph { //------------------------------------------------------------------------------ // Numerical class constants. constexpr size_t TSVWriter::BUFFER_SIZE; constexpr size_t ManualTSVWriter::BUFFER_SIZE; constexpr size_t ManualTSVWriter::BUFFER_FULL; constexpr size_t PathIdMap::MAX_HAPLOTYPES; //------------------------------------------------------------------------------ TSVWriter::TSVWriter(std::ostream& out) : out(out) { this->buffer.reserve(BUFFER_SIZE); } TSVWriter::~TSVWriter() { this->flush(); } void TSVWriter::write(std::string_view view) { size_t offset = 0; while(offset < view.size()) { size_t length = std::min(view.size() - offset, BUFFER_SIZE - this->buffer.size()); this->buffer.insert(this->buffer.end(), view.begin() + offset, view.begin() + offset + length); if(this->buffer.size() >= BUFFER_SIZE) { this->flush(); } offset += length; } } void TSVWriter::flush() { if(!(this->buffer.empty())) { this->out.write(this->buffer.data(), this->buffer.size()); this->buffer.clear(); } } //------------------------------------------------------------------------------ ManualTSVWriter::ManualTSVWriter(std::ostream& out) : out(out) { this->buffer.reserve(BUFFER_SIZE); } void ManualTSVWriter::flush() { if(!(this->buffer.empty())) { this->out.write(this->buffer.data(), this->buffer.size()); this->buffer.clear(); } } //------------------------------------------------------------------------------ std::string GFAGrammar::first_segment(const std::string& symbol) const { std::string result = symbol; for(rule_type rule = this->expand(result); rule != this->no_rule(); rule = this->expand(result)) { if(rule->second.empty()) { break; } result = rule->second.front().first; } return result; } void GFAGrammar::validate(const NaiveGraph& graph) const { // Check the rules individually. for(const auto& rule : this->rules) { if(rule.first.empty()) { throw std::runtime_error("GFAGrammar::validate(): Empty rule name"); } if(graph.has_node_or_segment(rule.first)) { throw std::runtime_error("GFAGrammar::validate(): Rule name " + rule.first + " clashes with a node/segment name"); } if(rule.second.size() < 2) { throw std::runtime_error("GFAGrammar::validate(): Expansion of rule " + rule.first + " is trivial"); } for(const auto& symbol : rule.second) { rule_type expansion = this->expand(symbol.first); if(graph.has_node_or_segment(symbol.first)) { if(expansion != this->no_rule()) { throw std::runtime_error("GFAGrammar::validate(): Symbol " + symbol.first + " in the expansion of rule " + rule.first + " is both a node/segment and a rule"); } } else { if(expansion == this->no_rule()) { throw std::runtime_error("GFAGrammar::validate(): Symbol " + symbol.first + " in the expansion of rule " + rule.first + " is undefined"); } } } } // TODO: Use a stack to avoid stack overflows on unbalanced grammars. // Check for cycles using depth-first search. enum class State { ACTIVE, VISITED }; std::unordered_map states; std::function dfs = [&](const std::string& rule_name) { states[rule_name] = State::ACTIVE; const expansion_type& expansion = this->rules.at(rule_name); for(const auto& symbol : expansion) { if(this->rules.find(symbol.first) == this->rules.end()) { continue; } auto result = states.emplace(symbol.first, State::ACTIVE); if(result.second) { // Unvisited rule. dfs(symbol.first); } else if(result.first->second == State::ACTIVE) { throw std::runtime_error("GFAGrammar::validate(): Cycle detected at rule " + symbol.first); } } states[rule_name] = State::VISITED; }; for(const auto& rule : this->rules) { if(states.find(rule.first) == states.end()) { dfs(rule.first); } } } GFAGrammarIterator GFAGrammar::iter(const std::string& rule_name, bool is_reverse) const { return GFAGrammarIterator(*this, rule_name, is_reverse); } GFAGrammarIterator::GFAGrammarIterator(const GFAGrammar& grammar, const std::string& rule_name, bool is_reverse) : grammar(grammar) { GFAGrammar::rule_type rule = grammar.expand(rule_name); if(rule != grammar.no_rule()) { this->stack.push_back({ rule, is_reverse, 0 }); } } std::pair GFAGrammarIterator::next() { while(!(this->stack.empty())) { auto& top = this->stack.back(); auto& expansion = std::get<0>(top)->second; if(std::get<2>(top) >= expansion.size()) { this->stack.pop_back(); continue; } // Determine the next symbol and its orientation. bool is_reverse = std::get<1>(top); size_t index = (is_reverse ? (expansion.size() - 1 - std::get<2>(top)) : std::get<2>(top)); std::get<2>(top)++; const std::string& symbol = expansion[index].first; is_reverse ^= expansion[index].second; GFAGrammar::rule_type rule = this->grammar.expand(symbol); if(rule != this->grammar.no_rule()) { this->stack.push_back({ rule, is_reverse, 0 }); } else { return std::make_pair(std::string_view(symbol), is_reverse); } } return std::make_pair(std::string_view(), false); } //------------------------------------------------------------------------------ LargeRecordCache::LargeRecordCache(const gbwt::GBWT& index, size_t bytes) : index(index) { for(gbwt::node_type node = this->index.firstNode(); node < this->index.sigma(); node++) { std::pair range = this->index.bwt.getRange(this->index.toComp(node)); if(range.second - range.first > bytes && !(this->index.empty(node))) { this->cache[node] = gbwt::DecompressedRecord(this->index.record(node)); } } } gbwt::vector_type LargeRecordCache::extract(gbwt::size_type sequence) const { gbwt::vector_type result; if(sequence > this->sequences()) { return result; } gbwt::edge_type pos = this->index.start(sequence); while(pos.first != gbwt::ENDMARKER) { result.push_back(pos.first); auto iter = this->cache.find(pos.first); if(iter != this->cache.end()) { pos = iter->second.LF(pos.second); } else { pos = this->index.LF(pos); } } return result; } //------------------------------------------------------------------------------ std::vector> sample_path_positions(const GBZ& gbz, path_handle_t path, size_t sample_interval, size_t* length) { std::vector> result; gbwt::size_type seq_id = gbwt::Path::encode(gbz.graph.handle_to_path(path), false); size_t offset = 0, next_sample = 0; for(gbwt::edge_type pos = gbz.index.start(seq_id); pos.first != gbwt::ENDMARKER; pos = gbz.index.LF(pos)) { if(offset >= next_sample) { result.push_back({ offset, pos }); next_sample = offset + sample_interval; } offset += gbz.graph.get_length(GBWTGraph::node_to_handle(pos.first)); } if(length != nullptr) { *length = offset; } return result; } //------------------------------------------------------------------------------ PathIdMap::PathIdMap(const gbwt::Metadata& metadata) : mask ( std::numeric_limits::max(), std::numeric_limits::max(), std::numeric_limits::max(), 0 ), key(KeyType::SAMPLE_CONTIG_HAPLOTYPE) { // Sort the paths by (sample, phase, contig, count) instead of the natural order. // This is because we fall back from (sample, haplotype, contig) to (sample, haplotype). std::vector sorted_paths; sorted_paths.reserve(metadata.paths()); for(gbwt::size_type i = 0; i < metadata.paths(); i++) { sorted_paths.push_back(metadata.path(i)); } std::sort(sorted_paths.begin(), sorted_paths.end(), [](const gbwt::PathName& left, const gbwt::PathName& right) { if(left.sample != right.sample) { return (left.sample < right.sample); } if(left.phase != right.phase) { return (left.phase < right.phase); } if(left.contig != right.contig) { return (left.contig < right.contig); } return (left.count < right.count); }); // First attempt: (sample, contig, haplotype). if(this->build_map(sorted_paths)) { return; } // Second attempt: (sample, haplotype). this->mask.contig = 0; this->key = KeyType::SAMPLE_HAPLOTYPE; if(this->build_map(sorted_paths)) { return; } // Third attempt: (sample). Falls back to an empty map on failure. this->mask.phase = 0; this->key = KeyType::SAMPLE; this->build_map(sorted_paths); } bool PathIdMap::build_map(const std::vector& sorted_paths) { size_t next = 0; for(const gbwt::PathName& path : sorted_paths) { gbwt::PathName key = this->mask_name(path); if(this->path_to_id.find(key) == this->path_to_id.end()) { if(next >= MAX_HAPLOTYPES) { this->path_to_id.clear(); this->key = KeyType::NONE; return false; } this->path_to_id[key] = next; next++; } } return true; } std::string PathIdMap::key_type_str(KeyType key) { switch(key) { case KeyType::NONE: return "()"; case KeyType::SAMPLE: return "(sample)"; case KeyType::SAMPLE_HAPLOTYPE: return "(sample, haplotype)"; case KeyType::SAMPLE_CONTIG_HAPLOTYPE: return "(sample, contig, haplotype)"; default: return "(unknown)"; } } //------------------------------------------------------------------------------ std::vector extract_kmer_path(const GBWTGraph& graph, const std::vector& path, size_t path_offset, size_t node_offset, size_t k, bool is_reverse) { if(is_reverse) { node_offset = graph.get_length(path[path_offset]) - node_offset - 1; } std::vector result; size_t path_length = 0; while(path_length < k && path_offset < path.size()) { handle_t handle = path[path_offset]; path_length += graph.get_length(handle) - node_offset; node_offset = 0; gbwt::node_type node = GBWTGraph::handle_to_node(handle); if(is_reverse) { result.push_back(gbwt::Node::reverse(node)); if(path_offset == 0) { break; } path_offset--; } else { result.push_back(node); path_offset++; } } return result; } //------------------------------------------------------------------------------ } // namespace gbwtgraph jltsiren-gbwtgraph-e27bc43/src/kmer_freq.cpp000066400000000000000000000204341524136466400212260ustar00rootroot00000000000000#include #include #include #include #include #include #include #include using namespace gbwtgraph; //------------------------------------------------------------------------------ const std::string tool_name = "Kmer Frequencies"; struct Config { Config(int argc, char** argv); constexpr static size_t DEFAULT_K = 29; size_t k = DEFAULT_K; bool distribution = false; size_t threshold = 0; size_t threads = 1; size_t hash_table_size = KmerIndex::INITIAL_CAPACITY; bool space_efficient = false; bool verbose = true; // TODO: Do we need an option to silence this? std::string filename; }; void print_distribution(const std::map& distribution, const std::string& header1, const std::string& header2); //------------------------------------------------------------------------------ int main(int argc, char** argv) { typedef KmerIndex index_type; double start = gbwt::readTimer(); Config config(argc, argv); omp_set_num_threads(config.threads); // Load the graph. GBZ gbz; if(config.verbose) { std::cerr << "Loading GBZ from " << config.filename << std::endl; } sdsl::simple_sds::load_from(gbz, config.filename); // (number of hits, number of kmers) std::map distribution; auto update_distribution = [&](const index_type& index) { index.for_each_kmer([&](index_type::key_type, index_type::multi_value_type values) { distribution[values.second]++; }); }; // Build the kmer indexes and extract the kmer frequency distribution. double checkpoint = gbwt::readTimer(); size_t total_kmers = 0, total_unique = 0, total_values = 0; if(config.space_efficient) { for(char base : { 'A', 'C', 'G', 'T' }) { if(config.verbose) { std::cerr << "Building a " << config.k << "-mer index for middle base " << base << std::endl; } index_type index(config.hash_table_size); std::function include = [&](Key64 key) -> bool { return (key.access(config.k, config.k / 2) == base); }; build_kmer_index(gbz.graph, index, config.k, include); if(config.verbose) { std::cerr << index.size() << " kmers (" << index.unique_keys() << " unique) with " << index.number_of_values() << " hits" << std::endl; } total_kmers += index.size(); total_unique += index.unique_keys(); total_values += index.number_of_values(); update_distribution(index); } } else { if(config.verbose) { std::cerr << "Building all " << config.k << "-mer indexes in parallel" << std::endl; } std::array indexes { index_type(config.hash_table_size), index_type(config.hash_table_size), index_type(config.hash_table_size), index_type(config.hash_table_size) }; build_kmer_indexes(gbz.graph, indexes, config.k); for(auto& index : indexes) { total_kmers += index.size(); total_unique += index.unique_keys(); total_values += index.number_of_values(); update_distribution(index); } } if(config.verbose) { double seconds = gbwt::readTimer() - checkpoint; std::cerr << "Built the kmer indexes in " << seconds << " seconds" << std::endl; std::cerr << total_kmers << " kmers (" << total_unique << " unique) with " << total_values << " hits" << std::endl; } if(config.threshold > 0) { size_t count = 0; for(auto iter = distribution.begin(); iter != distribution.end(); ++iter) { if(iter->first > config.threshold) { count += iter->second; } } std::cout << count << std::endl; } if(config.distribution) { print_distribution(distribution, "Frequency", "Kmers"); } // Deleting the hash table may take some time, so we delete it before returning final usage. if(config.verbose) { double seconds = gbwt::readTimer() - start; std::cerr << "Used " << seconds << " seconds, " << gbwt::inGigabytes(gbwt::memoryUsage()) << " GiB" << std::endl; std::cerr << std::endl; } return 0; } //------------------------------------------------------------------------------ void printUsage(int exit_code) { Version::print(std::cerr, tool_name); std::cerr << "Usage: kmer_freq [options] graph.gbz" << std::endl; std::cerr << std::endl; std::cerr << "Options:" << std::endl; std::cerr << " -d, --distribution print kmer frequency distribution" << std::endl; std::cerr << " -f, --frequent N count the number of kmers with frequency > N" << std::endl; std::cerr << " -k, --kmer-length N count N-mers (default: " << Config::DEFAULT_K << ")" << std::endl; std::cerr << " -s, --save-memory use the slower space-efficient algorithm" << std::endl; std::cerr << " -t, --threads N use N parallel threads" << std::endl; std::cerr << " -H, --hash-table N initialize the hash table with 2^N cells" << std::endl; std::cerr << std::endl; std::exit(exit_code); } //------------------------------------------------------------------------------ Config::Config(int argc, char** argv) { if(argc < 2) { printUsage(EXIT_SUCCESS); } size_t max_threads = omp_get_max_threads(); size_t min_width = 10; size_t max_width = 36; // Data for `getopt_long()`. int c = 0, option_index = 0; option long_options[] = { { "distribution", no_argument, 0, 'd' }, { "frequent", required_argument, 0, 'f' }, { "kmer-length", required_argument, 0, 'k' }, { "save-memory", no_argument, 0, 's' }, { "threads", required_argument, 0, 't' }, { "hash-tables", required_argument, 0, 'H' }, { 0, 0, 0, 0 } }; // Process options. while((c = getopt_long(argc, argv, "df:k:st:H:", long_options, &option_index)) != -1) { switch(c) { case 'd': this->distribution = true; break; case 'f': try { this->threshold = std::stoul(optarg); } catch(const std::invalid_argument&) { std::cerr << "kmer_freq: Invalid frequency threshold: " << optarg << std::endl; std::exit(EXIT_FAILURE); } break; case 'k': try { this->k = std::stoul(optarg); } catch(const std::invalid_argument&) { std::cerr << "kmer_freq: Invalid kmer length: " << optarg << std::endl; std::exit(EXIT_FAILURE); } if(this->k < 2 || this->k > Key64::KMER_MAX_LENGTH) { std::cerr << "kmer_freq: Invalid kmer length: " << this->k << " (must be from 2 to " << Key64::KMER_MAX_LENGTH << ")" << std::endl; std::exit(EXIT_FAILURE); } break; case 's': this->space_efficient = true; break; case 't': try { this->threads = std::stoul(optarg); } catch(const std::invalid_argument&) { std::cerr << "kmer_freq: Invalid number of threads: " << optarg << std::endl; std::exit(EXIT_FAILURE); } if(this->threads == 0 || this->threads > max_threads) { std::cerr << "kmer_freq: Invalid number of threads: " << this->threads << " (must be from 1 to " << max_threads << ")" << std::endl; std::exit(EXIT_FAILURE); } break; case 'H': size_t width; try { width = std::stoul(optarg); } catch(const std::invalid_argument&) { std::cerr << "kmer_freq: Invalid hash table width: " << optarg << std::endl; std::exit(EXIT_FAILURE); } if(width < min_width || width > max_width) { std::cerr << "kmer_freq: Invalid hash table width: " << width << " (must be from " << min_width << " to " << max_width << ")" << std::endl; std::exit(EXIT_FAILURE); } this->hash_table_size = size_t(1) << width; break; case '?': std::exit(EXIT_FAILURE); default: std::exit(EXIT_FAILURE); } } // Sanity checks. if(optind >= argc) { printUsage(EXIT_FAILURE); } this->filename = argv[optind]; optind++; } //------------------------------------------------------------------------------ void print_distribution(const std::map& distribution, const std::string& header1, const std::string& header2) { std::cout << header1 << "\t" << header2 << std::endl; for(auto iter = distribution.begin(); iter != distribution.end(); ++iter) { std::cout << iter->first << "\t" << iter->second << std::endl; } } //------------------------------------------------------------------------------ jltsiren-gbwtgraph-e27bc43/src/minimizer.cpp000066400000000000000000000421431524136466400212570ustar00rootroot00000000000000#include namespace gbwtgraph { //------------------------------------------------------------------------------ // KmerEncoding: Numerical class constants. constexpr size_t KmerEncoding::FIELD_BITS; constexpr size_t KmerEncoding::PACK_WIDTH; constexpr size_t KmerEncoding::PACK_OVERFLOW; constexpr size_t KmerEncoding::FIELD_CHARS; constexpr KmerEncoding::code_type KmerEncoding::PACK_MASK; // KmerEncoding: Other class variables. const std::vector KmerEncoding::CHAR_TO_PACK = { 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 4, 1, 4, 4, 4, 2, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 4, 1, 4, 4, 4, 2, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4 }; const std::vector KmerEncoding::PACK_TO_CHAR = { 'A', 'C', 'G', 'T' }; const std::vector KmerEncoding::LOW_MASK = { // k = 0 0x0000000000000000ull, // k = 1 to 32 in the low part of the key. 0x0000000000000003ull, 0x000000000000000Full, 0x000000000000003Full, 0x00000000000000FFull, 0x00000000000003FFull, 0x0000000000000FFFull, 0x0000000000003FFFull, 0x000000000000FFFFull, 0x000000000003FFFFull, 0x00000000000FFFFFull, 0x00000000003FFFFFull, 0x0000000000FFFFFFull, 0x0000000003FFFFFFull, 0x000000000FFFFFFFull, 0x000000003FFFFFFFull, 0x00000000FFFFFFFFull, 0x00000003FFFFFFFFull, 0x0000000FFFFFFFFFull, 0x0000003FFFFFFFFFull, 0x000000FFFFFFFFFFull, 0x000003FFFFFFFFFFull, 0x00000FFFFFFFFFFFull, 0x00003FFFFFFFFFFFull, 0x0000FFFFFFFFFFFFull, 0x0003FFFFFFFFFFFFull, 0x000FFFFFFFFFFFFFull, 0x003FFFFFFFFFFFFFull, 0x00FFFFFFFFFFFFFFull, 0x03FFFFFFFFFFFFFFull, 0x0FFFFFFFFFFFFFFFull, 0x3FFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull, // k = 33 to 63 in the high part of the key 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull }; const std::vector KmerEncoding::HIGH_MASK = { // k = 0 0x0000000000000000ull, // k = 1 to 32 in the low part of the key. 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, 0x0000000000000000ull, // k = 33 to 63 in the high part of the key 0x0000000000000003ull, 0x000000000000000Full, 0x000000000000003Full, 0x00000000000000FFull, 0x00000000000003FFull, 0x0000000000000FFFull, 0x0000000000003FFFull, 0x000000000000FFFFull, 0x000000000003FFFFull, 0x00000000000FFFFFull, 0x00000000003FFFFFull, 0x0000000000FFFFFFull, 0x0000000003FFFFFFull, 0x000000000FFFFFFFull, 0x000000003FFFFFFFull, 0x00000000FFFFFFFFull, 0x00000003FFFFFFFFull, 0x0000000FFFFFFFFFull, 0x0000003FFFFFFFFFull, 0x000000FFFFFFFFFFull, 0x000003FFFFFFFFFFull, 0x00000FFFFFFFFFFFull, 0x00003FFFFFFFFFFFull, 0x0000FFFFFFFFFFFFull, 0x0003FFFFFFFFFFFFull, 0x000FFFFFFFFFFFFFull, 0x003FFFFFFFFFFFFFull, 0x00FFFFFFFFFFFFFFull, 0x03FFFFFFFFFFFFFFull, 0x0FFFFFFFFFFFFFFFull, 0x3FFFFFFFFFFFFFFFull }; //------------------------------------------------------------------------------ // MinimizerHeader: Numerical class constants. constexpr std::uint32_t MinimizerHeader::TAG; constexpr std::uint32_t MinimizerHeader::VERSION; constexpr std::uint64_t MinimizerHeader::FLAG_MASK; constexpr std::uint64_t MinimizerHeader::FLAG_KEY_MASK; constexpr size_t MinimizerHeader::FLAG_KEY_OFFSET; constexpr std::uint64_t MinimizerHeader::FLAG_WEIGHT_MASK; constexpr size_t MinimizerHeader::FLAG_WEIGHT_OFFSET; constexpr std::uint64_t MinimizerHeader::FLAG_SYNCMERS; constexpr std::uint64_t MinimizerHeader::FLAG_PAYLOAD_MASK; constexpr size_t MinimizerHeader::FLAG_PAYLOAD_OFFSET; constexpr std::uint32_t MinimizerHeader::TAGS_VERSION; //------------------------------------------------------------------------------ // Position: Numerical class constants. constexpr size_t Position::OFFSET_BITS; constexpr size_t Position::ID_OFFSET; constexpr Position::code_type Position::REV_MASK; constexpr Position::code_type Position::OFF_MASK; constexpr Position::code_type Position::NO_POSITION; //------------------------------------------------------------------------------ // Key64: Numerical class constants. constexpr std::size_t Key64::KEY_BITS; constexpr std::size_t Key64::KMER_LENGTH; constexpr std::size_t Key64::WINDOW_LENGTH; constexpr std::size_t Key64::SMER_LENGTH; constexpr std::size_t Key64::KMER_MAX_LENGTH; constexpr Key64::code_type Key64::EMPTY_KEY; constexpr Key64::code_type Key64::NO_KEY; constexpr Key64::code_type Key64::KEY_MASK; constexpr Key64::code_type Key64::IS_POINTER; //------------------------------------------------------------------------------ // Key128: Numerical class constants. constexpr std::size_t Key128::KEY_BITS; constexpr std::size_t Key128::KMER_LENGTH; constexpr std::size_t Key128::WINDOW_LENGTH; constexpr std::size_t Key128::SMER_LENGTH; constexpr std::size_t Key128::KMER_MAX_LENGTH; constexpr Key128::code_type Key128::EMPTY_KEY; constexpr Key128::code_type Key128::NO_KEY; constexpr Key128::code_type Key128::KEY_MASK; constexpr Key128::code_type Key128::IS_POINTER; //------------------------------------------------------------------------------ MinimizerHeader::MinimizerHeader() : tag(TAG), version(VERSION), k(0), w_or_s(0), keys(0), unused(0), capacity(0), values(0), unique(0), flags(0) { } MinimizerHeader::MinimizerHeader(size_t kmer_length, size_t window_length, size_t key_bits, size_t payload_size) : tag(TAG), version(VERSION), k(kmer_length), w_or_s(window_length), keys(0), unused(0), capacity(0), values(0), unique(0), flags(0) { this->set_int(FLAG_KEY_MASK, FLAG_KEY_OFFSET, key_bits); this->set_int(FLAG_PAYLOAD_MASK, FLAG_PAYLOAD_OFFSET, payload_size); } void MinimizerHeader::sanitize(size_t kmer_max_length) { if(this->k > kmer_max_length) { std::cerr << "MinimizerHeader::sanitize(): Adjusting k from " << this->k << " to " << kmer_max_length << std::endl; this->k = kmer_max_length; } if(this->k <= 1) { std::cerr << "MinimizerHeader::sanitize(): Adjusting k from " << this->k << " to " << 2 << std::endl; this->k = 2; } if(this->get_flag(FLAG_SYNCMERS)) { if(this->w_or_s == 0) { std::cerr << "MinimizerHeader::sanitize(): Adjusting s from " << this->w_or_s << " to " << 1 << std::endl; this->w_or_s = 1; } if(this->w_or_s >= this->k) { std::cerr << "MinimizerHeader::sanitize(): Adjusting s from " << this->w_or_s << " to " << (this->k - 1) << std::endl; this->w_or_s = this->k - 1; } if(this->downweight() > 0) { std::cerr << "MinimizerHeader::sanizize(): Weights cannot be used with syncmers" << std::endl; this->set_int(FLAG_WEIGHT_MASK, FLAG_WEIGHT_OFFSET, 0); } } else { if(this->w_or_s == 0) { std::cerr << "MinimizerHeader::sanitize(): Adjusting w from " << this->w_or_s << " to " << 1 << std::endl; this->w_or_s = 1; } } } void MinimizerHeader::check() const { if(this->tag != TAG) { throw sdsl::simple_sds::InvalidData("MinimizerHeader: Invalid tag"); } if(this->version < VERSION || this->version > VERSION) { std::string msg = "MinimizerHeader: Expected version " + std::to_string(VERSION) + ", got version " + std::to_string(this->version); throw sdsl::simple_sds::InvalidData(msg); } std::uint64_t mask = (FLAG_MASK); if((this->flags & mask) != this->flags) { throw sdsl::simple_sds::InvalidData("MinimizerHeader: Invalid flags"); } } void MinimizerHeader::update_version() { this->version = VERSION; } void MinimizerHeader::set_int(std::uint64_t mask, size_t offset, size_t value) { this->unset(mask); this->set((value << offset) & mask); } size_t MinimizerHeader::get_int(std::uint64_t mask, size_t offset) const { return (this->flags & mask) >> offset; } size_t MinimizerHeader::key_bits() const { return this->get_int(FLAG_KEY_MASK, FLAG_KEY_OFFSET); } size_t MinimizerHeader::downweight() const { return this->get_int(FLAG_WEIGHT_MASK, FLAG_WEIGHT_OFFSET); } size_t MinimizerHeader::payload_size() const { return this->get_int(FLAG_PAYLOAD_MASK, FLAG_PAYLOAD_OFFSET); } bool MinimizerHeader::operator==(const MinimizerHeader& another) const { return (this->tag == another.tag && this->version == another.version && this->k == another.k && this->w_or_s == another.w_or_s && this->flags == another.flags); } //------------------------------------------------------------------------------ Key64 Key64::encode(const std::string& sequence) { code_type packed = 0; for(auto c : sequence) { auto packed_char = KmerEncoding::CHAR_TO_PACK[static_cast(c)]; if(packed_char > KmerEncoding::PACK_MASK) { throw std::runtime_error("Key64::encode(): Cannot encode character '" + std::to_string(c) + "'"); } packed = (packed << KmerEncoding::PACK_WIDTH) | packed_char; } return Key64(packed); } std::string Key64::decode(size_t k) const { std::string result; result.reserve(k); for(size_t i = 0; i < k; i++) { result.push_back(KmerEncoding::PACK_TO_CHAR[(this->key >> ((k - i - 1) * KmerEncoding::PACK_WIDTH)) & KmerEncoding::PACK_MASK]); } return result; } Key64 Key64::reverse_complement(size_t k) const { value_type source = ~(this->get_key()); // Complement of the kmer, plus weird high-order bits. code_type result = 0; for(size_t i = 0; i < k; i++) { result = (result << KmerEncoding::PACK_WIDTH) | (source & KmerEncoding::PACK_MASK); source >>= KmerEncoding::PACK_WIDTH; } return Key64(result); } std::ostream& operator<<(std::ostream& out, Key64 value) { out << value.key; return out; } Key128 Key128::encode(const std::string& sequence) { size_t low_limit = (sequence.size() > KmerEncoding::FIELD_CHARS ? KmerEncoding::FIELD_CHARS : sequence.size()); code_type packed_high = 0; code_type packed_low = 0; for(size_t i = 0; i < sequence.size(); i++) { auto c = sequence[i]; auto packed_char = KmerEncoding::CHAR_TO_PACK[static_cast(c)]; if(packed_char > KmerEncoding::PACK_MASK) { throw std::runtime_error("Key128::encode(): Cannot encode character '" + std::to_string(c) + "'"); } code_type& pack_to = (i < sequence.size() - low_limit) ? packed_high : packed_low; pack_to = (pack_to << KmerEncoding::PACK_WIDTH) | packed_char; } return Key128(packed_high, packed_low); } std::string Key128::decode(size_t k) const { std::string result; result.reserve(k); size_t low_limit = (k > KmerEncoding::FIELD_CHARS ? KmerEncoding::FIELD_CHARS : k); for(size_t i = KmerEncoding::FIELD_CHARS; i < k; i++) { result.push_back(KmerEncoding::PACK_TO_CHAR[(this->high >> ((k - i - 1) * KmerEncoding::PACK_WIDTH)) & KmerEncoding::PACK_MASK]); } for(size_t i = 0; i < low_limit; i++) { result.push_back(KmerEncoding::PACK_TO_CHAR[(this->low >> ((low_limit - i - 1) * KmerEncoding::PACK_WIDTH)) & KmerEncoding::PACK_MASK]); } return result; } Key128 Key128::reverse_complement(size_t k) const { constexpr size_t HIGH_SHIFT = KmerEncoding::FIELD_BITS - KmerEncoding::PACK_WIDTH; // Get the complement of the kmer, plus weird high-order bits. value_type source = this->get_key(); source.first = ~(source.first); source.second = ~(source.second); code_type high = 0, low = 0; for(size_t i = 0; i < k; i++) { high = (high << KmerEncoding::PACK_WIDTH) | ((low >> HIGH_SHIFT) & KmerEncoding::PACK_MASK); low = (low << KmerEncoding::PACK_WIDTH) | (source.second & KmerEncoding::PACK_MASK); source.second = ((source.first & KmerEncoding::PACK_MASK) << HIGH_SHIFT) | (source.second >> KmerEncoding::PACK_WIDTH); source.first >>= KmerEncoding::PACK_WIDTH; } return Key128(high, low); } std::ostream& operator<<(std::ostream& out, Key128 value) { out << "(" << value.high << ", " << value.low << ")"; return out; } //------------------------------------------------------------------------------ template void hits_in_subgraph ( const MinimizerIndex& index, KmerEncoding::multi_value_type hits, const std::unordered_set& subgraph, const std::function& report_hit ) { for(size_t i = 0; i < hits.second; i++) { auto value = index.get_value(hits, i); auto iter = subgraph.find(value.first.id()); if(iter != subgraph.end()) { report_hit(value); } } } /* Exponential search that returns the first offset with get_value(offset) >= target. We assume that start < limit and get_value(start) < target. Returns limit if get_value(offset) < target for all offset < limit. */ size_t exponential_search(size_t start, size_t limit, nid_t target, const std::function& get_value) { // Exponential search: low is too early. size_t step = 1; size_t low = start, candidate = start + step; while(candidate < limit && get_value(candidate) < target) { step *= 2; low = candidate; candidate += step; } // Binary search: low + 1 is the first candidate while candidate is the last. low++; size_t count = std::min(limit, candidate + 1) - low; while(count > 0) { step = count / 2; candidate = low + step; if(get_value(candidate) < target) { low = candidate + 1; count -= step + 1; } else { count = step; } } return low; } template void hits_in_subgraph ( const MinimizerIndex& index, KmerEncoding::multi_value_type hits, const std::vector& subgraph, const std::function& report_hit ) { size_t hit_offset = 0, subgraph_offset = 0; while(hit_offset < hits.second && subgraph_offset < subgraph.size()) { auto value = index.get_value(hits, hit_offset); nid_t node = value.first.id(); if(node < subgraph[subgraph_offset]) { hit_offset = exponential_search(hit_offset, hits.second, subgraph[subgraph_offset], [&](size_t offset) -> nid_t { return index.get_value(hits, offset).first.id(); }); } else if(node > subgraph[subgraph_offset]) { subgraph_offset = exponential_search(subgraph_offset, subgraph.size(), node, [&](size_t offset) -> nid_t { return subgraph[offset]; }); } else { report_hit(value); hit_offset++; } } } // Instantiate templates, as we did not define the functions in the header. template void hits_in_subgraph ( const MinimizerIndex& index, KmerEncoding::multi_value_type hits, const std::unordered_set& subgraph, const std::function& report_hit ); template void hits_in_subgraph ( const MinimizerIndex& index, KmerEncoding::multi_value_type hits, const std::unordered_set& subgraph, const std::function& report_hit ); template void hits_in_subgraph ( const MinimizerIndex& index, KmerEncoding::multi_value_type hits, const std::vector& subgraph, const std::function& report_hit ); template void hits_in_subgraph ( const MinimizerIndex& index, KmerEncoding::multi_value_type hits, const std::vector& subgraph, const std::function& report_hit ); //------------------------------------------------------------------------------ } // namespace gbwtgraph jltsiren-gbwtgraph-e27bc43/src/naive_graph.cpp000066400000000000000000000243721524136466400215430ustar00rootroot00000000000000#include namespace gbwtgraph { //------------------------------------------------------------------------------ // Other class variables. const std::string NaiveGraph::TRANSLATION_EXTENSION = ".trans"; //------------------------------------------------------------------------------ // Construction. NaiveGraph::NaiveGraph() : min_id(std::numeric_limits::max()), max_id(0) { } void NaiveGraph::normalize_sequence(size_t from_offset, size_t length) { for(size_t i = from_offset; i < from_offset + length; i++) { this->sequences[i] = std::toupper(static_cast(this->sequences[i])); } } void NaiveGraph::create_node(nid_t node_id, std::string_view sequence) { assert(!this->uses_translation()); if(sequence.empty()) { throw std::runtime_error("NaiveGraph::create_node: empty sequence for node " + std::to_string(node_id)); } auto result = this->nodes.try_emplace(node_id, Node()); if(!result.second) { throw std::runtime_error("NaiveGraph::create_node: duplicate node " + std::to_string(node_id)); } result.first->second.sequence_offset = this->sequences.size(); result.first->second.sequence_length = sequence.size(); this->sequences.insert(this->sequences.end(), sequence.begin(), sequence.end()); this->normalize_sequence(result.first->second.sequence_offset, result.first->second.sequence_length); if(node_id < this->min_id) { this->min_id = node_id; } if(node_id > this->max_id) { this->max_id = node_id; } } std::pair NaiveGraph::translate_segment(const std::string& name, std::string_view sequence, size_t max_length) { assert(this->nodes.empty() || this->uses_translation()); if(sequence.empty()) { throw std::runtime_error("NaiveGraph::translate_segment: empty sequence for segment " + name); } auto result = this->segment_translation.try_emplace(name, std::pair(this->max_id + 1, this->max_id + 1)); if(!result.second) { throw std::runtime_error("NaiveGraph::translate_segment: duplicate segment " + name); } for(size_t offset = 0; offset < sequence.size(); offset += max_length) { size_t length = std::min(max_length, sequence.size() - offset); Node& node = this->nodes[result.first->second.second]; result.first->second.second++; node.sequence_offset = this->sequences.size() + offset; node.sequence_length = length; } size_t from_offset = this->sequences.size(); this->sequences.insert(this->sequences.end(), sequence.begin(), sequence.end()); this->normalize_sequence(from_offset, sequence.size()); // We always add at least one node beyond the previous max_id. this->min_id = std::min(this->min_id, result.first->second.first); this->max_id = result.first->second.second - 1; return result.first->second; } void NaiveGraph::create_edge(gbwt::node_type from, gbwt::node_type to) { nid_t from_id = gbwt::Node::id(from); nid_t to_id = gbwt::Node::id(to); auto from_iter = this->get_node_mut(from_id); auto to_iter = this->get_node_mut(to_id); if(from_iter == this->nodes.end() || to_iter == this->nodes.end()) { throw std::runtime_error("NaiveGraph::create_edge: Cannot create an edge between nodes " + std::to_string(from_id) + " and " + std::to_string(to_id)); } if(gbwt::Node::is_reverse(from)) { from_iter->second.predecessors.push_back(gbwt::Node::reverse(to)); } else { from_iter->second.successors.push_back(to); } if(gbwt::Node::is_reverse(to)) { to_iter->second.successors.push_back(gbwt::Node::reverse(from)); } else { to_iter->second.predecessors.push_back(from); } } void NaiveGraph::remove_duplicate_edges() { for(auto& node_pair : this->nodes) { gbwt::removeDuplicates(node_pair.second.predecessors, false); gbwt::removeDuplicates(node_pair.second.successors, false); } } //------------------------------------------------------------------------------ // HandleGraph interface. bool NaiveGraph::has_node(nid_t node_id) const { return this->has_node_impl(node_id); } handle_t NaiveGraph::get_handle(const nid_t& node_id, bool is_reverse) const { return node_to_handle(gbwt::Node::encode(node_id, is_reverse)); } nid_t NaiveGraph::get_id(const handle_t& handle) const { return gbwt::Node::id(handle_to_node(handle)); } bool NaiveGraph::get_is_reverse(const handle_t& handle) const { return gbwt::Node::is_reverse(handle_to_node(handle)); } handle_t NaiveGraph::flip(const handle_t& handle) const { return node_to_handle(gbwt::Node::reverse(handle_to_node(handle))); } size_t NaiveGraph::get_length(const handle_t& handle) const { auto iter = this->get_node(handle); if(iter == this->nodes.end()) { return 0; } return iter->second.sequence_length; } std::string NaiveGraph::get_sequence(const handle_t& handle) const { auto iter = this->get_node(handle); if(iter == this->nodes.end()) { return std::string(); } size_t offset = iter->second.sequence_offset; size_t length = iter->second.sequence_length; std::string sequence(this->sequences.begin() + offset, this->sequences.begin() + offset + length); if(this->get_is_reverse(handle)) { reverse_complement_in_place(sequence); } return sequence; } char NaiveGraph::get_base(const handle_t& handle, size_t index) const { auto iter = this->get_node(handle); if(iter == this->nodes.end()) { return 'N'; } size_t offset = iter->second.sequence_offset; size_t length = iter->second.sequence_length; if(index >= length) { return 'N'; } char base = this->sequences[offset + index]; if(this->get_is_reverse(handle)) { base = complement(base); } return base; } std::string NaiveGraph::get_subsequence(const handle_t& handle, size_t index, size_t size) const { auto iter = this->get_node(handle); if(iter == this->nodes.end()) { return std::string(); } size_t offset = iter->second.sequence_offset; size_t length = iter->second.sequence_length; if(index >= length) { return std::string(); } size_t actual_size = std::min(size, length - index); std::string subsequence(this->sequences.begin() + offset + index, this->sequences.begin() + offset + index + actual_size); if(this->get_is_reverse(handle)) { reverse_complement_in_place(subsequence); } return subsequence; } size_t NaiveGraph::get_node_count() const { return this->nodes.size(); } nid_t NaiveGraph::min_node_id() const { return this->min_id; } nid_t NaiveGraph::max_node_id() const { return this->max_id; } bool NaiveGraph::follow_edges_impl(const handle_t& handle, bool go_left, const std::function& iteratee) const { auto iter = this->get_node(handle); if(iter == this->nodes.end()) { return true; } bool flip = this->get_is_reverse(handle); const std::vector& edges = (go_left ^ flip ? iter->second.predecessors : iter->second.successors); for(gbwt::node_type node : edges) { handle_t next = node_to_handle(flip ? gbwt::Node::reverse(node) : node); if(!iteratee(next)) { return false; } } return true; } bool NaiveGraph::for_each_handle_impl(const std::function& iteratee, bool) const { for(auto iter = this->nodes.begin(); iter != this->nodes.end(); ++iter) { handle_t handle = node_to_handle(gbwt::Node::encode(iter->first, false)); if(!iteratee(handle)) { return false; } } return true; } size_t NaiveGraph::get_degree(const handle_t& handle, bool go_left) const { auto iter = this->get_node(handle); if(iter == this->nodes.end()) { return 0; } bool flip = this->get_is_reverse(handle); const std::vector& edges = (go_left ^ flip ? iter->second.predecessors : iter->second.successors); return edges.size(); } //------------------------------------------------------------------------------ // Custom interface. std::string_view NaiveGraph::get_sequence_view(nid_t node_id) const { auto iter = this->get_node(node_id); if(iter == this->nodes.end()) { return std::string_view(); } size_t offset = iter->second.sequence_offset; size_t length = iter->second.sequence_length; return std::string_view(this->sequences.data() + offset, length); } bool NaiveGraph::has_node_or_segment(const std::string& name) const { if(this->uses_translation()) { return (this->segment_translation.find(name) != this->segment_translation.end()); } else { auto parse = parse_unsigned(name); return (parse.second && this->has_node_impl(parse.first)); } } std::pair NaiveGraph::translate(const std::string& segment_name) const { if(this->uses_translation()) { auto iter = this->segment_translation.find(segment_name); if(iter == this->segment_translation.end()) { return no_translation(); } return iter->second; } else { auto parse = parse_unsigned(segment_name); if(parse.second && this->has_node_impl(parse.first)) { return std::pair(parse.first, parse.first + 1); } return no_translation(); } } // invert_translation std::pair> NaiveGraph::invert_translation(const std::function)>& is_present) const { std::pair> result; // Invert the translation. // This stores half-open ranges of node identifiers corresponding to segments, and views to their segment names. std::vector, std::string_view>> inverse; inverse.reserve(this->segment_translation.size()); for(auto iter = this->segment_translation.begin(); iter != this->segment_translation.end(); ++iter) { inverse.emplace_back(iter->second, std::string_view(iter->first)); } gbwt::parallelQuickSort(inverse.begin(), inverse.end()); // Store the segment names. std::string empty; result.first = gbwt::StringArray(inverse.size(), [&](size_t offset) -> std::string_view { // This produces a view to each string to store. if(is_present(inverse[offset].first)) { return inverse[offset].second; } else { return std::string_view(empty); } }); // Store the mapping. sdsl::sd_vector_builder builder(this->max_id + 1, inverse.size()); for(auto& translation : inverse) { builder.set_unsafe(translation.first.first); } result.second = sdsl::sd_vector<>(builder); return result; } //------------------------------------------------------------------------------ } // namespace gbwtgraph jltsiren-gbwtgraph-e27bc43/src/path_cover.cpp000066400000000000000000000651741524136466400214170ustar00rootroot00000000000000#include #include #include #include #include #include #include namespace gbwtgraph { //------------------------------------------------------------------------------ // Similar to std::lower_bound(). template size_t find_first(std::vector& array, nid_t id) { size_t first = 0, mid = 0; size_t count = array.size(); while(count > 0) { size_t step = count / 2; mid = first + step; if(array[mid].first < id) { first = mid + 1; count -= step + 1; } else { count = step; } } return first; } std::vector reverse_complement(const HandleGraph& graph, std::vector& forward) { std::vector result = forward; std::reverse(result.begin(), result.end()); for(handle_t& handle : result) { handle = graph.flip(handle); } return result; } std::vector forward_window(const HandleGraph& graph, const std::deque& path, const handle_t& successor, size_t k) { if(path.size() + 1 < k) { k = path.size() + 1; } // Handle the short initial paths in DAGs. std::vector forward; forward.reserve(k); forward.insert(forward.end(), path.end() - (k - 1), path.end()); forward.push_back(successor); std::vector reverse = reverse_complement(graph, forward); return (forward < reverse ? forward : reverse); } std::vector backward_window(const HandleGraph& graph, const std::deque& path, const handle_t& predecessor, size_t k) { std::vector forward; forward.reserve(k); forward.push_back(predecessor); forward.insert(forward.end(), path.begin(), path.begin() + (k - 1)); std::vector reverse = reverse_complement(graph, forward); return (forward < reverse ? forward : reverse); } template struct BestCoverage { typedef typename Coverage::coverage_t coverage_t; coverage_t coverage; handle_t handle; BestCoverage() : coverage(Coverage::worst_coverage()), handle() {} void update(const coverage_t& new_coverage, const handle_t& new_handle) { if(new_coverage < this->coverage) { this->coverage = new_coverage; this->handle = new_handle; } } }; //------------------------------------------------------------------------------ /* The best candidate is the one with the lowest coverage so far. */ struct SimpleCoverage { typedef HandleGraph graph_t; typedef size_t coverage_t; typedef std::pair node_coverage_t; static std::vector init_node_coverage(const graph_t& graph, const std::vector& component) { std::vector node_coverage; node_coverage.reserve(component.size()); for(nid_t id : component) { // We are actually interested in the intersection of this graph and the component. // For example, some nodes of the original graph may be missing from a GBWTGraph. if(!(graph.has_node(id))) { continue; } node_coverage.emplace_back(id, static_cast(0)); } return node_coverage; } static bool extend_forward(const graph_t& graph, std::deque& path, size_t k, std::vector& node_coverage, std::map, coverage_t>& path_coverage, bool acyclic) { bool success = false; BestCoverage best; graph.follow_edges(path.back(), false, [&](const handle_t& next) { success = true; if(!acyclic && path.size() + 1 < k) // Node coverage. { size_t first = find_first(node_coverage, graph.get_id(next)); best.update(node_coverage[first].second, next); } else { std::vector window = forward_window(graph, path, next, k); best.update(path_coverage[window], next); } }); if(success) { if(acyclic || path.size() + 1 >= k) { std::vector window = forward_window(graph, path, best.handle, k); increase_coverage(path_coverage[window]); } if(!acyclic) { size_t first = find_first(node_coverage, graph.get_id(best.handle)); increase_coverage(node_coverage[first]); } path.push_back(best.handle); } return success; } static bool extend_backward(const graph_t& graph, std::deque& path, size_t k, std::vector& node_coverage, std::map, coverage_t>& path_coverage) { bool success = false; BestCoverage best; graph.follow_edges(path.front(), true, [&](const handle_t& prev) { success = true; if(path.size() + 1 < k) // Node coverage. { size_t first = find_first(node_coverage, graph.get_id(prev)); best.update(node_coverage[first].second, prev); } else { std::vector window = backward_window(graph, path, prev, k); best.update(path_coverage[window], prev); } }); if(success) { if(path.size() + 1 >= k) { std::vector window = backward_window(graph, path, best.handle, k); increase_coverage(path_coverage[window]); } size_t first = find_first(node_coverage, graph.get_id(best.handle)); increase_coverage(node_coverage[first]); path.push_front(best.handle); } return success; } static void increase_coverage(coverage_t& coverage) { coverage++; } static void increase_coverage(node_coverage_t& node) { increase_coverage(node.second); } static coverage_t worst_coverage() { return std::numeric_limits::max(); } static std::string name() { return "SimpleCoverage"; } }; //------------------------------------------------------------------------------ /* The best candidate is the one with the highest ratio true_coverage / (selected_coverage + 1). */ struct LocalHaplotypes { typedef GBWTGraph graph_t; struct coverage_t { size_t selected_coverage; size_t true_coverage; double score; // Give priority to high scores. bool operator<(const coverage_t& another) const { return (this->score > another.score); } coverage_t() : selected_coverage(0), true_coverage(0), score(0.0) {} coverage_t(size_t true_coverage) : selected_coverage(0), true_coverage(true_coverage), score(true_coverage) {} void compute_score() { this->score = this->true_coverage / (this->selected_coverage + 1.0); } }; typedef std::pair node_coverage_t; static std::vector init_node_coverage(const graph_t& graph, const std::vector& component) { std::vector node_coverage; node_coverage.reserve(component.size()); for(nid_t id : component) { // We are actually interested in the intersection of this graph and the component. // For example, some nodes of the original graph may be missing from a GBWTGraph. // This also implies that the true coverage of selected nodes is nonzero. if(!(graph.has_node(id))) { continue; } coverage_t coverage; coverage.selected_coverage = 0; coverage.true_coverage = graph.index->nodeSize(gbwt::Node::encode(id, false)); if(coverage.true_coverage == 0) { continue; } coverage.compute_score(); node_coverage.emplace_back(id, coverage); } return node_coverage; } static bool extend_forward(const graph_t& graph, std::deque& path, size_t k, std::vector& node_coverage, std::map, coverage_t>& path_coverage, bool acyclic) { bool success = false; BestCoverage best; auto start = (path.size() + 1 < k ? path.begin() : path.end() - (k - 1)); std::vector context(start, path.end()); gbwt::BidirectionalState state = graph.bd_find(context); graph.follow_paths(state, false, [&](const gbwt::BidirectionalState& next) -> bool { success = true; handle_t handle = GBWTGraph::node_to_handle(next.forward.node); if(!acyclic && path.size() + 1 < k) // Node coverage. { size_t first = find_first(node_coverage, graph.get_id(handle)); best.update(node_coverage[first].second, handle); } else { std::vector window = forward_window(graph, path, handle, k); // Insert empty coverage or find the existing coverage. auto result = path_coverage.insert({ window, coverage_t(next.size()) }); best.update(result.first->second, handle); } return true; }); if(success) { if(acyclic || path.size() + 1 >= k) { std::vector window = forward_window(graph, path, best.handle, k); increase_coverage(path_coverage[window]); } if(!acyclic) { size_t first = find_first(node_coverage, graph.get_id(best.handle)); increase_coverage(node_coverage[first]); } path.push_back(best.handle); } return success; } static bool extend_backward(const graph_t& graph, std::deque& path, size_t k, std::vector& node_coverage, std::map, coverage_t>& path_coverage) { bool success = false; BestCoverage best; auto limit = (path.size() + 1 < k ? path.end() : path.begin() + (k - 1)); std::vector context(path.begin(), limit); gbwt::BidirectionalState state = graph.bd_find(context); graph.follow_paths(state, true, [&](const gbwt::BidirectionalState& prev) -> bool { success = true; handle_t handle = GBWTGraph::node_to_handle(prev.backward.node); handle = graph.flip(handle); // Get the correct orientation. if(path.size() + 1 < k) // Node coverage. { size_t first = find_first(node_coverage, graph.get_id(handle)); best.update(node_coverage[first].second, handle); } else { std::vector window = backward_window(graph, path, handle, k); // Insert empty coverage or find the existing coverage. auto result = path_coverage.insert({ window, coverage_t(prev.size()) }); best.update(result.first->second, handle); } return true; }); if(success) { if(path.size() + 1 >= k) { std::vector window = backward_window(graph, path, best.handle, k); increase_coverage(path_coverage[window]); } size_t first = find_first(node_coverage, graph.get_id(best.handle)); increase_coverage(node_coverage[first]); path.push_front(best.handle); } return success; } static void increase_coverage(coverage_t& coverage) { coverage.selected_coverage++; coverage.compute_score(); } static void increase_coverage(node_coverage_t& node) { increase_coverage(node.second); } static coverage_t worst_coverage() { return coverage_t(); } static std::string name() { return "LocalHaplotypes"; } }; //------------------------------------------------------------------------------ bool path_cover_sanity_checks(const HandleGraph& graph, size_t n, size_t k) { if(graph.get_node_count() == 0 || n == 0) { return false; } if(k < PATH_COVER_MIN_K) { std::cerr << "path_cover_sanity_checks(): Window length (" << k << ") must be at least " << PATH_COVER_MIN_K << std::endl; return false; } nid_t min_id = graph.min_node_id(); if(min_id < 1) { std::cerr << "path_cover_sanity_checks(): Minimum node id (" << min_id << ") must be positive" << std::endl; return false; } return true; } // Create a path cover of the given component and insert it into the builder. // This does not create metadata. Returns true if the path cover was successfully // created. template bool component_path_cover( const typename Coverage::graph_t& graph, gbwt::GBWTBuilder& builder, const std::vector& component, size_t job_id, size_t component_id, const PathCoverParameters& parameters) { typedef typename Coverage::coverage_t coverage_t; typedef typename Coverage::node_coverage_t node_coverage_t; std::vector head_nodes = is_nice_and_acyclic(graph, component); bool acyclic = !(head_nodes.empty()); if(parameters.show_progress) { std::string msg = "Job " + std::to_string(job_id) + ", component " + std::to_string(component_id) + ": " + Coverage::name(); if(acyclic) { msg += " (acyclic)"; } #pragma omp critical { std::cerr << msg << std::endl; } } // Node coverage for the potential starting nodes. std::vector node_coverage = Coverage::init_node_coverage(graph, (acyclic ? head_nodes : component)); std::map, coverage_t> path_coverage; // Path and its reverse complement are equivalent. // Node coverage will be empty if we cannot create this type of path cover for the component. // For example, if there are no haplotypes for LocalHaplotypes. if(node_coverage.empty()) { if(parameters.show_progress) { #pragma omp critical { std::cerr << Coverage::name() << ": Cannot find this type of path cover for component " << component_id << std::endl; } } return false; } // Generate n paths in the component. for(size_t i = 0; i < parameters.num_paths; i++) { // Choose a starting node with minimum coverage and then sort the nodes by id. std::deque path; std::sort(node_coverage.begin(), node_coverage.end(), [](const node_coverage_t& a, const node_coverage_t& b) -> bool { return (a.second < b.second); }); path.push_back(graph.get_handle(node_coverage.front().first, false)); Coverage::increase_coverage(node_coverage.front()); std::sort(node_coverage.begin(), node_coverage.end(), [](const node_coverage_t& a, const node_coverage_t& b) -> bool { return (a.first < b.first); }); // Extend the path forward if acyclic or in both directions otherwise. bool success = true; while(success && path.size() < component.size()) { success = false; success |= Coverage::extend_forward(graph, path, parameters.context, node_coverage, path_coverage, acyclic); if(!acyclic && path.size() < component.size()) { success |= Coverage::extend_backward(graph, path, parameters.context, node_coverage, path_coverage); } } // Insert the path and its name into the index. gbwt::vector_type buffer; buffer.reserve(path.size()); for(handle_t handle : path) { buffer.push_back(gbwt::Node::encode(graph.get_id(handle), graph.get_is_reverse(handle))); } builder.insert(buffer, true); } return true; } //------------------------------------------------------------------------------ void store_named_paths(gbwt::GBWTBuilder& builder, const PathHandleGraph& graph, const std::function* path_filter) { // What path senses should we copy across? std::unordered_set named_senses = {PathSense::GENERIC, PathSense::REFERENCE}; store_paths(builder, graph, named_senses, path_filter); } void store_paths(gbwt::GBWTBuilder& builder, const PathHandleGraph& graph, const std::unordered_set& senses, const std::function* path_filter) { if(!builder.index.metadata.hasContigNames() && builder.index.metadata.contigs() > 0) { throw std::logic_error("Cannot add paths to an index with existing unnamed contigs"); } if(!builder.index.metadata.hasSampleNames() && builder.index.metadata.samples() > 0) { throw std::logic_error("Cannot add paths to an index with existing unnamed samples"); } if(!builder.index.metadata.hasPathNames() && builder.index.metadata.paths() > 0) { throw std::logic_error("Cannot add paths to an index with existing unnamed paths"); } // Turn the metadata back into a MetadataBuilder. MetadataBuilder metadata_builder(builder.index.metadata); // We set this to true if we found any paths bool added_paths = false; // We can only use one sense per sample currently. We use this to work out what it should be. std::unordered_map sample_sense; // Work out what new contigs paths correspond to graph.for_each_path_matching(&senses, nullptr, nullptr, [&](const path_handle_t& path) { if(path_filter != nullptr && !(*path_filter)(path)) { // The filter wants us to skip this path return; } // Get the sense PathSense sense = graph.get_sense(path); // Check for divergent senses for the sample std::string sample_name = graph.get_sample_name(path); auto it = sample_sense.find(sample_name); if(it != sample_sense.end()) { if(it->second != sense) { // Complain to the user. std::cerr << "gbwtgraph::store_paths(): Warning: multiple senses of path in sample \"" << sample_name << "\"; only one will be stored"; } } else { sample_sense.emplace_hint(it, sample_name, sense); } // Store the path metadata metadata_builder.add_path( sense, sample_name, graph.get_locus_name(path), graph.get_haplotype(path), graph.get_phase_block(path), graph.get_subrange(path) ); // Add the path to the GBWT gbwt::vector_type buffer; // TODO: This counting can be a whole path scan itself. Can we know and avoid it? buffer.reserve(graph.get_step_count(path)); for(handle_t handle : graph.scan_path(path)) { buffer.push_back(gbwt::Node::encode(graph.get_id(handle), graph.get_is_reverse(handle))); } builder.insert(buffer, true); // Insert in both orientations. added_paths = true; }); if(!sample_sense.empty()) { // Paths were stored // Commit back metadata changes builder.index.metadata = metadata_builder.get_metadata(); // Commit path senses set_sample_path_senses(builder.index.tags, sample_sense); } } //------------------------------------------------------------------------------ gbwt::GBWT path_cover_gbwt( const PathHandleGraph& graph, const PathCoverParameters& parameters, bool include_named_paths, const std::function* path_filter) { // Sanity checks. if(!path_cover_sanity_checks(graph, parameters.num_paths, parameters.context)) { return gbwt::GBWT(); } // GBWT construction parameters. gbwt::Verbosity::set(gbwt::Verbosity::SILENT); gbwt::size_type node_width = sdsl::bits::length(gbwt::Node::encode(graph.max_node_id(), true)); size_t size_bound = graph.get_node_count() / std::max(size_t(1), parameters.approximate_num_jobs); // Determine GBWT construction jobs. ConstructionJobs jobs = gbwt_construction_jobs(graph, size_bound); MetadataBuilder metadata; // Assign the paths we want to include to construction jobs. std::vector> paths_to_include(jobs.size()); if(include_named_paths) { paths_to_include = assign_paths(graph, jobs, &metadata, path_filter); } // Create metadata for the path cover. std::vector contig_names; if(include_named_paths && path_filter != nullptr) { contig_names = jobs.contig_names(graph, *path_filter); } else { contig_names = jobs.contig_names(graph); } for(size_t component = 0; component < jobs.components(); component++) { size_t job = jobs.job_for_component(component); if(job >= paths_to_include.size()) { continue; } for(size_t i = 0; i < parameters.num_paths; i++) { metadata.add_haplotype(COVER_PATH_SAMPLE_PREFIX + std::to_string(i), contig_names[component], 0, 0, job); } } // Create the actual path cover. std::vector partial_indexes(jobs.size()); std::vector> components_per_job = jobs.components_per_job(); int old_threads = omp_get_max_threads(); omp_set_num_threads(parameters.parallel_jobs); #pragma omp parallel for schedule(dynamic, 1) for(size_t job = 0; job < jobs.size(); job++) { gbwt::GBWTBuilder builder(node_width, parameters.batch_size, parameters.sample_interval); insert_paths(graph, paths_to_include[job], builder, job, parameters.show_progress); for(size_t component : components_per_job[job]) { component_path_cover( graph, builder, jobs.weakly_connected_components[component], job, component, parameters ); } builder.finish(); partial_indexes[job] = gbwt::GBWT(builder.index); } omp_set_num_threads(old_threads); // Merge the GBWTs and add metadata. if(parameters.show_progress) { std::cerr << "Merging " << partial_indexes.size() << " partial GBWTs" << std::endl; } gbwt::GBWT result(partial_indexes); result.addMetadata(); result.metadata = metadata.get_metadata(); return result; } //------------------------------------------------------------------------------ gbwt::GBWT local_haplotypes( const PathHandleGraph& graph, const gbwt::GBWT& index, const PathCoverParameters& parameters, bool include_named_paths, const std::function* path_filter) { // Sanity checks. if(!path_cover_sanity_checks(graph, parameters.num_paths, parameters.context)) { return gbwt::GBWT(); } // GBWT construction parameters. gbwt::Verbosity::set(gbwt::Verbosity::SILENT); gbwt::size_type node_width = sdsl::bits::length(gbwt::Node::encode(graph.max_node_id(), true)); size_t size_bound = graph.get_node_count() / std::max(size_t(1), parameters.approximate_num_jobs); // Determine GBWT construction jobs. ConstructionJobs jobs = gbwt_construction_jobs(graph, size_bound); MetadataBuilder metadata; // Assign the paths we want to include to construction jobs. std::vector> paths_to_include(jobs.size()); if(include_named_paths) { paths_to_include = assign_paths(graph, jobs, &metadata, path_filter); } // Create metadata for the path cover. std::vector contig_names; if(include_named_paths && path_filter != nullptr) { contig_names = jobs.contig_names(graph, *path_filter); } else { contig_names = jobs.contig_names(graph); } for(size_t component = 0; component < jobs.components(); component++) { size_t job = jobs.job_for_component(component); if(job >= paths_to_include.size()) { continue; } for(size_t i = 0; i < parameters.num_paths; i++) { metadata.add_haplotype("path_cover_" + std::to_string(i), contig_names[component], 0, 0, job); } } // If the graph we were given is a GBWTGraph using the same GBWT index, we can // use it directly for sampling local haplotypes. Otherwise we have to build a // temporary graph. const GBWTGraph* gbwt_graph = dynamic_cast(&graph); if(gbwt_graph != nullptr) { if(gbwt_graph->index != &index) { gbwt_graph = nullptr; } } GBWTGraph created_gbwt_graph; if(gbwt_graph == nullptr) { if(parameters.show_progress) { std::cerr << "Building a temporary GBWTGraph" << std::endl; } created_gbwt_graph = GBWTGraph(index, graph, nullptr); gbwt_graph = &created_gbwt_graph; } // Create the actual path cover. std::vector partial_indexes(jobs.size()); std::vector> components_per_job = jobs.components_per_job(); int old_threads = omp_get_max_threads(); omp_set_num_threads(parameters.parallel_jobs); #pragma omp parallel for schedule(dynamic, 1) for(size_t job = 0; job < jobs.size(); job++) { gbwt::GBWTBuilder builder(node_width, parameters.batch_size, parameters.sample_interval); insert_paths(graph, paths_to_include[job], builder, job, parameters.show_progress); for(size_t component : components_per_job[job]) { if(component_path_cover( *gbwt_graph, builder, jobs.weakly_connected_components[component], job, component, parameters )) { continue; } component_path_cover( graph, builder, jobs.weakly_connected_components[component], job, component, parameters ); } builder.finish(); partial_indexes[job] = gbwt::GBWT(builder.index); } omp_set_num_threads(old_threads); // Merge the GBWTs and add metadata. if(parameters.show_progress) { std::cerr << "Merging " << partial_indexes.size() << " partial GBWTs" << std::endl; } gbwt::GBWT result(partial_indexes); result.addMetadata(); result.metadata = metadata.get_metadata(); return result; } //------------------------------------------------------------------------------ size_t augment_gbwt( const HandleGraph& graph, gbwt::DynamicGBWT& index, const PathCoverParameters& parameters) { // Sanity checks. if(!path_cover_sanity_checks(graph, parameters.num_paths, parameters.context)) { return 0; } if(!(index.bidirectional())) { std::cerr << "augment_gbwt(): The GBWT index must be bidirectional" << std::endl; return 0; } if(!(index.hasMetadata())) { std::cerr << "augment_gbwt(): The GBWT index must have metadata" << std::endl; return 0; } if(!(index.empty())) { if(!(index.metadata.hasSampleNames() && index.metadata.hasContigNames() && index.metadata.hasPathNames())) { std::cerr << "augment_gbwt(): The GBWT index must have sample, contig, and path names" << std::endl; return 0; } } // GBWT construction parameters. gbwt::Verbosity::set(gbwt::Verbosity::SILENT); gbwt::size_type node_width = sdsl::bits::length(gbwt::Node::encode(graph.max_node_id(), true)); gbwt::GBWTBuilder builder(node_width, parameters.batch_size, parameters.sample_interval); builder.swapIndex(index); MetadataBuilder metadata(builder.index.metadata); // Find weakly connected components, ignoring the direction of the edges. std::vector> components = weakly_connected_components(graph); // Handle each component separately, but only if there are no GBWT paths in it. size_t augmented_components = 0; for(size_t component = 0; component < components.size(); component++) { bool has_paths = false; for(nid_t nid : components[component]) { gbwt::node_type node = gbwt::Node::encode(nid, false); if(builder.index.contains(node) && !(builder.index.empty(node))) { has_paths = true; break; } } if(has_paths) { continue; } component_path_cover( graph, builder, components[component], 0, component, parameters ); for(size_t i = 0; i < parameters.num_paths; i++) { metadata.add_haplotype("path_cover_" + std::to_string(i), "component_" + std::to_string(component), 0, 0, 0); } augmented_components++; } // Finish the construction and add the updated metadata. builder.finish(); builder.index.metadata = metadata.get_metadata(); builder.swapIndex(index); return augmented_components; } //------------------------------------------------------------------------------ } // namespace gbwtgraph jltsiren-gbwtgraph-e27bc43/src/subgraph.cpp000066400000000000000000000536431524136466400210760ustar00rootroot00000000000000#include #include #include #include #include #include namespace gbwtgraph { //------------------------------------------------------------------------------ // Numerical class constants. constexpr size_t PathIndex::DEFAULT_SAMPLE_INTERVAL; //------------------------------------------------------------------------------ PathIndex::PathIndex(const GBZ& gbz, size_t sample_interval) : sequence_positions(gbz.graph.named_paths.size()), gbwt_positions(gbz.graph.named_paths.size()) { #pragma omp parallel for schedule(dynamic, 1) for(size_t i = 0; i < gbz.graph.named_paths.size(); i++) { size_t length = 0; auto positions = sample_path_positions(gbz, handlegraph::as_path_handle(i), sample_interval, &length); sdsl::sd_vector_builder builder(length, positions.size()); for(const auto& pos : positions) { builder.set_unsafe(pos.first); this->gbwt_positions[i].push_back(pos.second); } this->sequence_positions[i] = sdsl::sd_vector<>(builder); } } std::pair PathIndex::sampled_position(path_handle_t handle, size_t offset) const { size_t path_id = handlegraph::as_integer(handle); if(path_id >= this->paths()) { return std::make_pair(0, gbwt::invalid_edge()); } auto iter = this->sequence_positions[path_id].predecessor(offset); if(iter == this->sequence_positions[path_id].one_end()) { return std::make_pair(0, gbwt::invalid_edge()); } return std::make_pair(iter->second, this->gbwt_positions[path_id][iter->first]); } //------------------------------------------------------------------------------ SubgraphQuery SubgraphQuery::path_offset(const gbwt::FullPathName& path_name, size_t offset, size_t context, HaplotypeOutput output) { gbwt::FullPathName path_query = path_name; path_query.offset = offset; SubgraphQuery result { QueryType::path_offset_query, output, path_query, 0, context }; return result; } SubgraphQuery SubgraphQuery::path_interval(const gbwt::FullPathName& path_name, size_t from, size_t to, size_t context, HaplotypeOutput output) { if(from >= to) { std::string msg = "SubgraphQuery::path_interval(): Empty interval [" + std::to_string(from) + ", " + std::to_string(to) + ")"; throw std::runtime_error(msg); } gbwt::FullPathName path_query = path_name; path_query.offset = from + (to - from) / 2; SubgraphQuery result { QueryType::path_offset_query, output, path_query, 0, context }; return result; } SubgraphQuery SubgraphQuery::node(nid_t node, size_t context, HaplotypeOutput output) { gbwt::FullPathName empty { "", "", 0, 0 }; SubgraphQuery result { QueryType::node_query, output, empty, node, context }; return result; } std::string output_type(SubgraphQuery::HaplotypeOutput output) { switch(output) { case SubgraphQuery::HaplotypeOutput::all_haplotypes: return "all"; case SubgraphQuery::HaplotypeOutput::distinct_haplotypes: return "distinct"; case SubgraphQuery::HaplotypeOutput::reference_only: return "reference only"; default: return "invalid output"; } } std::string SubgraphQuery::to_string() const { if(this->type == QueryType::path_offset_query) { std::string path_name = "sample " + this->path_query.sample_name + ", contig " + this->path_query.contig_name + ", haplotype " + std::to_string(this->path_query.haplotype); std::string output = output_type(this->haplotype_output); return "(" + path_name + ", offset " + std::to_string(this->path_query.offset) + ", context " + std::to_string(this->context) + ", " + output + ")"; } else if(this->type == QueryType::node_query) { std::string output = output_type(this->haplotype_output); return "(node " + std::to_string(this->node_id) + ", context " + std::to_string(this->context) + ", " + output + ")"; } else { // NOTE: `path_interval_query` is currently implemented as a `path_offset_query` around the midpoint. return "(invalid query)"; } } //------------------------------------------------------------------------------ /* Helper functions for the constructor. */ std::pair find_position(const GBZ& gbz, const PathIndex& path_index, path_handle_t path, size_t offset) { std::pair position = path_index.sampled_position(path, offset); if(position.second == gbwt::invalid_edge()) { std::string msg = "Subgraph::Subgraph(): Could not find an indexed path position for path " + gbz.graph.get_path_name(path) + ", offset " + std::to_string(offset); throw std::runtime_error(msg); } while(true) { size_t node_len = gbz.graph.get_length(GBWTGraph::node_to_handle(position.second.first)); if(position.first + node_len > offset) { pos_t graph_pos = make_pos_t(gbwt::Node::id(position.second.first), gbwt::Node::is_reverse(position.second.first), offset - position.first); return std::make_pair(graph_pos, position.second); } position.first += node_len; position.second = gbz.index.LF(position.second); if(position.second.first == gbwt::ENDMARKER) { std::string msg = "Subgraph::Subgraph(): Path " + gbz.graph.get_path_name(path) + " does not contain offset " + std::to_string(offset); throw std::runtime_error(msg); } } } std::set find_subgraph(const GBWTGraph& graph, pos_t position, size_t context) { std::set result; struct Comparator { bool operator()(const std::pair& a, const std::pair& b) const { return a.first > b.first; } }; // (distance, position) std::priority_queue, std::vector>, Comparator> queue; queue.push(std::make_pair(0, position)); while(!queue.empty()) { std::pair current = queue.top(); queue.pop(); if(result.find(id(current.second)) != result.end()) { continue; } result.insert(id(current.second)); for(bool is_reverse : { false, true }) { handle_t handle = graph.get_handle(id(current.second), is_reverse); size_t distance = current.first; if(is_reverse == is_rev(current.second)) { distance += graph.get_length(handle) - offset(current.second); } else { distance += offset(current.second) + 1; } if(distance <= context) { graph.follow_edges(handle, false, [&](const handle_t& next) { queue.push(std::make_pair(distance, make_pos_t(graph.get_id(next), graph.get_is_reverse(next), 0))); }); } } } return result; } //------------------------------------------------------------------------------ /* Helper functions for determining CIGAR strings relative to the reference. */ void append_edit(std::vector>& edits, char op, size_t length) { if(length == 0) { return; } if(!edits.empty() && edits.back().first == op) { edits.back().second += length; } else { edits.emplace_back(op, length); } } size_t subpath_length(const GBWTGraph& graph, subpath_type subpath) { size_t result = 0; for(size_t i = 0; i < subpath.second; i++) { result += graph.get_length(GBWTGraph::node_to_handle(subpath.first[i])); } return result; } // Returns the number of matching bases at the start of the two subpaths. size_t prefix_matches(const GBWTGraph& graph, subpath_type a, subpath_type b) { size_t result = 0; size_t a_offset = 0, b_offset = 0; size_t a_base = 0, b_base = 0; while(a_offset < a.second && b_offset < b.second) { std::string_view a_seq = graph.get_sequence_view(GBWTGraph::node_to_handle(a.first[a_offset])); std::string_view b_seq = graph.get_sequence_view(GBWTGraph::node_to_handle(b.first[b_offset])); while(a_base < a_seq.size() && b_base < b_seq.size()) { if(a_seq[a_base] != b_seq[b_base]) { return result; } result++; a_base++; b_base++; } if(a_base == a_seq.size()) { a_offset++; a_base = 0; } if(b_base == b_seq.size()) { b_offset++; b_base = 0; } } return result; } // Returns the number of matching bases at the end of the two subpaths. size_t suffix_matches(const GBWTGraph& graph, subpath_type a, subpath_type b) { size_t result = 0; size_t a_offset = a.second, b_offset = b.second; size_t a_base = 0, b_base = 0; while(a_offset > 0 && b_offset > 0) { std::string_view a_seq = graph.get_sequence_view(GBWTGraph::node_to_handle(a.first[a_offset - 1])); std::string_view b_seq = graph.get_sequence_view(GBWTGraph::node_to_handle(b.first[b_offset - 1])); while(a_base < a_seq.size() && b_base < b_seq.size()) { if(a_seq[a_seq.size() - 1 - a_base] != b_seq[b_seq.size() - 1 - b_base]) { return result; } result++; a_base++; b_base++; } if(a_base == a_seq.size()) { a_offset--; a_base = 0; } if(b_base == b_seq.size()) { b_offset--; b_base = 0; } } return result; } size_t mismatch_penalty(size_t length) { return 4 * length; } size_t gap_penalty(size_t length) { if(length == 0) { return 0; } return 6 + (length - 1); } void align_diverging( const GBWTGraph& graph, subpath_type path, subpath_type reference, std::vector>& edits ) { size_t path_length = subpath_length(graph, path); size_t ref_length = subpath_length(graph, reference); size_t prefix = prefix_matches(graph, reference, path); size_t suffix = suffix_matches(graph, reference, path); if(prefix + suffix > path_length) { suffix = path_length - prefix; } if(prefix + suffix > ref_length) { suffix = ref_length - prefix; } append_edit(edits, 'M', prefix); // Match. size_t path_middle = path_length - prefix - suffix; size_t ref_middle = ref_length - prefix - suffix; if(path_middle == 0) { append_edit(edits, 'D', ref_middle); // Deletion. } else if(ref_middle == 0) { append_edit(edits, 'I', path_middle); // Insertion. } else { size_t mismatch = std::min(ref_middle, path_middle); size_t mismatch_indel = mismatch_penalty(mismatch) + gap_penalty(ref_middle - mismatch) + gap_penalty(path_middle - mismatch); size_t insertion_deletion = gap_penalty(ref_middle) + gap_penalty(path_middle); if(mismatch_indel <= insertion_deletion) { append_edit(edits, 'M', mismatch); // Mismatch. append_edit(edits, 'I', path_middle - mismatch); // Insertion. append_edit(edits, 'D', ref_middle - mismatch); // Deletion. } else { append_edit(edits, 'I', path_middle); // Insertion. append_edit(edits, 'D', ref_middle); // Deletion. } } append_edit(edits, 'M', suffix); // Match. } std::string to_cigar(const std::vector>& edits) { std::string result; for(std::pair edit : edits) { result += std::to_string(edit.second); result.push_back(edit.first); } return result; } // Returns a CIGAR string. std::string align_path(const GBWTGraph& graph, const gbwt::vector_type& reference, const gbwt::vector_type& path) { // Find the LCS weighted by sequence length. // The values are (path offset, reference offset). std::vector> lcs = path_lcs(graph, path, reference); // Convert the LCS to a sequence of edits. std::vector> edits; size_t path_offset = 0, ref_offset = 0; for(const std::pair& match : lcs) { align_diverging( graph, get_subpath(path, path_offset, match.first), get_subpath(reference, ref_offset, match.second), edits ); append_edit(edits, 'M', graph.get_length(GBWTGraph::node_to_handle(path[match.first]))); path_offset = match.first + 1; ref_offset = match.second + 1; } align_diverging( graph, get_subpath(path, path_offset, path.size()), get_subpath(reference, ref_offset, reference.size()), edits ); return to_cigar(edits); } //------------------------------------------------------------------------------ /* Private member functions used in the constructor. */ void Subgraph::extract_paths(const GBZ& gbz, const SubgraphQuery& query, size_t query_offset, const std::pair& ref_pos) { const GBWTGraph& graph = gbz.graph; const gbwt::GBWT& index = gbz.index; // For each GBWT node in the subgraph, mark which GBWT offsets have a // predecessor in the subgraph. std::map has_predecessor; for(nid_t node : this->nodes) { for(bool is_reverse : { false, true }) { gbwt::node_type gbwt_node = gbwt::Node::encode(node, is_reverse); gbwt::CompressedRecord record = index.record(gbwt_node); has_predecessor[gbwt_node] = sdsl::bit_vector(record.size(), 0); } } // Mark the positions that have predecessors in the subgraph. for(nid_t node : this->nodes) { for(bool is_reverse : { false, true }) { gbwt::node_type gbwt_node = gbwt::Node::encode(node, is_reverse); gbwt::CompressedRecord record = index.record(gbwt_node); gbwt::DecompressedRecord decompressed(record); for(size_t i = 0; i < decompressed.size(); i++) { gbwt::edge_type successor = decompressed.LF(i); auto iter = has_predecessor.find(successor.first); if(iter != has_predecessor.end()) { iter->second[successor.second] = true; } } } } // FIXME: If the GBWT index is invalid, we could have infinite loops. // Extract all paths and determine reference path information if necessary. for(const auto& start_node : has_predecessor) { for(size_t gbwt_offset = 0; gbwt_offset < start_node.second.size(); gbwt_offset++) { if(start_node.second[gbwt_offset]) { continue; } // There is a predecessor. gbwt::edge_type curr(start_node.first, gbwt_offset); bool is_ref = false; gbwt::vector_type path; size_t path_length = 0; while(curr.first != gbwt::ENDMARKER && has_predecessor.find(curr.first) != has_predecessor.end()) { if(curr == ref_pos.second) { this->reference_path = this->paths.size(); this->reference_start = query_offset - path_length - offset(ref_pos.first); is_ref = true; } path.push_back(curr.first); path_length += graph.get_length(GBWTGraph::node_to_handle(curr.first)); curr = index.LF(curr); } if(is_ref) { if(!path_is_canonical(path)) { // TODO: Do we want this? std::cerr << "Subgraph::Subgraph(): Warning: Reference path for query " << query.to_string() << " is not in canonical orientation" << std::endl; } this->paths.push_back(path); this->path_lengths.push_back(path_length); } else if(path_is_canonical(path)) { this->paths.push_back(path); this->path_lengths.push_back(path_length); } } } if(ref_pos.second != gbwt::invalid_edge() && this->reference_path == std::numeric_limits::max()) { std::string msg = "Subgraph::Subgraph(): Reference path not found in the subgraph"; throw std::runtime_error(msg); } } void Subgraph::update_paths(const SubgraphQuery& query) { if(query.haplotype_output == SubgraphQuery::HaplotypeOutput::all_haplotypes) { return; } if(query.haplotype_output == SubgraphQuery::HaplotypeOutput::reference_only) { std::vector new_paths { this->paths[this->reference_path] }; this->paths.swap(new_paths); this->reference_path = 0; return; } // TODO: This could be more efficient by using normal GBWT operations. if(query.haplotype_output == SubgraphQuery::HaplotypeOutput::distinct_haplotypes) { gbwt::vector_type ref_path = (this->reference_path < this->paths.size() ? this->paths[this->reference_path] : gbwt::vector_type(1, gbwt::ENDMARKER)); std::vector order; order.reserve(this->paths.size()); for(size_t i = 0; i < this->paths.size(); i++) { order.push_back(i); } std::sort(order.begin(), order.end(), [&](size_t a, size_t b) -> bool { return (this->paths[a] < this->paths[b]); }); std::vector new_paths; std::vector new_lengths; for(size_t i = 0; i < this->paths.size(); i++) { const gbwt::vector_type& path = this->paths[order[i]]; if(new_paths.empty() || path != new_paths.back()) { if(path == ref_path) { this->reference_path = new_paths.size(); } new_paths.push_back(path); new_lengths.push_back(this->path_lengths[order[i]]); this->path_weights.push_back(1); } else { this->path_weights.back()++; } } this->paths.swap(new_paths); this->path_lengths.swap(new_lengths); } } //------------------------------------------------------------------------------ Subgraph::Subgraph(const GBZ& gbz, const PathIndex* path_index, const SubgraphQuery& query) : reference_path(std::numeric_limits::max()), reference_start(0) { std::pair position; size_t context = query.context; size_t query_offset = 0; switch(query.type) { case SubgraphQuery::QueryType::path_offset_query: { if(path_index == nullptr) { throw std::runtime_error("Subgraph::Subgraph(): Path index required for path queries"); } const gbwt::Metadata& metadata = gbz.index.metadata; gbwt::size_type path_id = metadata.findFragment(query.path_query); if(path_id >= metadata.paths()) { std::string msg = "Subgraph::Subgraph(): Could not find a path fragment for sample " + query.path_query.sample_name + ", contig " + query.path_query.contig_name + ", haplotype " + std::to_string(query.path_query.haplotype) + ", offset " + std::to_string(query.path_query.offset); throw std::runtime_error(msg); } path_handle_t reference_handle = gbz.graph.path_to_handle(path_id); this->reference_name = metadata.fullPath(path_id); query_offset = query.path_query.offset - this->reference_name.offset; position = find_position(gbz, *path_index, reference_handle, query_offset); break; } case SubgraphQuery::QueryType::node_query: { // TODO: This has the same issue as in GBZ-base. If node length is even, // the right context is one bp too long. What about odd lengths? size_t node_len = gbz.graph.get_length(gbz.graph.get_handle(query.node_id)); position.first = make_pos_t(query.node_id, false, node_len / 2); position.second = gbwt::invalid_edge(); context += node_len / 2; break; } default: { // NOTE: `path_interval_query` is currently implemented as a `path_offset_query` around the midpoint. throw std::runtime_error("Subgraph::Subgraph(): Invalid query type"); } } this->nodes = find_subgraph(gbz.graph, position.first, context); this->extract_paths(gbz, query, query_offset, position); this->update_paths(query); if(this->reference_path < this->paths.size()) { for(size_t i = 0; i < this->paths.size(); i++) { if(i == this->reference_path) { this->path_cigars.push_back(""); } else { this->path_cigars.push_back(align_path(gbz.graph, this->paths[this->reference_path], this->paths[i])); } } } } //------------------------------------------------------------------------------ const size_t* Subgraph::weight(size_t path_id) const { if(path_id >= this->path_weights.size()) { return nullptr; } return &(this->path_weights[path_id]); } const std::string* Subgraph::cigar(size_t path_id) const { if(path_id >= this->path_cigars.size()) { return nullptr; } if(path_id == this->reference_path) { return nullptr; } return &(this->path_cigars[path_id]); } void Subgraph::to_gfa(const GBZ& gbz, std::ostream& out) const { TSVWriter writer(out); // GFA header. if(this->reference_path < this->paths.size()) { write_gfa_header(writer, &(this->reference_name.sample_name)); } else { write_gfa_header(writer, nullptr); } // Nodes / segments. for(nid_t node : this->nodes) { handle_t handle = gbz.graph.get_handle(node); write_gfa_segment(writer, node, gbz.graph.get_sequence_view(handle)); } // Edges / links. for(nid_t node : this->nodes) { for(bool is_reverse : { false, true }) { gbwt::node_type from = gbwt::Node::encode(node, is_reverse); handle_t handle = GBWTGraph::node_to_handle(from); gbz.graph.follow_edges(handle, false, [&](const handle_t& next) { gbwt::node_type to = GBWTGraph::handle_to_node(next); nid_t successor = gbwt::Node::id(to); if(this->nodes.find(successor) != this->nodes.end() && edge_is_canonical(from, to)) { write_gfa_link(writer, from, to); } }); } } // W-lines: reference path std::string contig_name = "unknown"; if(this->reference_path < this->paths.size()) { const gbwt::vector_type& path = this->paths[this->reference_path]; gbwt::FullPathName path_name = this->reference_name; // Offset is for the path relative to the entire haplotype. // We transform it into the starting offset of the path fragment within the subgraph relative to the entire haplotype. path_name.offset += this->reference_start; contig_name = path_name.contig_name; size_t length = this->path_lengths[this->reference_path]; const size_t* weight = this->weight(this->reference_path); const std::string* cigar = this->cigar(this->reference_path); write_gfa_walk(writer, path, path_name, length, weight, cigar); } // W-lines: anonymous haplotypes size_t haplotype_id = 1; for(size_t i = 0; i < this->paths.size(); i++) { if(i == this->reference_path) { continue; } const gbwt::vector_type& path = this->paths[i]; gbwt::FullPathName path_name { "unknown", contig_name, haplotype_id, 0 }; size_t length = this->path_lengths[i]; const size_t* weight = this->weight(i); const std::string* cigar = this->cigar(i); write_gfa_walk(writer, path, path_name, length, weight, cigar); haplotype_id++; } writer.flush(); } //------------------------------------------------------------------------------ } // namespace gbwtgraph jltsiren-gbwtgraph-e27bc43/src/subgraph_query.cpp000066400000000000000000000155541524136466400223220ustar00rootroot00000000000000#include #include #include #include #include #include #include using namespace gbwtgraph; //------------------------------------------------------------------------------ const std::string tool_name = "Subgraph Query"; struct Config { Config(int argc, char** argv); std::string graph_file; SubgraphQuery::QueryType query_type = SubgraphQuery::QueryType::invalid_query; SubgraphQuery::HaplotypeOutput haplotype_output = SubgraphQuery::HaplotypeOutput::all_haplotypes; gbwt::FullPathName path_query { GENERIC_PATH_SAMPLE_NAME, "", GBWTGraph::NO_PHASE, 0 }; size_t offset = 0, limit = 0; nid_t node_id = 0; size_t context = 100; }; std::unique_ptr create_path_index(const GBZ& gbz, const SubgraphQuery& query) { if(query.type == SubgraphQuery::QueryType::path_offset_query || query.type == SubgraphQuery::QueryType::path_interval_query) { return std::make_unique(gbz); } return nullptr; } SubgraphQuery create_query(const Config& config) { switch(config.query_type) { case SubgraphQuery::QueryType::path_offset_query: return SubgraphQuery::path_offset(config.path_query, config.offset, config.context, config.haplotype_output); case SubgraphQuery::QueryType::path_interval_query: return SubgraphQuery::path_interval(config.path_query, config.offset, config.limit, config.context, config.haplotype_output); case SubgraphQuery::QueryType::node_query: return SubgraphQuery::node(config.node_id, config.context, config.haplotype_output); default: throw std::runtime_error("Unknown query type"); } } //------------------------------------------------------------------------------ int main(int argc, char** argv) { double start = gbwt::readTimer(); try { Config config(argc, argv); GBZ gbz; sdsl::simple_sds::load_from(gbz, config.graph_file); SubgraphQuery query = create_query(config); std::unique_ptr path_index = create_path_index(gbz, query); Subgraph subgraph(gbz, path_index.get(), query); subgraph.to_gfa(gbz, std::cout); std::cout.flush(); } catch(const std::runtime_error& e) { std::cerr << "subgraph_query: " << e.what() << std::endl; } double seconds = gbwt::readTimer() - start; std::cerr << "Used " << seconds << " seconds, " << gbwt::inGigabytes(gbwt::memoryUsage()) << " GiB" << std::endl; return 0; } //------------------------------------------------------------------------------ void printUsage(int exit_code) { Version::print(std::cerr, tool_name); std::cerr << "Usage: subgraph_query [options] graph.gbz" << std::endl; std::cerr << std::endl; std::cerr << "Options:" << std::endl; std::cerr << " --sample NAME sample name for the reference path (default: no sample name)" << std::endl; std::cerr << " --contig NAME contig name for the reference path (required for --offset and --interval)" << std::endl; std::cerr << " --offset N query a reference path at offset N" << std::endl; std::cerr << " --interval M..N query a reference path in interval [M, N)" << std::endl; std::cerr << " --node N query a node with id N" << std::endl; std::cerr << " --context N context length around the query position in bp (default: 100)" << std::endl; std::cerr << " --distinct output distinct haplotypes only" << std::endl; std::cerr << " --reference-only only output the reference path" << std::endl; std::cerr << std::endl; std::exit(exit_code); } //------------------------------------------------------------------------------ Config::Config(int argc, char** argv) { if(argc < 2) { printUsage(EXIT_SUCCESS); } constexpr int OPT_SAMPLE = 1000; constexpr int OPT_CONTIG = 1001; constexpr int OPT_OFFSET = 1002; constexpr int OPT_INTERVAL = 1003; constexpr int OPT_NODE = 1004; constexpr int OPT_CONTEXT = 1005; constexpr int OPT_DISTINCT = 1006; constexpr int OPT_REFERENCE_ONLY = 1007; // Data for `getopt_long()`. int c = 0, option_index = 0; option long_options[] = { { "sample", required_argument, 0, OPT_SAMPLE }, { "contig", required_argument, 0, OPT_CONTIG }, { "offset", required_argument, 0, OPT_OFFSET }, { "interval", required_argument, 0, OPT_INTERVAL }, { "node", required_argument, 0, OPT_NODE }, { "context", required_argument, 0, OPT_CONTEXT }, { "distinct", no_argument, 0, OPT_DISTINCT }, { "reference-only", no_argument, 0, OPT_REFERENCE_ONLY }, { 0, 0, 0, 0 } }; // Process options. while((c = getopt_long(argc, argv, "", long_options, &option_index)) != -1) { switch(c) { case OPT_SAMPLE: this->path_query.sample_name = optarg; // By specifying a sample, we make the path a haplotype path rather than a generic path. // Therefore the haplotype must be a real haplotype rather than the NO_PHASE value. this->path_query.haplotype = 0; break; case OPT_CONTIG: this->path_query.contig_name = optarg; break; case OPT_OFFSET: this->query_type = SubgraphQuery::QueryType::path_offset_query; this->offset = std::stoul(optarg); break; case OPT_INTERVAL: this->query_type = SubgraphQuery::QueryType::path_interval_query; { std::string interval(optarg); size_t pos = interval.find(".."); if(pos == std::string::npos) { std::string msg = "Invalid path interval: " + interval; throw std::runtime_error(msg); } this->offset = std::stoul(interval.substr(0, pos)); this->limit = std::stoul(interval.substr(pos + 2)); } break; case OPT_NODE: this->query_type = SubgraphQuery::QueryType::node_query; this->node_id = std::stoul(optarg); break; case OPT_CONTEXT: this->context = std::stoul(optarg); break; case OPT_DISTINCT: this->haplotype_output = SubgraphQuery::HaplotypeOutput::distinct_haplotypes; break; case OPT_REFERENCE_ONLY: this->haplotype_output = SubgraphQuery::HaplotypeOutput::reference_only; break; case '?': std::exit(EXIT_FAILURE); default: std::exit(EXIT_FAILURE); } } // Sanity checks. if(optind >= argc) { std::string msg = "Missing graph file"; throw std::runtime_error(msg); } this->graph_file = argv[optind]; optind++; if(this->query_type == SubgraphQuery::QueryType::invalid_query) { std::string msg = "Path offset or interval or node id is required"; throw std::runtime_error(msg); } if((this->query_type == SubgraphQuery::QueryType::path_offset_query || this->query_type == SubgraphQuery::QueryType::path_interval_query) && this->path_query.contig_name.empty()) { std::string msg = "Contig name is required for path offset or interval"; throw std::runtime_error(msg); } } //------------------------------------------------------------------------------ jltsiren-gbwtgraph-e27bc43/src/utils.cpp000066400000000000000000001256341524136466400204230ustar00rootroot00000000000000#include #include #include #include #include #include #include namespace gbwtgraph { //------------------------------------------------------------------------------ // Numerical class constants. constexpr size_t Version::MAJOR_VERSION; constexpr size_t Version::MINOR_VERSION; constexpr size_t Version::PATCH_VERSION; constexpr size_t Version::GBZ_VERSION; constexpr size_t Version::GRAPH_VERSION; constexpr size_t Version::MINIMIZER_VERSION; constexpr char GraphName::GFA_HEADER_PREFIX; constexpr char GraphName::GAF_HEADER_PREFIX; constexpr char GraphName::GFA_GAF_FIELD_SEPARATOR; constexpr char GraphName::GFA_GAF_TAG_SEPARATOR; constexpr char GraphName::GFA_GAF_TAG_STR_TYPE; constexpr char GraphName::RELATIONSHIP_SEPARATOR; constexpr char GraphName::RELATIONSHIP_LIST_SEPARATOR; constexpr size_t MetadataBuilder::NO_FIELD; //------------------------------------------------------------------------------ // Global variables. const std::string GENERIC_PATH_SAMPLE_NAME = "_gbwt_ref"; const std::string REFERENCE_PATH_SAMPLE_NAME = "_gbwt_ref"; const std::string REFERENCE_SAMPLE_LIST_GBWT_TAG = "reference_samples"; const std::string REFERENCE_SAMPLE_LIST_GFA_TAG = "RS"; // Since spaces are allowed in GFA tags, we can use them as sample name separators. const char REFERENCE_SAMPLE_LIST_SEPARATOR = ' '; const std::string COVER_PATH_SAMPLE_PREFIX = "path_cover_"; //------------------------------------------------------------------------------ // Other class variables. const std::string Version::SOURCE_KEY = "source"; const std::string Version::SOURCE_VALUE = "jltsiren/gbwtgraph"; const std::string GraphName::GBZ_NAME_TAG = "pggname"; const std::string GraphName::GBZ_SUBGRAPH_TAG = "subgraph"; const std::string GraphName::GBZ_TRANSLATION_TAG = "translation"; const std::string GraphName::GBZ_TRANSLATION_TARGET_TAG = "translation_target"; const std::string GraphName::GFA_NAME_TAG = "NM"; const std::string GraphName::GAF_NAME_TAG = "RN"; const std::string GraphName::GFA_GAF_SUBGRAPH_TAG = "SG"; const std::string GraphName::GFA_GAF_TRANSLATION_TAG = "TL"; //------------------------------------------------------------------------------ std::vector split_view(std::string_view str_view, char separator) { std::vector result; const char* cursor = str_view.begin(); const char* end = str_view.end(); while(cursor != end) { // Find the next occurrence of the separator. const char* next = std::find(cursor, end, separator); result.emplace_back(cursor, next - cursor); cursor = next; if(cursor != end) { // Advance past the separator. ++cursor; } } return result; } //------------------------------------------------------------------------------ sample_name_set parse_reference_samples_tag(const char* cursor, const char* end) { return parse_reference_samples_tag(std::string_view(cursor, end - cursor)); } sample_name_set parse_reference_samples_tag(std::string_view tag_value) { std::vector sample_names = split_view(tag_value, REFERENCE_SAMPLE_LIST_SEPARATOR); sample_name_set reference_samples; for(const auto& sample_view : sample_names) { reference_samples.emplace(sample_view.data(), sample_view.size()); } return reference_samples; } sample_name_set parse_reference_samples_tag(const gbwt::GBWT& index) { return parse_reference_samples_tag(index.tags.get(REFERENCE_SAMPLE_LIST_GBWT_TAG)); } sample_name_set present_sample_names(const sample_name_set& sample_names, const gbwt::GBWT& index) { sample_name_set result; for(const auto& sample_name : sample_names) { if(index.metadata.sample(sample_name) < index.metadata.samples()) { result.insert(sample_name); } } return result; } std::string compose_reference_samples_tag(const sample_name_set& reference_samples) { std::string result; for(const std::string& sample_name : reference_samples) { if(!result.empty()) { result.push_back(REFERENCE_SAMPLE_LIST_SEPARATOR); } result.append(sample_name); } return result; } PathSense get_path_sense(const gbwt::Metadata& metadata, const gbwt::PathName& path_name, const sample_name_set& reference_samples) { return get_sample_sense(metadata, path_name.sample, reference_samples); } PathSense get_path_sense(const gbwt::GBWT& index, gbwt::size_type path_number, const sample_name_set& reference_samples) { if(!index.hasMetadata() || !index.metadata.hasPathNames() || path_number >= index.metadata.paths()) { return PathSense::HAPLOTYPE; } return get_path_sense(index.metadata, index.metadata.path(path_number), reference_samples); } PathSense get_sample_sense(const gbwt::Metadata& metadata, gbwt::size_type sample, const sample_name_set& reference_samples) { if(!metadata.hasSampleNames() || sample >= metadata.sample_names.size()) { // If there are no sample names, everything is a haplotype. return PathSense::HAPLOTYPE; } return get_sample_sense(metadata.sample(sample), reference_samples); } PathSense get_sample_sense(const std::string& sample_name, const sample_name_set& reference_samples) { if(sample_name == GENERIC_PATH_SAMPLE_NAME) { // Paths with the magic sample are generic named paths. return PathSense::GENERIC; } if(reference_samples.count(sample_name)) { // Paths with these samples are reference paths return PathSense::REFERENCE; } // Other paths are haplotypes. return PathSense::HAPLOTYPE; } std::string get_path_sample_name(const gbwt::Metadata& metadata, const gbwt::PathName& path_name, PathSense sense) { if(sense == PathSense::GENERIC) { // The libhandlegraph sample name should be the no-sample sentinel return PathMetadata::NO_SAMPLE_NAME; } if(!metadata.hasSampleNames()) { // If there are no sample names, use the sample number. return std::to_string(path_name.sample); } // Otherwise return what we have stored. return metadata.sample(path_name.sample); } std::string get_path_sample_name(const gbwt::GBWT& index, gbwt::size_type path_number, PathSense sense) { if(!index.hasMetadata() || !index.metadata.hasPathNames() || path_number >= index.metadata.paths()) { return PathMetadata::NO_SAMPLE_NAME; } return get_path_sample_name(index.metadata, index.metadata.path(path_number), sense); } // We want to pass all the fields to all of these in case we need to evolve storage format later. std::string get_path_locus_name(const gbwt::Metadata& metadata, const gbwt::PathName& path_name, [[maybe_unused]] PathSense sense) { if(!metadata.hasContigNames() || path_name.contig > metadata.contig_names.size()) { // If there are no contig names, use the contig number. return std::to_string(path_name.contig); } // This is the same for all senses. return metadata.contig(path_name.contig); } std::string get_path_locus_name(const gbwt::GBWT& index, gbwt::size_type path_number, PathSense sense) { if(!index.hasMetadata() || !index.metadata.hasPathNames() || path_number >= index.metadata.paths()) { return PathMetadata::NO_LOCUS_NAME; } return get_path_locus_name(index.metadata, index.metadata.path(path_number), sense); } size_t get_path_haplotype([[maybe_unused]] const gbwt::Metadata& metadata, const gbwt::PathName& path_name, PathSense sense) { if(sense == PathSense::GENERIC) { // Generic paths aren't allowed haplotypes return PathMetadata::NO_HAPLOTYPE; } // Otherwise it's just stored, but we need to detect the sentinel return path_name.phase == gbwtgraph::GBWTGraph::NO_PHASE ? PathMetadata::NO_HAPLOTYPE : path_name.phase; } size_t get_path_haplotype(const gbwt::GBWT& index, gbwt::size_type path_number, PathSense sense) { if(!index.hasMetadata() || !index.metadata.hasPathNames() || path_number >= index.metadata.paths()) { return PathMetadata::NO_HAPLOTYPE; } return get_path_haplotype(index.metadata, index.metadata.path(path_number), sense); } size_t get_path_phase_block([[maybe_unused]] const gbwt::Metadata& metadata, const gbwt::PathName& path_name, PathSense sense) { if(sense == PathSense::HAPLOTYPE) { // Only haplotype paths have phase blocks return path_name.count; } return PathMetadata::NO_PHASE_BLOCK; } size_t get_path_phase_block(const gbwt::GBWT& index, gbwt::size_type path_number, PathSense sense) { if(!index.hasMetadata() || !index.metadata.hasPathNames() || path_number >= index.metadata.paths()) { return PathMetadata::NO_PHASE_BLOCK; } return get_path_phase_block(index.metadata, index.metadata.path(path_number), sense); } subrange_t get_path_subrange([[maybe_unused]] const gbwt::Metadata& metadata, const gbwt::PathName& path_name, PathSense sense) { subrange_t subrange = PathMetadata::NO_SUBRANGE; if(sense != PathSense::HAPLOTYPE && path_name.count != 0) { // If we aren't a haplotype and we have a nonzero count, we use count to // store the subrange start. subrange.first = path_name.count; } return subrange; } subrange_t get_path_subrange(const gbwt::GBWT& index, gbwt::size_type path_number, PathSense sense) { if(!index.hasMetadata() || !index.metadata.hasPathNames() || path_number >= index.metadata.paths()) { return PathMetadata::NO_SUBRANGE; } return get_path_subrange(index.metadata, index.metadata.path(path_number), sense); } std::string compose_path_name(const gbwt::Metadata& metadata, const gbwt::PathName& path_name, PathSense sense) { // Put everything together into a string path name the way PathMetadata suggests. return PathMetadata::create_path_name( sense, get_path_sample_name(metadata, path_name, sense), get_path_locus_name(metadata, path_name, sense), get_path_haplotype(metadata, path_name, sense), get_path_phase_block(metadata, path_name, sense), get_path_subrange(metadata, path_name, sense) ); } std::string compose_path_name(const gbwt::GBWT& index, gbwt::size_type path_number, PathSense sense) { if(!index.hasMetadata() || !index.metadata.hasPathNames() || path_number >= index.metadata.paths()) { return ""; } return compose_path_name(index.metadata, index.metadata.path(path_number), sense); } void set_sample_path_senses(gbwt::Tags& tags, const std::unordered_map& senses) { // Find all the current reference samples sample_name_set reference_sample_names = parse_reference_samples_tag(tags.get(REFERENCE_SAMPLE_LIST_GBWT_TAG)); for(auto& kv : senses) { if(kv.first == PathMetadata::NO_SAMPLE_NAME) { // If the sample isn't set, it must be a generic named path. if(kv.second != PathSense::GENERIC) { throw std::runtime_error("Cannot store a sense other than generic in GBWT tags for the no-sample sample."); } continue; } if(kv.second == PathSense::REFERENCE) { // Add new reference samples reference_sample_names.insert(kv.first); } else if(kv.second == PathSense::HAPLOTYPE) { // And remove any that change sense back to haplotype reference_sample_names.erase(kv.first); } else { // We can't actually set this sense. throw std::runtime_error("Cannot store sense " + std::to_string((int)kv.second) + " in GBWT tags for sample " + kv.first); } } tags.set(REFERENCE_SAMPLE_LIST_GBWT_TAG, compose_reference_samples_tag(reference_sample_names)); } //------------------------------------------------------------------------------ std::string Version::str(bool verbose) { std::ostringstream ss; if(verbose) { ss << "GBWTGraph version "; } else { ss << "v"; } ss << MAJOR_VERSION << "." << MINOR_VERSION << "." << PATCH_VERSION; if(verbose) { ss << " (file format version " << GRAPH_VERSION << ")"; } return ss.str(); } void Version::print(std::ostream& out, const std::string& tool_name, bool verbose, size_t new_lines) { out << tool_name; if(verbose) { out << std::endl; } else { out << " "; } out << str(verbose); for(size_t i = 0; i < new_lines; i++) { out << std::endl; } } //------------------------------------------------------------------------------ DigestBuf::DigestBuf(const EVP_MD* algorithm) : std::streambuf(), context(EVP_MD_CTX_new()) { if(this->context != nullptr && EVP_DigestInit_ex(this->context, algorithm, nullptr) == -1) { EVP_MD_CTX_free(this->context); this->context = nullptr; } } DigestBuf::~DigestBuf() { if(this->context != nullptr) { EVP_MD_CTX_free(this->context); this->context = nullptr; } } int DigestBuf::overflow(int_type ch) { if(this->context == nullptr) { return traits_type::eof(); } if(ch != traits_type::eof()) { unsigned char buf = static_cast(ch); if(EVP_DigestUpdate(this->context, &buf, sizeof(buf)) == -1) { return traits_type::eof(); } } return ch; } std::streamsize DigestBuf::xsputn(const char* s, std::streamsize n) { if(this->context == nullptr) { return 0; } if(EVP_DigestUpdate(this->context, s, static_cast(n)) == -1) { return 0; } return n; } std::string DigestBuf::finish() { if(this->context == nullptr) { return ""; } unsigned char md_value[EVP_MAX_MD_SIZE]; unsigned int md_len = 0; if(EVP_DigestFinal_ex(this->context, md_value, &md_len) == -1) { return ""; } EVP_MD_CTX_free(this->context); this->context = nullptr; std::ostringstream ss; ss << std::hex; for(unsigned int i = 0; i < md_len; i++) { ss.width(2); ss.fill('0'); ss << static_cast(md_value[i]); } return ss.str(); } DigestStream::DigestStream(const EVP_MD* algorithm) : std::ostream(&buffer), buffer(algorithm) { if(!this->buffer.good()) { this->setstate(std::ios::badbit); } } std::string DigestStream::finish() { std::string digest = this->buffer.finish(); this->setstate(std::ios::badbit); return digest; } //------------------------------------------------------------------------------ std::vector parse_relationship(std::string_view entry, const std::string& type) { std::vector names = split_view(entry, GraphName::RELATIONSHIP_SEPARATOR); if(names.size() != 2 || names[0].empty() || names[1].empty()) { std::string msg = "Cannot parse " + type + " relationship: " + std::string(entry); throw std::runtime_error(msg); } return names; } GraphName::GraphName(const gbwt::Tags& tags) : pggname(tags.get(GBZ_NAME_TAG)) { std::string subgraph_tag = tags.get(GBZ_SUBGRAPH_TAG); std::vector subgraph_entries = split_view(std::string_view(subgraph_tag), RELATIONSHIP_LIST_SEPARATOR); for(const auto& entry_view : subgraph_entries) { std::vector names = parse_relationship(entry_view, "subgraph"); this->add_subgraph(names[0], names[1]); } std::string translation_tag = tags.get(GBZ_TRANSLATION_TAG); std::vector translation_entries = split_view(std::string_view(translation_tag), RELATIONSHIP_LIST_SEPARATOR); for(const auto& entry_view : translation_entries) { std::vector names = parse_relationship(entry_view, "translation"); this->add_translation(names[0], names[1]); } } // TODO: This should be in gfa.cpp/.h. std::tuple parse_typed_field(std::string_view field) { if(field.size() < 5 || field[2] != ':' || field[4] != ':') { std::string msg = "Cannot parse typed field: " + std::string(field); throw std::runtime_error(msg); } std::string_view tag(field.data(), 2); char type = field[3]; std::string_view value(field.data() + 5, field.size() - 5); return std::make_tuple(tag, type, value); } GraphName::GraphName(const std::vector& header_lines) { for(const std::string& line : header_lines) { std::vector fields = split_view(std::string_view(line), GFA_GAF_FIELD_SEPARATOR); if(fields.empty()) { continue; } if(fields[0].size() == 1 && fields[0][0] == GFA_HEADER_PREFIX) { for(size_t i = 1; i < fields.size(); i++) { auto parsed = parse_typed_field(fields[i]); if(std::get<0>(parsed) == GFA_NAME_TAG && std::get<1>(parsed) == GFA_GAF_TAG_STR_TYPE) { this->pggname = std::string(std::get<2>(parsed)); } else if(std::get<0>(parsed) == GFA_GAF_SUBGRAPH_TAG && std::get<1>(parsed) == GFA_GAF_TAG_STR_TYPE) { std::vector names = parse_relationship(std::get<2>(parsed), "subgraph"); this->add_subgraph(names[0], names[1]); } else if(std::get<0>(parsed) == GFA_GAF_TRANSLATION_TAG && std::get<1>(parsed) == GFA_GAF_TAG_STR_TYPE) { std::vector names = parse_relationship(std::get<2>(parsed), "translation"); this->add_translation(names[0], names[1]); } } } else if(fields[0].size() > 1 && fields[0][0] == GAF_HEADER_PREFIX) { fields[0].remove_prefix(1); // Skip '@'. if(fields[0] == GAF_NAME_TAG) { if(fields.size() < 2) { std::string msg = "Cannot parse GAF name line: " + line; throw std::runtime_error(msg); } this->pggname = std::string(fields[1]); } else if(fields[0] == GFA_GAF_SUBGRAPH_TAG) { if(fields.size() < 3) { std::string msg = "Cannot parse GAF subgraph line: " + line; throw std::runtime_error(msg); } this->add_subgraph(fields[1], fields[2]); } else if(fields[0] == GFA_GAF_TRANSLATION_TAG) { if(fields.size() < 3) { std::string msg = "Cannot parse GAF translation line: " + line; throw std::runtime_error(msg); } this->add_translation(fields[1], fields[2]); } } else { std::string msg = "Cannot parse header line: " + line; throw std::runtime_error(msg); } } } void GraphName::add_subgraph(std::string_view subgraph, std::string_view supergraph) { if(subgraph == supergraph || subgraph.empty() || supergraph.empty()) { return; } this->subgraph[std::string(subgraph)].insert(std::string(supergraph)); } void GraphName::add_translation(std::string_view from, std::string_view to) { if(from == to || from.empty() || to.empty()) { return; } this->translation[std::string(from)].insert(std::string(to)); } void GraphName::add_relationships(const GraphName& another) { for(const auto& kv : another.subgraph) { for(const auto& supergraph_name : kv.second) { this->add_subgraph(kv.first, supergraph_name); } } for(const auto& kv : another.translation) { for(const auto& to_name : kv.second) { this->add_translation(kv.first, to_name); } } } void compose_relationship_list(const std::map>& relationships, gbwt::Tags& tags, const std::string& tag_name) { std::string relationship_tag; for(const auto& kv : relationships) { for(const auto& related_name : kv.second) { if(!relationship_tag.empty()) { relationship_tag.push_back(GraphName::RELATIONSHIP_LIST_SEPARATOR); } relationship_tag.append(kv.first); relationship_tag.push_back(GraphName::RELATIONSHIP_SEPARATOR); relationship_tag.append(related_name); } } if(relationship_tag.empty()) { tags.unset(tag_name); } else { tags.set(tag_name, relationship_tag); } } void GraphName::set_tags(gbwt::Tags& tags) const { if(this->pggname.empty()) { tags.unset(GBZ_NAME_TAG); } else { tags.set(GBZ_NAME_TAG, this->pggname); } compose_relationship_list(this->subgraph, tags, GBZ_SUBGRAPH_TAG); compose_relationship_list(this->translation, tags, GBZ_TRANSLATION_TAG); } void append_typed_field(std::string& line, const std::string& tag, char type, const std::string& value) { line.push_back(GraphName::GFA_GAF_FIELD_SEPARATOR); line.append(tag); line.push_back(GraphName::GFA_GAF_TAG_SEPARATOR); line.push_back(type); line.push_back(GraphName::GFA_GAF_TAG_SEPARATOR); line.append(value); } std::vector GraphName::gfa_header_lines() const { std::vector result; if(!this->pggname.empty()) { std::string line; line.push_back(GFA_HEADER_PREFIX); append_typed_field(line, GFA_NAME_TAG, GFA_GAF_TAG_STR_TYPE, this->pggname); result.push_back(line); } for(const auto& kv : this->subgraph) { for(const auto& supergraph_name : kv.second) { std::string line; line.push_back(GFA_HEADER_PREFIX); append_typed_field(line, GFA_GAF_SUBGRAPH_TAG, GFA_GAF_TAG_STR_TYPE, kv.first + GraphName::RELATIONSHIP_SEPARATOR + supergraph_name); result.push_back(line); } } for(const auto& kv : this->translation) { for(const auto& to_name : kv.second) { std::string line; line.push_back(GFA_HEADER_PREFIX); append_typed_field(line, GFA_GAF_TRANSLATION_TAG, GFA_GAF_TAG_STR_TYPE, kv.first + GraphName::RELATIONSHIP_SEPARATOR + to_name); result.push_back(line); } } return result; } std::vector GraphName::gaf_header_lines() const { std::vector result; if(!this->pggname.empty()) { std::string line; line.push_back(GAF_HEADER_PREFIX); line.append(GAF_NAME_TAG); line.push_back(GraphName::GFA_GAF_FIELD_SEPARATOR); line.append(this->pggname); result.push_back(line); } for(const auto& kv : this->subgraph) { for(const auto& supergraph_name : kv.second) { std::string line; line.push_back(GAF_HEADER_PREFIX); line.append(GFA_GAF_SUBGRAPH_TAG); line.push_back(GraphName::GFA_GAF_FIELD_SEPARATOR); line.append(kv.first); line.push_back(GraphName::GFA_GAF_FIELD_SEPARATOR); line.append(supergraph_name); result.push_back(line); } } for(const auto& kv : this->translation) { for(const auto& to_name : kv.second) { std::string line; line.push_back(GAF_HEADER_PREFIX); line.append(GFA_GAF_TRANSLATION_TAG); line.push_back(GraphName::GFA_GAF_FIELD_SEPARATOR); line.append(kv.first); line.push_back(GraphName::GFA_GAF_FIELD_SEPARATOR); line.append(to_name); result.push_back(line); } } return result; } bool GraphName::subgraph_of(const GraphName& another) const { GraphName combined = *this; combined.add_relationships(another); return !(combined.find_subgraph_path(*this, another).empty()); } bool GraphName::translates_to(const GraphName& another) const { GraphName combined = *this; combined.add_relationships(another); return !(combined.find_path(*this, another).empty()); } void append_description(std::string& result, size_t step, const std::string& description) { result.append("Name "); result.append(std::to_string(step)); result.append(" is for "); result.append(description); result.push_back('\n'); } void append_relationship(std::string& result, size_t step, bool is_translation) { result.append("Graph "); result.append(std::to_string(step)); if(is_translation) { result.append(" translates to "); } else { result.append(" is a subgraph of "); } result.append("graph ");; result.append(std::to_string(step + 1)); result.push_back('\n'); } void append_graph(std::string& result, size_t step, const std::string& name) { result.append(std::to_string(step)); result.push_back('\t'); result.append(name); result.push_back('\n'); } std::string GraphName::describe_relationship(const GraphName& another, const std::string& this_desc, const std::string& another_desc) const { GraphName combined = *this; combined.add_relationships(another); // Try to find a path from this to another, or the other way around. auto path = combined.find_path(*this, another); std::pair from(this->pggname, this_desc); std::pair to(another.pggname, another_desc); if(path.empty()) { path = combined.find_path(another, *this); if(!path.empty()) { std::swap(from, to); } } // Graph descriptions and relationships. std::string result; append_description(result, 1, from.second); for(size_t i = 0; i + 1 < path.size(); i++) { append_relationship(result, i + 1, path[i].second); } if(path.empty()) { append_description(result, 2, to.second); } else { append_description(result, path.size(), to.second); } // Graph names involved in the relationships. result.append("With graph names:\n"); if(path.empty()) { std::string missing_name = "(no name)"; append_graph(result, 1, (from.first.empty() ? missing_name : from.first)); append_graph(result, 2, (to.first.empty() ? missing_name : to.first)); } else { for(size_t i = 0; i < path.size(); i++) { append_graph(result, i + 1, path[i].first); } } return result; } std::vector GraphName::find_subgraph_path(const GraphName& from, const GraphName& to) const { std::vector result; if(from.pggname.empty() || to.pggname.empty()) { return result; } std::unordered_map predecessor; predecessor[from.pggname] = ""; std::deque queue; queue.push_back(from.pggname); while(!queue.empty()) { std::string curr = queue.front(); queue.pop_front(); if(curr == to.pggname) { break; } auto it = this->subgraph.find(curr); if(it == this->subgraph.end()) { continue; } for(const auto& neighbor : it->second) { if(predecessor.find(neighbor) == predecessor.end()) { predecessor[neighbor] = curr; queue.push_back(neighbor); } } } if(predecessor.find(to.pggname) == predecessor.end()) { return result; } for(std::string curr = to.pggname; !curr.empty(); curr = predecessor[curr]) { result.push_back(curr); } std::reverse(result.begin(), result.end()); return result; } std::vector> GraphName::find_path(const GraphName& from, const GraphName& to) const { std::vector> result; if(from.pggname.empty() || to.pggname.empty()) { return result; } std::unordered_map> predecessor; predecessor[from.pggname] = std::make_pair("", false); std::deque queue; queue.push_back(from.pggname); while(!queue.empty()) { std::string curr = queue.front(); queue.pop_front(); if(curr == to.pggname) { break; } // Prioritize subgraph edges. auto it = this->subgraph.find(curr); if(it != this->subgraph.end()) { for(const auto& neighbor : it->second) { if(predecessor.find(neighbor) == predecessor.end()) { predecessor[neighbor] = std::make_pair(curr, false); queue.push_back(neighbor); } } } // Then try translation edges. it = this->translation.find(curr); if(it != this->translation.end()) { for(const auto& neighbor : it->second) { if(predecessor.find(neighbor) == predecessor.end()) { predecessor[neighbor] = std::make_pair(curr, true); queue.push_back(neighbor); } } } } auto iter = predecessor.find(to.pggname); if(iter == predecessor.end()) { return result; } result.push_back(std::make_pair(to.pggname, false)); while(iter->second.first != "") { result.push_back(iter->second); iter = predecessor.find(iter->second.first); } std::reverse(result.begin(), result.end()); return result; } //------------------------------------------------------------------------------ const std::vector complement_table = { 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', '$', '#', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', '-', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'T', 'V', 'G', 'H', 'N', 'N', 'C', 'D', 'N', 'N', 'M', 'N', 'K', 'N', 'N', 'N', 'Q', 'Y', 'W', 'A', 'A', 'B', 'S', 'N', 'R', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 't', 'v', 'g', 'h', 'N', 'N', 'c', 'd', 'N', 'N', 'm', 'N', 'k', 'n', 'N', 'N', 'q', 'y', 'w', 'a', 'a', 'b', 's', 'N', 'r', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N' }; char complement(char nucleotide) { return complement_table[static_cast(nucleotide)]; } std::string reverse_complement(std::string_view seq) { std::string result(seq); reverse_complement_in_place(result); return result; } void reverse_complement_in_place(std::string& seq) { size_t swap_size = seq.size() / 2; for(size_t i = 0, j = seq.size() - 1; i < swap_size; i++, j--) { char tmp = seq[i]; seq[i] = complement_table[static_cast(seq[j])]; seq[j] = complement_table[static_cast(tmp)]; } if(seq.size() % 2 != 0) { seq[swap_size] = complement_table[seq[swap_size]]; } } bool edge_is_canonical(gbwt::node_type from, gbwt::node_type to) { nid_t from_id = gbwt::Node::id(from), to_id = gbwt::Node::id(to); if(!gbwt::Node::is_reverse(from)) { return (to_id >= from_id); } else { return ((to_id > from_id) || ((to_id == from_id) && !gbwt::Node::is_reverse(to))); } } bool path_is_canonical(const gbwt::vector_type& path) { if(path.empty()) { return true; } if(gbwt::Node::is_reverse(path.front()) == gbwt::Node::is_reverse(path.back())) { return !gbwt::Node::is_reverse(path.front()); } return edge_is_canonical(path.front(), path.back()); } //------------------------------------------------------------------------------ MetadataBuilder::PathMetadataBuilder::PathMetadataBuilder(const std::string& path_name_regex, const std::string& path_name_fields, PathSense path_sense) : sample_field(NO_FIELD), contig_field(NO_FIELD), haplotype_field(NO_FIELD), fragment_field(NO_FIELD), sense(path_sense) { // Initialize the regex. try { this->parser = std::regex(path_name_regex); } catch(std::regex_error& e) { throw std::runtime_error("MetadataBuilder: Invalid regex: " + path_name_regex); } if(path_name_fields.size() > this->parser.mark_count() + 1) { throw std::runtime_error("MetadataBuilder: Field string too long: " + path_name_fields); } // Initialize the fields. for(size_t i = 0; i < path_name_fields.size(); i++) { switch(std::tolower(path_name_fields[i])) { case 's': if(this->sample_field != NO_FIELD) { throw std::runtime_error("MetadataBuilder: Duplicate sample field"); } this->sample_field = i; break; case 'c': if(this->contig_field != NO_FIELD) { throw std::runtime_error("MetadataBuilder: Duplicate contig field"); } this->contig_field = i; break; case 'h': if(this->haplotype_field != NO_FIELD) { throw std::runtime_error("MetadataBuilder: Duplicate haplotype field"); } this->haplotype_field = i; break; case 'f': if(this->fragment_field != NO_FIELD) { throw std::runtime_error("MetadataBuilder: Duplicate fragment field"); } this->fragment_field = i; break; } } } MetadataBuilder::MetadataBuilder() : generic_path_sample_warning(false) { } MetadataBuilder::MetadataBuilder(const gbwt::Metadata& metadata) : path_names{metadata.path_names}, generic_path_sample_warning(false) { // Sanity checks. if(!metadata.hasSampleNames() && metadata.samples() > 0) { throw std::runtime_error("MetadataBuilder(): Cannot use metadata without sample names"); } if(!metadata.hasContigNames() && metadata.contigs() > 0) { throw std::runtime_error("MetadataBuilder(): Cannot use metadata without contig names"); } if(!metadata.hasPathNames() && metadata.paths() > 0) { throw std::runtime_error("MetadataBuilder(): Cannot use metadata without path names"); } for(size_t i = 0; i < metadata.sample_names.size(); i++) { this->sample_names.emplace(metadata.sample(i), i); } for(size_t i = 0; i < metadata.contig_names.size(); i++) { this->contig_names.emplace(metadata.contig(i), i); } for(gbwt::PathName copy : metadata.path_names) { // Record all the phases this->haplotypes.emplace(copy.sample, copy.phase); { // Count the actual observed path name auto& self_count = this->counts[copy]; self_count = std::max(self_count, 1); } { // Make sure the count stored for the no-count version of the path name // will not generate this path name again when inferring counts. size_t observed_count = copy.count; copy.count = 0; auto& guess_count = this->counts[copy]; guess_count = std::max(guess_count, observed_count + 1); } } } MetadataBuilder::MetadataBuilder(const std::string& path_name_regex, const std::string& path_name_fields, PathSense path_sense) : generic_path_sample_warning(false) { this->add_path_name_format(path_name_regex, path_name_fields, path_sense); } void MetadataBuilder::add_path_name_format(const std::string& path_name_regex, const std::string& path_name_fields, PathSense path_sense) { this->path_name_formats.emplace_back(path_name_regex, path_name_fields, path_sense); } void MetadataBuilder::add_path(PathSense sense, const std::string& sample_name, const std::string& locus_name, size_t haplotype, size_t phase_block, const handlegraph::subrange_t& subrange, size_t job) { // For error handling we need to be able to describe the path // This returns a string starting with " ". auto describe_path = [&]() { std::stringstream ss; if(sample_name != PathMetadata::NO_SAMPLE_NAME) { ss << " sample " << sample_name; } if(locus_name != PathMetadata::NO_LOCUS_NAME) { ss << " contig " << locus_name; } if(haplotype != PathMetadata::NO_HAPLOTYPE) { ss << " phase " << haplotype; } if(phase_block != PathMetadata::NO_PHASE_BLOCK) { ss << " count " << phase_block; } if(subrange != PathMetadata::NO_SUBRANGE) { ss << " range [" << subrange.first; if(subrange.second != PathMetadata::NO_END_POSITION) { ss << "-" << subrange.second; } ss << "]"; } return ss.str(); }; gbwt::PathName path_name = { static_cast(0), static_cast(0), static_cast(0), static_cast(0) }; if(sample_name != PathMetadata::NO_SAMPLE_NAME || sense == PathSense::GENERIC) { // We need sample name metadata. // If using generic sense, use the magic sample name. auto& sample_name_to_store = (sense == PathSense::GENERIC) ? GENERIC_PATH_SAMPLE_NAME : sample_name; // Apply the sample name. auto iter = this->sample_names.find(sample_name_to_store); if(iter == this->sample_names.end()) { path_name.sample = this->sample_names.size(); this->sample_names[sample_name_to_store] = path_name.sample; } else { path_name.sample = iter->second; } } if(locus_name != PathMetadata::NO_LOCUS_NAME) { // Apply the locus as the contig name auto iter = this->contig_names.find(locus_name); if(iter == this->contig_names.end()) { path_name.contig = this->contig_names.size(); this->contig_names[locus_name] = path_name.contig; } else { path_name.contig = iter->second; } } if(haplotype == PathMetadata::NO_HAPLOTYPE) { // Record a sentinel phase number path_name.phase = gbwtgraph::GBWTGraph::NO_PHASE; } else { // Apply the phase number path_name.phase = haplotype; } // Remember that this phase of this sample exists. this->haplotypes.insert(std::pair(path_name.sample, path_name.phase)); if(subrange != PathMetadata::NO_SUBRANGE) { if(phase_block != PathMetadata::NO_PHASE_BLOCK && phase_block != 0 && subrange.first != 0) { // TODO: We can't represent both a nonzero phase block and a nonzero subrange. throw std::runtime_error("MetadataBuilder: Can't represent a nonzero phase block and a nonzero subrange in the same path" + describe_path()); } // Use the subrange start as the count path_name.count = subrange.first; } else if(phase_block != PathMetadata::NO_PHASE_BLOCK) { // Use the phase block as the count path_name.count = phase_block; } { auto iter = this->counts.find(path_name); if(iter != this->counts.end()) { // This path is duplicate or ambiguous. if(phase_block != PathMetadata::NO_PHASE_BLOCK || subrange != PathMetadata::NO_SUBRANGE) { // The count was user-specified, so bail out. throw std::runtime_error("MetadataBuilder: Duplicate path for" + describe_path()); } else { // We are going to infer a count; just warn. std::cerr << "MetadataBuilder::add_path(): Warning: Path for" << describe_path() << " is not unique" << std::endl; std::cerr << "MetadataBuilder::add_path(): Warning: Using fragment/count field to disambiguate" << std::endl; std::cerr << "MetadataBuilder::add_path(): Warning: Decompression may not produce valid GFA" << std::endl; path_name.count = iter->second; iter->second++; } } else { // This path is unique but the next one that looks like it won't be. this->counts.emplace_hint(iter, path_name, 1); } } // Record the structured path name in the metadata, referencing the strings // we have saved. this->add_path_name(path_name, job); } void MetadataBuilder::add_path(const std::string& name, size_t job) { for(auto& format : this->path_name_formats) { std::smatch fields; if(!std::regex_match(name, fields, format.parser)) { continue; } std::string sample_name = PathMetadata::NO_SAMPLE_NAME; if(format.sample_field != NO_FIELD) { sample_name = fields[format.sample_field]; if(!(this->generic_path_sample_warning) && sample_name == GENERIC_PATH_SAMPLE_NAME) { std::cerr << "MetadataBuilder::add_path(): Warning: Sample name " << GENERIC_PATH_SAMPLE_NAME << " is reserved" << std::endl; this->generic_path_sample_warning = true; } } std::string locus_name = PathMetadata::NO_LOCUS_NAME; if(format.contig_field != NO_FIELD) { locus_name = fields[format.contig_field]; } size_t haplotype = PathMetadata::NO_HAPLOTYPE; if(format.haplotype_field != NO_FIELD) { haplotype = parse_unsigned_or_throw(fields[format.haplotype_field], "MetadataBuilder: Invalid haplotype field "); } size_t phase_block = PathMetadata::NO_PHASE_BLOCK; if(format.fragment_field != NO_FIELD) { phase_block = parse_unsigned_or_throw(fields[format.fragment_field], "MetadataBuilder: Invalid fragment field "); } this->add_path(format.sense, sample_name, locus_name, haplotype, phase_block, PathMetadata::NO_SUBRANGE, job); return; } throw std::runtime_error("MetadataBuilder: Cannot parse path name " + name); } void MetadataBuilder::add_walk(const std::string& sample, const std::string& haplotype, const std::string& contig, const std::string& start, size_t job) { // Check sample name. if(!(this->generic_path_sample_warning) && sample == GENERIC_PATH_SAMPLE_NAME) { std::cerr << "MetadataBuilder::add_walk(): Warning: Sample name " << GENERIC_PATH_SAMPLE_NAME << " is reserved for generic paths" << std::endl; this->generic_path_sample_warning = true; } if(sample == "*") { // Treat this as an elided sample name and a generic path. subrange_t subrange = PathMetadata::NO_SUBRANGE; subrange.first = parse_unsigned_or_throw(start, "MetadataBuilder: Invalid start position "); this->add_path(PathSense::GENERIC, PathMetadata::NO_SAMPLE_NAME, contig, PathMetadata::NO_HAPLOTYPE, PathMetadata::NO_PHASE_BLOCK, subrange, job); } else { // Parse the haplotype size_t haplotype_number = parse_unsigned_or_throw(haplotype, "MetadataBuilder: Invalid haplotype field "); // Start position as fragment identifier. size_t phase_block = PathMetadata::NO_PHASE_BLOCK; if (start == "*") { phase_block = 0; } else { phase_block = parse_unsigned_or_throw(start, "MetadataBuilder: Invalid start position "); } // Add as a haplotype this->add_path(PathSense::HAPLOTYPE, sample, contig, haplotype_number, phase_block, PathMetadata::NO_SUBRANGE, job); } } void MetadataBuilder::add_gbwt_path(const gbwt::FullPathName& path_name, size_t job) { // This also works with generic paths. this->add_path ( PathSense::HAPLOTYPE, path_name.sample_name, path_name.contig_name, path_name.haplotype, path_name.offset, PathMetadata::NO_SUBRANGE, job ); } void MetadataBuilder::add_haplotype(const std::string& sample, const std::string& contig, size_t haplotype, size_t fragment, size_t job) { this->add_path(PathSense::HAPLOTYPE, sample, contig, haplotype, fragment, PathMetadata::NO_SUBRANGE, job); } void MetadataBuilder::add_generic_path(const std::string& name, size_t job) { this->add_path(PathSense::GENERIC, PathMetadata::NO_SAMPLE_NAME, name, PathMetadata::NO_HAPLOTYPE, PathMetadata::NO_PHASE_BLOCK, PathMetadata::NO_SUBRANGE, job); } gbwt::Metadata MetadataBuilder::get_metadata() const { gbwt::Metadata metadata; size_t path_count = 0; for(auto& names_by_job : this->path_names) { for(auto& path_name : names_by_job) { metadata.addPath(path_name); path_count++; } } if(this->sample_names.empty() && path_count > 0) { metadata.setSamples(1); } else { metadata.setSamples(map_to_vector(this->sample_names)); } metadata.setHaplotypes(this->haplotypes.size()); if(this->contig_names.empty() && path_count > 0) { metadata.setContigs(1); } else { metadata.setContigs(map_to_vector(this->contig_names)); } return metadata; } void MetadataBuilder::clear() { this->sample_names = std::map(); this->contig_names = std::map(); this->haplotypes = std::set>(); this->path_names = std::vector>(); this->counts = std::map(); } void MetadataBuilder::add_path_name(const gbwt::PathName& path_name, size_t job) { if(job >= this->path_names.size()) { this->path_names.resize(job + 1); } this->path_names[job].push_back(path_name); } std::vector MetadataBuilder::map_to_vector(const std::map& source) { std::vector result(source.size()); for(auto& name : source) { result[name.second] = name.first; } return result; } //------------------------------------------------------------------------------ } // namespace gbwtgraph jltsiren-gbwtgraph-e27bc43/tests/000077500000000000000000000000001524136466400171175ustar00rootroot00000000000000jltsiren-gbwtgraph-e27bc43/tests/Makefile000066400000000000000000000063121524136466400205610ustar00rootroot00000000000000SDSL_DIR=../../sdsl-lite include $(SDSL_DIR)/Make.helper MAIN_DIR=.. LIBRARY=$(MAIN_DIR)/lib/libgbwtgraph.a # Multithreading with OpenMP. PARALLEL_FLAGS=-fopenmp -pthread # Directories for dependencies. INCLUDES=-I$(MAIN_DIR)/include -I$(INC_DIR) LIBS=-L$(LIB_DIR) -lgbwt -lhandlegraph -lsdsl -lgtest -lgtest_main # Use pkg-config to find system dependencies. ifeq ($(shell pkg-config --exists libcrypto --exists libzstd && echo 1), 1) $(info Found libcrypto and libzstd.) INCLUDES += $(shell pkg-config --cflags libcrypto libzstd) LIBS += $(shell pkg-config --libs libcrypto libzstd) else $(error Could not find libcrypto or libzstd. Please update PKG_CONFIG_PATH) endif # Apple Clang does not support OpenMP directly, so we need special handling. ifeq ($(shell uname -s), Darwin) # The compiler complains about -fopenmp instead of missing input. ifeq ($(strip $(shell $(MY_CXX) -fopenmp /dev/null -o/dev/null 2>&1 | grep fopenmp | wc -l)), 1) $(info The compiler is Apple Clang that needs libomp for OpenMP support.) # The compiler only needs to do the preprocessing. PARALLEL_FLAGS=-Xpreprocessor -fopenmp -pthread # Find libomp installed by Homebrew or MacPorts. ifeq ($(shell if [ -e $(HOMEBREW_PREFIX)/include/omp.h ]; then echo 1; else echo 0; fi), 1) $(info Found libomp installed by Homebrew and linked to $(HOMEBREW_PREFIX).) INCLUDES += -I$(HOMEBREW_PREFIX)/include LIBS += -L$(HOMEBREW_PREFIX)/lib else ifeq ($(shell if [ -d $(HOMEBREW_PREFIX)/opt/libomp/include ]; then echo 1; else echo 0; fi), 1) $(info Found a keg-only libomp installed by Homebrew at $(HOMEBREW_PREFIX)/opt/libomp.) INCLUDES += -I$(HOMEBREW_PREFIX)/opt/libomp/include LIBS += -L$(HOMEBREW_PREFIX)/opt/libomp/lib else ifeq ($(shell if [ -d /opt/local/lib/libomp ]; then echo 1; else echo 0; fi), 1) $(info Found libomp installed by MacPorts at /opt/local.) INCLUDES += -I/opt/local/include/libomp LIBS += -L/opt/local/lib/libomp else $(error Could not find libomp. Please install it using Homebrew or MacPorts) endif # We also need to link it. LIBS += -lomp endif # gtest isn't in pkg-config but also we can't find its libs by default if # they are installed via Homebrew. ifeq ($(shell if [ -e $(HOMEBREW_PREFIX)/lib/libgtest.a ]; then echo 1; else echo 0; fi), 1) $(info Found gtest installed by Homebrew and linked to $(HOMEBREW_PREFIX).) INCLUDES += -I$(HOMEBREW_PREFIX)/include LIBS += -L$(HOMEBREW_PREFIX)/lib endif endif CXX_FLAGS=$(MY_CXX_FLAGS) $(PARALLEL_FLAGS) $(MY_CXX_OPT_FLAGS) $(INCLUDES) HEADERS=$(wildcard $(MAIN_DIR)/include/gbwtgraph/*.h) shared.h PROGRAMS=test_utils test_gbwtgraph test_cached_gbwtgraph test_gfa test_gbz test_minimizer test_index test_algorithms test_path_cover test_subgraph test_naive_graph .PHONY: all clean test all:$(PROGRAMS) %.o:%.cpp $(HEADERS) $(MY_CXX) $(CXX_FLAGS) -c $< $(PROGRAMS):%:%.o $(LIBRARY) $(MY_CXX) $(CXX_FLAGS) -o $@ $< $(LIBRARY) $(LIBS) test:$(PROGRAMS) $(foreach PROG,$(PROGRAMS),./$(PROG) || exit 1;) clean: rm -f $(PROGRAMS) *.o jltsiren-gbwtgraph-e27bc43/tests/gfas/000077500000000000000000000000001524136466400200375ustar00rootroot00000000000000jltsiren-gbwtgraph-e27bc43/tests/gfas/components.gfa000066400000000000000000000006431524136466400227060ustar00rootroot00000000000000H VN:Z:1.1 H NM:Z:81b160c814182a12aaf95fd458e191590e95fb13c71e1c2f61ff827f605cf970 S 11 G S 12 A S 13 T S 14 T S 15 A S 16 C S 17 A L 11 + 12 + L 11 + 13 + L 12 + 14 + L 13 + 14 + L 14 + 15 + L 14 + 16 + L 15 + 17 + L 16 + 17 + P A1 11+,12+,14+,15+,17+ P A2 11+,13+,14+,16+,17+ S 21 G S 22 A S 23 T S 24 T S 25 A L 21 + 22 + L 21 + 23 + L 22 + 24 + L 24 + 23 - L 24 + 25 + P B1 21+,22+,24+,23-,21- P B2 21+,22+,24+,25+ jltsiren-gbwtgraph-e27bc43/tests/gfas/components_first.gfa000066400000000000000000000004261524136466400241140ustar00rootroot00000000000000H VN:Z:1.1 H NM:Z:ffaba5c896f7b761a77735dbc8dce2220eba511cedadf227a32ed9fa269c9e9b S 11 G S 12 A S 13 T S 14 T S 15 A S 16 C S 17 A L 11 + 12 + L 11 + 13 + L 12 + 14 + L 13 + 14 + L 14 + 15 + L 14 + 16 + L 15 + 17 + L 16 + 17 + P A1 11+,12+,14+,15+,17+ P A2 11+,13+,14+,16+,17+ jltsiren-gbwtgraph-e27bc43/tests/gfas/components_ref.gfa000066400000000000000000000010601524136466400235340ustar00rootroot00000000000000H VN:Z:1.1 RS:Z:ref H NM:Z:81b160c814182a12aaf95fd458e191590e95fb13c71e1c2f61ff827f605cf970 S 11 G S 12 A S 13 T S 14 T S 15 A S 16 C S 17 A S 21 G S 22 A S 23 T S 24 T S 25 A L 11 + 12 + 0M L 11 + 13 + 0M L 12 + 14 + 0M L 13 + 14 + 0M L 14 + 15 + 0M L 14 + 16 + 0M L 15 + 17 + 0M L 16 + 17 + 0M L 21 + 22 + 0M L 21 + 23 + 0M L 22 + 24 + 0M L 23 + 24 - 0M L 24 + 25 + 0M W ref 0 A 0 5 >11>13>14>15>17 W sample 1 A 0 5 >11>12>14>15>17 W sample 2 A 0 5 >11>13>14>16>17 W ref 0 B 0 5 >21>23<24<22<21 W sample 1 B 0 5 >21>22>24<23<21 W sample 2 B 0 4 >21>22>24>25 jltsiren-gbwtgraph-e27bc43/tests/gfas/components_walks.gfa000066400000000000000000000007531524136466400241110ustar00rootroot00000000000000H VN:Z:1.1 H NM:Z:81b160c814182a12aaf95fd458e191590e95fb13c71e1c2f61ff827f605cf970 S 11 G S 12 A S 13 T S 14 T S 15 A S 16 C S 17 A S 21 G S 22 A S 23 T S 24 T S 25 A L 11 + 12 + 0M L 11 + 13 + 0M L 12 + 14 + 0M L 13 + 14 + 0M L 14 + 15 + 0M L 14 + 16 + 0M L 15 + 17 + 0M L 16 + 17 + 0M L 21 + 22 + 0M L 21 + 23 + 0M L 22 + 24 + 0M L 23 + 24 - 0M L 24 + 25 + 0M W sample 1 A 0 5 >11>12>14>15>17 W sample 2 A 0 5 >11>13>14>16>17 W sample 1 B 0 5 >21>22>24<23<21 W sample 2 B 0 4 >21>22>24>25 jltsiren-gbwtgraph-e27bc43/tests/gfas/components_walks_compressed.gfa000066400000000000000000000010301524136466400263220ustar00rootroot00000000000000H VN:Z:1.1 H NM:Z:81b160c814182a12aaf95fd458e191590e95fb13c71e1c2f61ff827f605cf970 S 11 G S 12 A S 13 T S 14 T S 15 A S 16 C S 17 A S 21 G S 22 A S 23 T S 24 T S 25 A Q q1 >11>13 Q q2 >16>17 Q q3 >q1>14 Q q4 >21>22 Q q5 <25<24 L 11 + 12 + 0M L 11 + 13 + 0M L 12 + 14 + 0M L 13 + 14 + 0M L 14 + 15 + 0M L 14 + 16 + 0M L 15 + 17 + 0M L 16 + 17 + 0M L 21 + 22 + 0M L 21 + 23 + 0M L 22 + 24 + 0M L 23 + 24 - 0M L 24 + 25 + 0M W sample 1 A 0 5 >11>12>14>15>17 Z sample 2 A 0 5 >q3>q2 W sample 1 B 0 5 >21>22>24<23<21 W sample 2 B 0 4 >q411>12>14>15>17 W sample 2 A * * >11>13>14>16>17 W sample 1 B * * >21>22>24<23<21 W sample 2 B * * >21>22>24>25 jltsiren-gbwtgraph-e27bc43/tests/gfas/cyclic.gfa000066400000000000000000000003671524136466400217720ustar00rootroot00000000000000H VN:Z:1.1 H NM:Z:c15924178e845eb7c1f92237abfe9cc7284555d799ad30d9a1690ea40e0fcbe4 S 1 G S 2 A S 3 T S 4 GGG S 5 T S 6 A S 7 C S 8 A S 9 A P short1 1+,4+,5+,6+,7+,9+ P alt 1+,2+,4+,5+,6+,8+,9+ P short2 1+,4+,5+,6+,7+,9+ P cyclic 1+,4+,5+,6+,8+,2+ jltsiren-gbwtgraph-e27bc43/tests/gfas/default.gfa000066400000000000000000000010331524136466400221370ustar00rootroot00000000000000H VN:Z:1.1 H NM:Z:81b160c814182a12aaf95fd458e191590e95fb13c71e1c2f61ff827f605cf970 S 11 G S 12 A S 13 T S 14 T S 15 A S 16 C S 17 A S 21 G S 22 A S 23 T S 24 T S 25 A L 11 + 12 + 0M L 11 + 13 + 0M L 12 + 14 + 0M L 13 + 14 + 0M L 14 + 15 + 0M L 14 + 16 + 0M L 15 + 17 + 0M L 16 + 17 + 0M L 21 + 22 + 0M L 21 + 23 + 0M L 22 + 24 + 0M L 23 + 24 - 0M L 24 + 25 + 0M P A 11+,12+,14+,15+,17+ * P B 21+,22+,24+,25+ * W sample 1 A 0 5 >11>12>14>15>17 W sample 2 A 0 5 >11>13>14>16>17 W sample 1 B 0 5 >21>22>24<23<21 W sample 2 B 0 4 >21>22>24>25 jltsiren-gbwtgraph-e27bc43/tests/gfas/example.gfa000066400000000000000000000003341524136466400221510ustar00rootroot00000000000000H VN:Z:1.1 H NM:Z:c15924178e845eb7c1f92237abfe9cc7284555d799ad30d9a1690ea40e0fcbe4 S 1 G S 2 A S 3 T S 4 GGG S 5 T S 6 A S 7 C S 8 A S 9 A P short1 1+,4+,5+,6+,7+,9+ P alt 1+,2+,4+,5+,6+,8+,9+ P short2 1+,4+,5+,6+,7+,9+ jltsiren-gbwtgraph-e27bc43/tests/gfas/example_0-based.gfa000066400000000000000000000003341524136466400234440ustar00rootroot00000000000000H VN:Z:1.1 H NM:Z:997a7dc91bcfb668776507360e2e71bb3134fb83f468d841807230f6a7d631cc S 0 G S 1 A S 2 T S 3 GGG S 4 T S 5 A S 6 C S 7 A S 8 A P short1 0+,3+,4+,5+,6+,8+ P alt 0+,1+,3+,4+,5+,7+,8+ P short2 0+,3+,4+,5+,6+,8+ jltsiren-gbwtgraph-e27bc43/tests/gfas/example_chopped.gfa000066400000000000000000000005211524136466400236510ustar00rootroot00000000000000H VN:Z:1.1 H NM:Z:382f0416bfb164cf8e96a2fbd9b0d90f04f50b4b75aa5c637405011b4e475a37 S 1 G S 2 A S 4 GGG S 5 T S 6 A S 7 C S 8 A S 9 A L 1 + 2 + 0M L 1 + 4 + 0M L 2 + 4 + 0M L 4 + 5 + 0M L 5 + 6 + 0M L 6 + 7 + 0M L 6 + 8 + 0M L 7 + 9 + 0M L 8 + 9 + 0M P short1 1+,4+,5+,6+,7+,9+ * P alt 1+,2+,4+,5+,6+,8+,9+ * P short2 1+,4+,5+,6+,7+,9+ * jltsiren-gbwtgraph-e27bc43/tests/gfas/example_chopping.gfa000066400000000000000000000003161524136466400240400ustar00rootroot00000000000000H VN:Z:1.1 H NM:Z:a6d5b69bf0d1f5b5997d49d5bccae4b4809801ef0c9db5f30df5e9c191771e6d S 1 G S 2 A S 3 T S 4 GGGT S 6 A S 7 C S 8 A S 9 A P short1 1+,4+,6+,7+,9+ P alt 1+,2+,4+,6+,8+,9+ P short2 1+,4+,6+,7+,9+ jltsiren-gbwtgraph-e27bc43/tests/gfas/example_from_chopping.gfa000066400000000000000000000004661524136466400250710ustar00rootroot00000000000000H VN:Z:1.1 H NM:Z:1ab7e931f4c342315e4359609919557eb880cad53013d5bc667b9a02d3d3cf07 S 1 G S 2 A S 4 GGGT S 6 A S 7 C S 8 A S 9 A L 1 + 2 + 0M L 1 + 4 + 0M L 2 + 4 + 0M L 4 + 6 + 0M L 6 + 7 + 0M L 6 + 8 + 0M L 7 + 9 + 0M L 8 + 9 + 0M P short1 1+,4+,6+,7+,9+ * P alt 1+,2+,4+,6+,8+,9+ * P short2 1+,4+,6+,7+,9+ * jltsiren-gbwtgraph-e27bc43/tests/gfas/example_more_reference.gfa000066400000000000000000000010001524136466400252000ustar00rootroot00000000000000H VN:Z:1.1 RS:Z:GRCh37 GRCh38 H NM:Z:382f0416bfb164cf8e96a2fbd9b0d90f04f50b4b75aa5c637405011b4e475a37 S 1 G S 2 A S 4 GGG S 5 T S 6 A S 7 C S 8 A S 9 A L 1 + 2 + 0M L 1 + 4 + 0M L 2 + 4 + 0M L 4 + 5 + 0M L 5 + 6 + 0M L 6 + 7 + 0M L 6 + 8 + 0M L 7 + 9 + 0M L 8 + 9 + 0M P GRCh38#1#chr1 1+,4+,5+,6+,7+,9+ *,*,*,*,* P GRCh37#2#chr1 1+,2+,4+,5+,6+,8+,9+ *,*,*,*,*,* P coolergene 1+,2+,4+,5+,6+,8+,9+ *,*,*,*,*,* W sample1 1 chr2 0 8 >1>4>5>6>7>9 W CHM13v2 0 chr1 0 9 >1>2>4>5>6>8>9 W sample1 2 chr2 0 8 >1>4>5>6>7>9 jltsiren-gbwtgraph-e27bc43/tests/gfas/example_reference.gfa000066400000000000000000000007741524136466400241770ustar00rootroot00000000000000H VN:Z:1.1 RS:Z:GRCh37 GRCh38 H NM:Z:382f0416bfb164cf8e96a2fbd9b0d90f04f50b4b75aa5c637405011b4e475a37 S 1 G S 2 A S 4 GGG S 5 T S 6 A S 7 C S 8 A S 9 A L 1 + 2 + 0M L 1 + 4 + 0M L 2 + 4 + 0M L 4 + 5 + 0M L 5 + 6 + 0M L 6 + 7 + 0M L 6 + 8 + 0M L 7 + 9 + 0M L 8 + 9 + 0M P GRCh38#0#chr1 1+,4+,5+,6+,7+,9+ *,*,*,*,* P GRCh37#0#chr1 1+,2+,4+,5+,6+,8+,9+ *,*,*,*,*,* P coolgene 1+,2+,4+,5+,6+,8+,9+ *,*,*,*,*,* W sample1 1 chr1 0 8 >1>4>5>6>7>9 W CHM13 0 chr1 0 9 >1>2>4>5>6>8>9 W sample1 2 chr1 0 8 >1>4>5>6>7>9 jltsiren-gbwtgraph-e27bc43/tests/gfas/example_str-names.gfa000066400000000000000000000003701524136466400241420ustar00rootroot00000000000000H VN:Z:1.1 H NM:Z:3484edc98b478ea85e4257f4e0da57613e9aaf07acc9f1ef04892c183df6225e S s1 G S s2 A S s3 T S s4 GGG S s5 T S s6 A S s7 C S s8 A S s9 A P short1 s1+,s4+,s5+,s6+,s7+,s9+ P alt s1+,s2+,s4+,s5+,s6+,s8+,s9+ P short2 s1+,s4+,s5+,s6+,s7+,s9+ jltsiren-gbwtgraph-e27bc43/tests/gfas/example_walks-only.gfa000066400000000000000000000006601524136466400243330ustar00rootroot00000000000000H VN:Z:1.1 H NM:Z:382f0416bfb164cf8e96a2fbd9b0d90f04f50b4b75aa5c637405011b4e475a37 S 1 G S 2 A S 4 GGG S 5 T S 6 A S 7 C S 8 A S 9 A L 1 + 2 + 0M L 1 + 4 + 0M L 2 + 4 + 0M L 4 + 5 + 0M L 5 + 6 + 0M L 6 + 7 + 0M L 6 + 8 + 0M L 7 + 9 + 0M L 8 + 9 + 0M W * 0 short 0 8 >1>4>5>6>7>9 W * 0 alt1 0 9 >1>2>4>5>6>8>9 W * 0 alt2 0 9 >1>2>4>5>6>8>9 W short 1 chr 0 8 >1>4>5>6>7>9 W alt 0 chr 0 9 >1>2>4>5>6>8>9 W short 2 chr 0 8 >1>4>5>6>7>9 jltsiren-gbwtgraph-e27bc43/tests/gfas/example_walks.gfa000066400000000000000000000006571524136466400233620ustar00rootroot00000000000000H VN:Z:1.1 H NM:Z:382f0416bfb164cf8e96a2fbd9b0d90f04f50b4b75aa5c637405011b4e475a37 S 1 G S 2 A S 4 GGG S 5 T S 6 A S 7 C S 8 A S 9 A L 1 + 2 + 0M L 1 + 4 + 0M L 2 + 4 + 0M L 4 + 5 + 0M L 5 + 6 + 0M L 6 + 7 + 0M L 6 + 8 + 0M L 7 + 9 + 0M L 8 + 9 + 0M P short 1+,4+,5+,6+,7+,9+ * P alt1 1+,2+,4+,5+,6+,8+,9+ * P alt2 1+,2+,4+,5+,6+,8+,9+ * W short 1 chr 0 8 >1>4>5>6>7>9 W alt 0 chr 0 9 >1>2>4>5>6>8>9 W short 2 chr 0 8 >1>4>5>6>7>9 jltsiren-gbwtgraph-e27bc43/tests/gfas/for_subgraph.gfa000066400000000000000000000007741524136466400232070ustar00rootroot00000000000000H VN:Z:1.1 RS:Z:odd H NM:Z:a148a3d153848874143026b80854aea55825ae27ef81a774d1d4afd0ad0166f9 # Segments contain sequences that can be chopped. # If we skip odd walks, we do not use the first and the last segment. S 1 AGAGA S 2 CAT S 3 GATTACA S 4 GATTA S 5 A S 6 GATTACAT S 7 T S 8 ATTA L 1 + 3 + 0M L 2 + 3 + 0M L 3 + 4 + 0M L 4 + 5 + 0M L 4 + 6 + 0M L 5 + 6 + 0M L 6 + 7 + 0M L 6 + 8 + 0M W even 1 chr 0 25 >2>3>4>5>6>7 W odd 1 chr 0 30 >1>3>4>5>6>8 W even 2 chr 0 24 >2>3>4>6>7 W odd 2 chr 0 29 >1>3>4>6>8 jltsiren-gbwtgraph-e27bc43/tests/gfas/fragmented.gfa000066400000000000000000000007401524136466400226330ustar00rootroot00000000000000H VN:Z:1.1 RS:Z:ref H NM:Z:9b7b12c022f47df56e243d83f8f416d1f93f55205fd54b9cedd1be7714ba07b6 # The designated reference sample is fragmented, missing node 4/5. S 1 GAT S 2 TA S 3 CA S 4 C S 5 G S 6 ATTA S 7 C S 8 G S 9 AT L 1 + 2 + 0M L 2 + 3 + 0M L 3 + 4 + 0M L 3 + 5 + 0M L 4 + 6 + 0M L 5 + 6 + 0M L 6 + 7 + 0M L 6 + 8 + 0M L 7 + 9 + 0M L 8 + 9 + 0M W ref 0 contig 0 7 >1>2>3 W ref 0 contig 8 15 >6>7>9 W sample 1 contig 0 15 >1>2>3>4>6>8>9 W sample 2 contig 0 15 >1>2>3>5>6>7>9 jltsiren-gbwtgraph-e27bc43/tests/gfas/generic-only.gfa000066400000000000000000000006321524136466400231120ustar00rootroot00000000000000H VN:Z:1.1 H NM:Z:81b160c814182a12aaf95fd458e191590e95fb13c71e1c2f61ff827f605cf970 S 11 G S 12 A S 13 T S 14 T S 15 A S 16 C S 17 A S 21 G S 22 A S 23 T S 24 T S 25 A L 11 + 12 + 0M L 11 + 13 + 0M L 12 + 14 + 0M L 13 + 14 + 0M L 14 + 15 + 0M L 14 + 16 + 0M L 15 + 17 + 0M L 16 + 17 + 0M L 21 + 22 + 0M L 21 + 23 + 0M L 22 + 24 + 0M L 23 + 24 - 0M L 24 + 25 + 0M P A 11+,12+,14+,15+,17+ * P B 21+,22+,24+,25+ * jltsiren-gbwtgraph-e27bc43/tests/gfas/pan-sn.gfa000066400000000000000000000011401524136466400217060ustar00rootroot00000000000000H VN:Z:1.1 H NM:Z:81b160c814182a12aaf95fd458e191590e95fb13c71e1c2f61ff827f605cf970 S 11 G S 12 A S 13 T S 14 T S 15 A S 16 C S 17 A S 21 G S 22 A S 23 T S 24 T S 25 A L 11 + 12 + 0M L 11 + 13 + 0M L 12 + 14 + 0M L 13 + 14 + 0M L 14 + 15 + 0M L 14 + 16 + 0M L 15 + 17 + 0M L 16 + 17 + 0M L 21 + 22 + 0M L 21 + 23 + 0M L 22 + 24 + 0M L 23 + 24 - 0M L 24 + 25 + 0M P _gbwt_ref#18446744073709551615#A 11+,12+,14+,15+,17+ * P sample#1#A 11+,12+,14+,15+,17+ * P sample#2#A 11+,13+,14+,16+,17+ * P _gbwt_ref#18446744073709551615#B 21+,22+,24+,25+ * P sample#1#B 21+,22+,24+,23-,21- * P sample#2#B 21+,22+,24+,25+ * jltsiren-gbwtgraph-e27bc43/tests/gfas/ref-only.gfa000066400000000000000000000006571524136466400222610ustar00rootroot00000000000000H VN:Z:1.1 RS:Z:ref H NM:Z:81b160c814182a12aaf95fd458e191590e95fb13c71e1c2f61ff827f605cf970 S 11 G S 12 A S 13 T S 14 T S 15 A S 16 C S 17 A S 21 G S 22 A S 23 T S 24 T S 25 A L 11 + 12 + 0M L 11 + 13 + 0M L 12 + 14 + 0M L 13 + 14 + 0M L 14 + 15 + 0M L 14 + 16 + 0M L 15 + 17 + 0M L 16 + 17 + 0M L 21 + 22 + 0M L 21 + 23 + 0M L 22 + 24 + 0M L 23 + 24 - 0M L 24 + 25 + 0M W ref 0 A 0 5 >11>13>14>15>17 W ref 0 B 0 5 >21>23<24<22<21 jltsiren-gbwtgraph-e27bc43/tests/gfas/reversal.gfa000066400000000000000000000002221524136466400223350ustar00rootroot00000000000000H VN:Z:1.1 H NM:Z:ecbc15d24e45778a342f78b75a602d0a2ead9ca7bef607f8aa9e2638e40849e3 S 1 G S 2 A S 3 T S 4 TAC S 5 A P path#0#chr 1+,2+,5-,4-,3-,1- jltsiren-gbwtgraph-e27bc43/tests/gfas/reversal_chopping.gfa000066400000000000000000000002041524136466400242240ustar00rootroot00000000000000H VN:Z:1.1 H NM:Z:fbecfc57cd41013d09354c6058aa5ab396ed2f3e8ba6a4c2c3301a943df65ec9 S 1 G S 2 A S 3 T S 4 TACA P path 1+,2+,4-,3-,1- jltsiren-gbwtgraph-e27bc43/tests/gfas/reversal_walks.gfa000066400000000000000000000002211524136466400235350ustar00rootroot00000000000000H VN:Z:1.1 H NM:Z:ecbc15d24e45778a342f78b75a602d0a2ead9ca7bef607f8aa9e2638e40849e3 S 1 G S 2 A S 3 T S 4 TAC S 5 A W path 0 chr 0 8 >1>2<5<4<3<1 jltsiren-gbwtgraph-e27bc43/tests/gfas/set-pggname.py000077500000000000000000000017451524136466400226320ustar00rootroot00000000000000#! /usr/bin/env python3 # -*- coding: utf-8 -*- # set-pggname.py name graph.gfa # set-pggname.py $(pggname graph.gfa) import sys if len(sys.argv) != 3: print("Usage: set-pggname.py name graph.gfa") sys.exit(1) name = sys.argv[1] gfa_file = sys.argv[2] # Read the lines from the GFA file lines = [] with open(gfa_file, 'r') as f: lines = f.readlines() # If there is already a header line with a NM:Z tag, do nothing. for i in range(len(lines)): if lines[i].startswith('H'): if 'NM:Z:' in lines[i]: sys.exit(0) # Check if the first line is a file header file_header = None if lines and lines[0].startswith('H'): if 'VN:Z:' in lines[0]: file_header = lines[0] lines = lines[1:] # Write the possible file header, the new header line, and the rest of the lines back to the file new_header = f"H\tNM:Z:{name}\n" with open(gfa_file, 'w') as f: if file_header: f.write(file_header) f.write(new_header) f.writelines(lines) jltsiren-gbwtgraph-e27bc43/tests/shared.h000066400000000000000000000755121524136466400205500ustar00rootroot00000000000000#ifndef GBWTGRAPH_TESTS_SHARED_H #define GBWTGRAPH_TESTS_SHARED_H #include #include #include #include #include #include #include #include #include /* shared.h: Utility functions and data definitions shared between the tests. */ namespace { //------------------------------------------------------------------------------ using gbwtgraph::nid_t; using gbwtgraph::handle_t; using gbwtgraph::pos_t; //------------------------------------------------------------------------------ // GBWT / GBWTGraph / GBZ comparisons. // Flags for checking components that are not always preserved. constexpr std::uint64_t FLAG_GBZ_TAGS = 0x01; constexpr std::uint64_t FLAG_METADATA = 0x02; constexpr std::uint64_t FLAG_TRANSLATION = 0x04; constexpr std::uint64_t NO_FLAGS = 0x00; constexpr std::uint64_t DEFAULT_FLAGS = (FLAG_GBZ_TAGS | FLAG_METADATA | FLAG_TRANSLATION); // Do not call directly in tests. void compare_tags(const gbwt::Tags& tags, const gbwt::Tags& truth, const std::string& test_case) { EXPECT_EQ(tags.size(), truth.size()) << "Wrong number of tags" << test_case; size_t min_size = std::min(tags.size(), truth.size()); size_t i = 0; for(auto first = tags.tags.begin(), second = truth.tags.begin(); i < min_size; ++first, ++second, i++) { EXPECT_EQ(*first, *second) << "Wrong tag at index " << i << test_case; } } // TODO: We have a similar comparison function in GBWT tests. void compare_gbwts(const gbwt::GBWT& index, const gbwt::GBWT& truth, std::uint64_t flags, const std::string& test_case_name) { std::string test_case = (test_case_name.empty() ? std::string() : std::string(" in ") + test_case_name); bool check_metadata = ((flags & FLAG_METADATA) != 0); { gbwt::GBWTHeader index_header = index.header; gbwt::GBWTHeader truth_header = truth.header; if(!check_metadata) { index_header.unset(gbwt::GBWTHeader::FLAG_METADATA); truth_header.unset(gbwt::GBWTHeader::FLAG_METADATA); } bool same_header = (index_header == truth_header); if(!same_header) { EXPECT_EQ(index_header.sequences, truth_header.sequences) << "Wrong number of sequences in GBWT header" << test_case; EXPECT_EQ(index_header.size, truth_header.size) << "Wrong size in GBWT header" << test_case; EXPECT_EQ(index_header.offset, truth_header.offset) << "Wrong offset in GBWT header" << test_case; EXPECT_EQ(index_header.alphabet_size, truth_header.alphabet_size) << "Wrong alphabet size in GBWT header" << test_case; EXPECT_EQ(index_header.flags, truth_header.flags) << "Wrong flags in GBWT header" << test_case; } ASSERT_TRUE(same_header) << "Wrong GBWT header" << test_case; } bool same_tags = (index.tags == truth.tags); if(!same_tags) { compare_tags(index.tags, truth.tags, test_case); } ASSERT_TRUE(same_tags) << "Wrong GBWT tags" << test_case; bool same_recordarray_index = (index.bwt.index == truth.bwt.index); ASSERT_TRUE(same_recordarray_index) << "Wrong index for GBWT records" << test_case; bool same_recordarray_data = (index.bwt.data == truth.bwt.data); ASSERT_TRUE(same_recordarray_data) << "Wrong data for GBWT records" << test_case; bool same_dasamples_sampled = (index.da_samples.sampled_records == truth.da_samples.sampled_records); if(!same_dasamples_sampled) { EXPECT_EQ(index.da_samples.sampled_records.size(), truth.da_samples.sampled_records.size()) << "Wrong number of records in DA samples" << test_case; size_t min_size = std::min(index.da_samples.sampled_records.size(), truth.da_samples.sampled_records.size()); for(size_t i = 0; i < min_size; i++) { EXPECT_EQ(index.da_samples.sampled_records[i], truth.da_samples.sampled_records[i]) << "Wrong sampled status for record " << i << " in DA samples" << test_case; } } ASSERT_TRUE(same_dasamples_sampled) << "Wrong sampled records in DA samples" << test_case; bool same_dasamples_ranges = (index.da_samples.bwt_ranges == truth.da_samples.bwt_ranges); ASSERT_TRUE(same_dasamples_ranges) << "Wrong BWT ranges in DA samples" << test_case; bool same_dasamples_offsets = (index.da_samples.sampled_offsets == truth.da_samples.sampled_offsets); ASSERT_TRUE(same_dasamples_offsets) << "Wrong sampled offsets in DA samples" << test_case; bool same_dasamples_samples = (index.da_samples.array == truth.da_samples.array); ASSERT_TRUE(same_dasamples_samples) << "Wrong samples in DA samples" << test_case; if(check_metadata) { bool same_metadata = (index.metadata == truth.metadata); ASSERT_TRUE(same_metadata) << "Wrong GBWT metadata" << test_case; } } void compare_graphs(const gbwtgraph::GBWTGraph& graph, const gbwtgraph::GBWTGraph& truth, std::uint64_t flags, const std::string& test_case_name) { std::string test_case = (test_case_name.empty() ? std::string() : std::string(" in ") + test_case_name); bool check_translation = ((flags & FLAG_TRANSLATION) != 0); { gbwtgraph::GBWTGraph::Header graph_header = graph.header; gbwtgraph::GBWTGraph::Header truth_header = truth.header; if(!check_translation) { graph_header.unset(gbwtgraph::GBWTGraph::Header::FLAG_TRANSLATION); truth_header.unset(gbwtgraph::GBWTGraph::Header::FLAG_TRANSLATION); } bool same_header = (graph_header == truth_header); ASSERT_TRUE(same_header) << "Wrong GBWTGraph header" << test_case; } bool same_sequences = (graph.sequences == truth.sequences); ASSERT_TRUE(same_sequences) << "Wrong sequences in GBWTGraph" << test_case; bool same_real_nodes = (graph.real_nodes == truth.real_nodes); ASSERT_TRUE(same_real_nodes) << "Wrong real nodes in GBWTGraph" << test_case; if(check_translation) { bool same_segments = (graph.segments == truth.segments); ASSERT_TRUE(same_segments) << "Wrong segment names in GBWTGraph" << test_case; bool same_translation = (graph.node_to_segment == truth.node_to_segment); ASSERT_TRUE(same_translation) << "Wrong node-to-segment translation in GBWTGraph" << test_case; } } void compare_gbzs(const gbwtgraph::GBZ& gbz, const gbwtgraph::GBZ& truth, std::uint64_t flags, const std::string& test_case_name) { std::string test_case = (test_case_name.empty() ? std::string() : std::string(" in ") + test_case_name); bool check_tags = ((flags & FLAG_GBZ_TAGS) != 0); bool same_header = (gbz.header == truth.header); ASSERT_TRUE(same_header) << "Wrong GBZ header" << test_case; if(check_tags) { bool same_tags = (gbz.tags == truth.tags); if(!same_tags) { compare_tags(gbz.tags, truth.tags, test_case); } } compare_gbwts(gbz.index, truth.index, flags, test_case_name); compare_graphs(gbz.graph, truth.graph, flags, test_case_name); } //------------------------------------------------------------------------------ // Common HandleGraph tests. typedef std::pair node_type; // (id, sequence) typedef std::pair gbwt_edge; typedef std::pair> translation_type; // Set in_order to expect the handles in the given order. void handle_graph_handles(const handlegraph::HandleGraph& graph, const std::vector& nodes, bool in_order) { std::set visited; size_t index = 0; graph.for_each_handle([&](const handle_t& handle) { nid_t id = graph.get_id(handle); if(in_order) { EXPECT_EQ(id, nodes[index].first) << "Incorrect handle at index " << index; } EXPECT_FALSE(graph.get_is_reverse(handle)) << "Handle at index " << index << " is in reverse orientation for node " << id; handle_t flipped = graph.flip(handle); EXPECT_EQ(graph.get_id(flipped), id) << "Flipped handle at index " << index << " has incorrect id for node " << id; EXPECT_TRUE(graph.get_is_reverse(flipped)) << "Flipped handle at index " << index << " is in forward orientation for node " << id; handle_t flipped_from_graph = graph.get_handle(id, true); EXPECT_EQ(flipped, flipped_from_graph) << "Flipped handle at index " << index << " does not match handle from get_handle for node " << id; handle_t double_flipped = graph.flip(flipped); EXPECT_EQ(double_flipped, handle) << "Double flipped handle at index " << index << " does not match original handle for node " << id; index++; visited.insert(id); }, false); EXPECT_EQ(index, nodes.size()) << "Incorrect number of handles visited"; for(const node_type& node : nodes) { EXPECT_TRUE(visited.find(node.first) != visited.end()) << "Node id " << node.first << " was not visited"; } // Also test parallel for_each_handle. int old_thread_count = omp_get_max_threads(); omp_set_num_threads(2); std::array, 2> thread_visits; graph.for_each_handle([&](const handle_t& handle) -> bool { int thread_id = omp_get_thread_num(); nid_t id = graph.get_id(handle); bool is_reverse = graph.get_is_reverse(handle); thread_visits[thread_id].push_back(gbwt::Node::encode(id, is_reverse)); return true; }, true); omp_set_num_threads(old_thread_count); std::vector combined_visits = thread_visits[0]; combined_visits.insert(combined_visits.end(), thread_visits[1].begin(), thread_visits[1].end()); gbwt::removeDuplicates(combined_visits, false); ASSERT_EQ(combined_visits.size(), visited.size()) << "Parallel for_each_handle visited wrong number of handles"; for(size_t i = 0; i < combined_visits.size(); i++) { nid_t id = gbwt::Node::id(combined_visits[i]); EXPECT_TRUE(visited.find(id) != visited.end()) << "Wrong handle at index " << i; } } void handle_graph_nodes(const handlegraph::HandleGraph& graph, const std::vector& nodes) { nid_t min_id = std::numeric_limits::max(), max_id = 0; for(const node_type& node : nodes) { if(node.first < min_id) { min_id = node.first; } if(node.first > max_id) { max_id = node.first; } } ASSERT_EQ(graph.get_node_count(), nodes.size()) << "Incorrect number of nodes"; EXPECT_EQ(graph.min_node_id(), min_id) << "Incorrect minimum node id"; EXPECT_EQ(graph.max_node_id(), max_id) << "Incorrect maximum node id"; for(const node_type& node : nodes) { ASSERT_TRUE(graph.has_node(node.first)) << "Node id " << node.first << " is missing"; handle_t handle = graph.get_handle(node.first, false); EXPECT_EQ(graph.get_length(handle), node.second.length()) << "Invalid sequence length for node " << node.first; EXPECT_EQ(graph.get_sequence(handle), node.second) << "Invalid sequence for node " << node.first; std::string reverse = gbwtgraph::reverse_complement(node.second); handle_t rev_handle = graph.get_handle(node.first, true); EXPECT_EQ(graph.get_length(rev_handle), node.second.length()) << "Invalid sequence length for reverse node " << node.first; EXPECT_EQ(graph.get_sequence(rev_handle), reverse) << "Invalid sequence for reverse node " << node.first; for(size_t i = 0; i < node.second.length(); i++) { EXPECT_EQ(graph.get_base(handle, i), node.second[i]) << "Invalid base at index " << i << " for node " << node.first; EXPECT_EQ(graph.get_base(rev_handle, i), reverse[i]) << "Invalid base at index " << i << " for reverse node " << node.first; EXPECT_EQ(graph.get_subsequence(handle, i, 2), node.second.substr(i, 2)) << "Invalid subsequence at index " << i << " for node " << node.first; EXPECT_EQ(graph.get_subsequence(rev_handle, i, 2), reverse.substr(i, 2)) << "Invalid subsequence at index " << i << " for reverse node " << node.first; } } nid_t missing_id = graph.max_node_id() + 100; EXPECT_FALSE(graph.has_node(missing_id)) << "Missing node id " << missing_id << " is reported present"; } void handle_graph_edges(const handlegraph::HandleGraph& graph, const std::vector& nodes, const std::set& edges) { ASSERT_EQ(graph.get_edge_count(), edges.size()) << "Incorrect number of edges"; std::set reverse_edges; for(const auto& edge : edges) { reverse_edges.insert({ gbwt::Node::reverse(edge.second), gbwt::Node::reverse(edge.first) }); } // follow_edges and get_degree std::set fw_succ, fw_pred, rev_succ, rev_pred; for(const auto& node : nodes) { handle_t forward_handle = graph.get_handle(node.first, false); handle_t reverse_handle = graph.get_handle(node.first, true); size_t fw_out = 0, fw_in = 0, rev_out = 0, rev_in = 0; graph.follow_edges(forward_handle, false, [&](const handle_t& handle) { gbwt::node_type from = gbwt::Node::encode(node.first, false); gbwt::node_type to = gbwt::Node::encode(graph.get_id(handle), graph.get_is_reverse(handle)); fw_succ.insert(gbwt_edge(from, to)); fw_out++; }); graph.follow_edges(forward_handle, true, [&](const handle_t& handle) { gbwt::node_type from = gbwt::Node::encode(graph.get_id(handle), graph.get_is_reverse(handle)); gbwt::node_type to = gbwt::Node::encode(node.first, false); fw_pred.insert(gbwt_edge(from, to)); fw_in++; }); graph.follow_edges(reverse_handle, false, [&](const handle_t& handle) { gbwt::node_type from = gbwt::Node::encode(node.first, true); gbwt::node_type to = gbwt::Node::encode(graph.get_id(handle), graph.get_is_reverse(handle)); rev_succ.insert(gbwt_edge(from, to)); rev_out++; }); graph.follow_edges(reverse_handle, true, [&](const handle_t& handle) { gbwt::node_type from = gbwt::Node::encode(graph.get_id(handle), graph.get_is_reverse(handle)); gbwt::node_type to = gbwt::Node::encode(node.first, true); rev_pred.insert(gbwt_edge(from, to)); rev_in++; }); EXPECT_EQ(graph.get_degree(forward_handle, false), fw_out) << "Wrong outdegree for forward handle " << node.first; EXPECT_EQ(graph.get_degree(forward_handle, true), fw_in) << "Wrong indegree for forward handle " << node.first; EXPECT_EQ(graph.get_degree(reverse_handle, false), rev_out) << "Wrong outdegree for reverse handle " << node.first; EXPECT_EQ(graph.get_degree(reverse_handle, true), rev_in) << "Wrong indegree for reverse handle " << node.first; } EXPECT_EQ(fw_succ, edges) << "Wrong forward successors"; EXPECT_EQ(fw_pred, edges) << "Wrong forward predecessors"; EXPECT_EQ(rev_succ, reverse_edges) << "Wrong reverse successors"; EXPECT_EQ(rev_pred, reverse_edges) << "Wrong reverse predecessors"; // Presence / absence of edges for(nid_t from = graph.min_node_id(); from <= graph.max_node_id(); from++) { for(nid_t to = graph.min_node_id(); to <= graph.max_node_id(); to++) { for(bool from_rev : { false, true }) { for(bool to_rev : { false, true }) { handle_t from_handle = graph.get_handle(from, from_rev); handle_t to_handle = graph.get_handle(to, to_rev); gbwt_edge edge(gbwt::Node::encode(from, from_rev), gbwt::Node::encode(to, to_rev)); bool should_have = (edges.find(edge) != edges.end()); should_have |= (reverse_edges.find(edge) != reverse_edges.end()); EXPECT_EQ(graph.has_edge(from_handle, to_handle), should_have) << "has_edge() failed with (" << from << ", " << from_rev << ") to (" << to << ", " << to_rev <<")"; } } } } } //------------------------------------------------------------------------------ // Construction of the basic test case. gbwt::vector_type alt_path { static_cast(gbwt::Node::encode(1, false)), static_cast(gbwt::Node::encode(2, false)), static_cast(gbwt::Node::encode(4, false)), static_cast(gbwt::Node::encode(5, false)), static_cast(gbwt::Node::encode(6, false)), static_cast(gbwt::Node::encode(8, false)), static_cast(gbwt::Node::encode(9, false)) }; gbwt::vector_type short_path { static_cast(gbwt::Node::encode(1, false)), static_cast(gbwt::Node::encode(4, false)), static_cast(gbwt::Node::encode(5, false)), static_cast(gbwt::Node::encode(6, false)), static_cast(gbwt::Node::encode(7, false)), static_cast(gbwt::Node::encode(9, false)) }; gbwt::vector_type empty_path { }; // Build a GBWT index for the given paths without metadata. inline gbwt::GBWT build_gbwt(const std::vector& paths) { gbwt::size_type node_width = 1, total_length = 0; for(auto& path : paths) { for(auto node : path) { node_width = std::max(node_width, gbwt::size_type(sdsl::bits::length(gbwt::Node::encode(node, true)))); } total_length += 2 * (path.size() + 1); } gbwt::Verbosity::set(gbwt::Verbosity::SILENT); gbwt::GBWTBuilder builder(node_width, total_length); for(auto& path : paths) { builder.insert(path, true); } builder.finish(); return gbwt::GBWT(builder.index); } // Build a GBWT with three paths including a duplicate, plus n empty paths. inline gbwt::GBWT build_gbwt_index(size_t empty_paths = 0) { std::vector paths { short_path, alt_path, short_path }; for(size_t i = 0; i < empty_paths; i++) { paths.push_back(empty_path); } return build_gbwt(paths); } // Build a GBWT with 6 paths, 3 duplicates of each. // TODO: what's the "ref" here? // This should be identical to gfas/example_walks.gfa inline gbwt::GBWT build_gbwt_index_with_ref() { std::vector paths { short_path, alt_path, alt_path, short_path, alt_path, short_path }; return build_gbwt(paths); } // This should be identical to gfas/example_walks.gfa, modulo name ordering. inline gbwt::GBWT build_gbwt_example_walks() { gbwt::GBWT built = build_gbwt_index_with_ref(); built.addMetadata(); // Name the set of samples, including a special ref one for generic paths built.metadata.setSamples({gbwtgraph::GENERIC_PATH_SAMPLE_NAME, "short", "alt"}); // Name the set of contigs we are over. built.metadata.setContigs({"short", "alt1", "alt2", "chr"}); gbwt::PathName p_short; p_short.sample = 0; p_short.contig = 0; p_short.phase = 0; p_short.count = 0; built.metadata.addPath(p_short); gbwt::PathName p_alt1; p_alt1.sample = 0; p_alt1.contig = 1; p_alt1.phase = 0; p_alt1.count = 0; built.metadata.addPath(p_alt1); gbwt::PathName p_alt2; p_alt2.sample = 0; p_alt2.contig = 2; p_alt2.phase = 0; p_alt2.count = 0; built.metadata.addPath(p_alt2); gbwt::PathName w_short1; w_short1.sample = 1; w_short1.contig = 3; w_short1.phase = 1; w_short1.count = 0; built.metadata.addPath(w_short1); gbwt::PathName w_alt; w_alt.sample = 2; w_alt.contig = 3; w_alt.phase = 0; w_alt.count = 0; built.metadata.addPath(w_alt); gbwt::PathName w_short2; w_short2.sample = 1; w_short2.contig = 3; w_short2.phase = 2; w_short2.count = 0; built.metadata.addPath(w_short2); // Record that we have 6 total haplotypes in the GBWT. built.metadata.setHaplotypes(6); return built; } // This should be identical to gfas/example_reference.gfa, modulo name ordering. inline gbwt::GBWT build_gbwt_example_reference() { gbwt::GBWT built = build_gbwt_index_with_ref(); built.addMetadata(); // Name the set of samples, including a special one for generic paths built.metadata.setSamples({gbwtgraph::GENERIC_PATH_SAMPLE_NAME, "GRCh38", "GRCh37", "sample1", "CHM13"}); // Set which are references built.tags.set(gbwtgraph::REFERENCE_SAMPLE_LIST_GBWT_TAG, "GRCh37" + std::string(1, gbwtgraph::REFERENCE_SAMPLE_LIST_SEPARATOR) + "GRCh38"); // Name the set of contigs we are over. built.metadata.setContigs({"chr1", "coolgene"}); gbwt::PathName p_grch38_chr1; p_grch38_chr1.sample = 1; p_grch38_chr1.contig = 0; p_grch38_chr1.phase = 0; p_grch38_chr1.count = 0; built.metadata.addPath(p_grch38_chr1); gbwt::PathName p_grch37_chr1; p_grch37_chr1.sample = 2; p_grch37_chr1.contig = 0; p_grch37_chr1.phase = 0; p_grch37_chr1.count = 0; built.metadata.addPath(p_grch37_chr1); gbwt::PathName p_coolgene; p_coolgene.sample = 0; p_coolgene.contig = 1; p_coolgene.phase = 0; p_coolgene.count = 0; built.metadata.addPath(p_coolgene); gbwt::PathName w_sample1_1; w_sample1_1.sample = 3; w_sample1_1.contig = 0; w_sample1_1.phase = 1; w_sample1_1.count = 0; built.metadata.addPath(w_sample1_1); gbwt::PathName w_chm13; w_chm13.sample = 4; w_chm13.contig = 0; w_chm13.phase = 0; w_chm13.count = 0; built.metadata.addPath(w_chm13); gbwt::PathName w_sample1_2; w_sample1_2.sample = 3; w_sample1_2.contig = 0; w_sample1_2.phase = 2; w_sample1_2.count = 0; built.metadata.addPath(w_sample1_2); // Record that we have 6 total haplotypes in the GBWT. built.metadata.setHaplotypes(6); return built; } // Build a GBWT that contains 3 paths, two of which are named, reference paths in the metadata. inline gbwt::GBWT build_gbwt_index_with_named_paths() { // Start with the no-metadata GBWT we usually use, but with some empty paths. gbwt::GBWT built = build_gbwt_index(2); built.addMetadata(); // Name the set of samples, including a special one for generic paths built.metadata.setSamples({gbwtgraph::GENERIC_PATH_SAMPLE_NAME, "Jouni Sirén", "GRCh38"}); // Name the set of contigs we are over. built.metadata.setContigs({"chr1", "chr2", "empty1", "empty2"}); // We have a ref path on the first contig, and a ref path on the second // contig, and a sample path on the first contig. gbwt::PathName ref1; ref1.sample = 0; ref1.contig = 0; ref1.phase = 0; ref1.count = 0; built.metadata.addPath(ref1); gbwt::PathName ref2; ref2.sample = 0; ref2.contig = 1; ref2.phase = 0; ref2.count = 0; built.metadata.addPath(ref2); gbwt::PathName sample1; sample1.sample = 1; sample1.contig = 0; sample1.phase = 0; sample1.count = 0; built.metadata.addPath(sample1); gbwt::PathName empty1; empty1.sample = 0; empty1.contig = 2; empty1.phase = 0; empty1.count = 0; built.metadata.addPath(empty1); gbwt::PathName empty2; empty2.sample = 2; empty2.contig = 3; empty2.phase = gbwtgraph::GBWTGraph::NO_PHASE; empty2.count = 0; built.metadata.addPath(empty2); // Record that we have 5 total haplotypes in the GBWT. built.metadata.setHaplotypes(5); // Make GRCh38 a reference built.tags.set(gbwtgraph::REFERENCE_SAMPLE_LIST_GBWT_TAG, "GRCh38"); return built; } // Dump a GBWT and its metadata. inline void dump_gbwt(const gbwt::GBWT& built) { // Check to make sure threads and metadata match. if (built.hasMetadata()) { for(size_t path_num = 0; path_num < built.metadata.paths(); path_num++) { auto extracted = built.extract(gbwt::Path::encode(path_num, false)); std::cerr << "GBWT stored forward path " << path_num << std::endl; auto path_name = built.metadata.path(path_num); std::cerr << "\tSample " << path_name.sample << " Contig " << path_name.contig << " Phase " << path_name.phase << " Count " << path_name.count << std::endl; for(auto& gbwt_node : extracted) { std::cerr << "\t" << gbwt::Node::id(gbwt_node) << " " << gbwt::Node::is_reverse(gbwt_node) << std::endl; } } } } // Builds a NaiveGraph that could contain `alt_path` and `short_path`. inline gbwtgraph::NaiveGraph build_naive_graph(bool with_translation, bool lower_case = false) { gbwtgraph::NaiveGraph graph; if(with_translation) { std::string seq = (lower_case ? "gatgggtacaa" : "GATGGGTACAA"); graph.translate_segment("s1", std::string_view(seq.data() + 0, 1), 3); graph.translate_segment("s2", std::string_view(seq.data() + 1, 1), 3); graph.translate_segment("s3", std::string_view(seq.data() + 2, 1), 3); graph.translate_segment("s4", std::string_view(seq.data() + 3, 4), 3); graph.translate_segment("s5", std::string_view(seq.data() + 7, 1), 3); graph.translate_segment("s6", std::string_view(seq.data() + 8, 1), 3); graph.translate_segment("s7", std::string_view(seq.data() + 9, 1), 3); graph.translate_segment("s8", std::string_view(seq.data() + 10, 1), 3); } else { graph.create_node(1, (lower_case ? "g" : "G")); graph.create_node(2, (lower_case ? "a" : "A")); graph.create_node(3, (lower_case ? "t" : "T")); graph.create_node(4, (lower_case ? "ggg" : "GGG")); graph.create_node(5, (lower_case ? "t" : "T")); graph.create_node(6, (lower_case ? "a" : "A")); graph.create_node(7, (lower_case ? "c" : "C")); graph.create_node(8, (lower_case ? "a" : "A")); graph.create_node(9, (lower_case ? "a" : "A")); } // The paths are 1, 2, 4, 5, 6, 8, 9 and 1, 4, 5, 6, 7, 9. graph.create_edge(gbwt::Node::encode(1, false), gbwt::Node::encode(2, false)); graph.create_edge(gbwt::Node::encode(2, false), gbwt::Node::encode(4, false)); graph.create_edge(gbwt::Node::encode(4, false), gbwt::Node::encode(5, false)); graph.create_edge(gbwt::Node::encode(5, false), gbwt::Node::encode(6, false)); graph.create_edge(gbwt::Node::encode(6, false), gbwt::Node::encode(7, false)); graph.create_edge(gbwt::Node::encode(6, false), gbwt::Node::encode(8, false)); graph.create_edge(gbwt::Node::encode(7, false), gbwt::Node::encode(9, false)); graph.create_edge(gbwt::Node::encode(8, false), gbwt::Node::encode(9, false)); graph.remove_duplicate_edges(); return graph; } //------------------------------------------------------------------------------ // Helpers for k-mer / minimizer indexes. typedef std::pair> owned_value_type; typedef std::vector owned_multi_value_type; inline owned_value_type create_value(gbwtgraph::KmerEncoding::value_type value, size_t payload_size) { return std::make_pair(value.first, std::vector(value.second, value.second + payload_size)); } inline owned_value_type create_value(pos_t pos, size_t payload_size, std::uint64_t payload) { return std::make_pair(gbwtgraph::Position(pos), std::vector(payload_size, payload)); } inline void append_value(owned_multi_value_type& values, const owned_value_type& value, size_t payload_size) { constexpr size_t POS_SIZE = sizeof(gbwtgraph::Position) / sizeof(std::uint64_t); size_t offset = values.size(); values.resize(offset + POS_SIZE + payload_size); value.first.write(values.data() + offset); offset += POS_SIZE; for(size_t j = 0; j < payload_size; j++) { values[offset + j] = value.second[j]; } } // If with_paths is true, the last word of payload encodes the set of paths that contain the hit. // This may be a superset of the set encoded in the truth value. // If paths A and B both contain subpath X, the kmer corresponding to X may or may not be a // minimizer, depending on the context. inline bool same_value(gbwtgraph::KmerEncoding::value_type value, const owned_value_type& truth, size_t payload_size, bool with_paths) { if(value.first != truth.first) { std::cerr << "Wrong pos" << std::endl; return false; } std::vector payload(value.second, value.second + payload_size); if(with_paths && payload_size > 0) { if((payload.back() & truth.second.back()) != truth.second.back()) { std::cerr << "Wrong paths in payload" << std::endl; std::cerr << " Got: " << payload.back() << std::endl; std::cerr << " Expected at least: " << truth.second.back() << std::endl; return false; } payload.back() = truth.second.back(); } if(payload != truth.second) { std::cerr << "Wrong payload" << std::endl; std::cerr << " Got:"; for(auto p : payload) { std::cerr << " " << p; } std::cerr << std::endl; std::cerr << " Expected:"; for(auto p : truth.second) { std::cerr << " " << p; } std::cerr << std::endl; return false; } return true; } // See above for with_paths. inline bool same_values(gbwtgraph::KmerEncoding::multi_value_type values, const std::set& truth, size_t payload_size, bool with_paths) { constexpr size_t POS_SIZE = sizeof(gbwtgraph::Position) / sizeof(std::uint64_t); if(values.second != truth.size()) { std::cerr << "Wrong size: " << values.second << " != " << truth.size() << std::endl; return false; } size_t value_offset = 0; for(auto& correct : truth) { gbwtgraph::Position pos(values.first[value_offset]); if(pos != correct.first) { std::cerr << "Wrong pos" << std::endl; return false; } value_offset += POS_SIZE; std::vector payload(values.first + value_offset, values.first + value_offset + payload_size); if(with_paths && payload_size > 0) { if((payload.back() & correct.second.back()) != correct.second.back()) { std::cerr << "Wrong paths in payload" << std::endl; std::cerr << " Got: " << payload.back() << std::endl; std::cerr << " Expected at least: " << correct.second.back() << std::endl; return false; } payload.back() = correct.second.back(); } if(payload != correct.second) { std::cerr << "Wrong payload" << std::endl; std::cerr << " Got:"; for(auto p : payload) { std::cerr << " " << p; } std::cerr << std::endl; std::cerr << " Expected:"; for(auto p : correct.second) { std::cerr << " " << p; } std::cerr << std::endl; return false; } value_offset += payload_size; } return true; } template void insert_value(gbwtgraph::KmerIndex& index, KeyType key, const owned_value_type& value) { index.insert(key, std::make_pair(value.first, value.second.data())); } template void insert_value ( gbwtgraph::MinimizerIndex& index, typename gbwtgraph::MinimizerIndex::minimizer_type key, const owned_value_type& value ) { index.insert(key, std::make_pair(value.first, value.second.data())); } template gbwtgraph::Kmer get_minimizer(KeyType key, gbwtgraph::offset_type offset = 0, bool orientation = false) { return { key, key.hash(), offset, orientation }; } template gbwtgraph::Kmer get_minimizer(std::string key, gbwtgraph::offset_type offset = 0, bool orientation = false) { return get_minimizer(KeyType::encode(key), offset, orientation); } //------------------------------------------------------------------------------ inline std::string path_name_to_string(const gbwt::PathName& path_name) { std::string result = "(" + std::to_string(path_name.sample) + ", " + std::to_string(path_name.contig) + ", " + std::to_string(path_name.phase) + ", " + std::to_string(path_name.count) + ")"; return result; } inline std::string path_to_string(const gbwtgraph::GBWTGraph& graph, const gbwt::vector_type& path) { std::string str; for(gbwt::node_type node : path) { std::string_view view = graph.get_sequence_view(gbwtgraph::GBWTGraph::node_to_handle(node)); str.append(view.data(), view.size()); } return str; } //------------------------------------------------------------------------------ } // anonymous namespace #endif // GBWTGRAPH_TESTS_SHARED_H jltsiren-gbwtgraph-e27bc43/tests/test_algorithms.cpp000066400000000000000000000704461524136466400230460ustar00rootroot00000000000000#include #include #include #include #include #include #include #include "shared.h" using namespace gbwtgraph; namespace { //------------------------------------------------------------------------------ class ComponentTest : public ::testing::Test { public: gbwt::GBWT index; GBWTGraph graph; size_t components; std::vector> correct_paths; ComponentTest() { } void SetUp() override { auto gfa_parse = gfa_to_gbwt("gfas/components.gfa"); this->index = *(gfa_parse.first); this->graph = GBWTGraph(this->index, *(gfa_parse.second)); this->components = 2; } void check_path_assignments( const GBWTGraph& graph, const std::vector>& result, const std::vector>& correct, const std::string& filename) const { ASSERT_EQ(result.size(), correct.size()) << "Wrong number of components in " << filename; for(size_t i = 0; i < result.size(); i++) { const std::vector& paths = result[i]; const std::vector& correct_paths = correct[i]; ASSERT_EQ(paths.size(), correct_paths.size()) << "Wrong number of paths in component " << i << " in " << filename; for(size_t j = 0; j < paths.size(); j++) { size_t path_id = graph.handle_to_path(paths[j]); EXPECT_EQ(path_id, correct_paths[j]) << "Incorrect path " << j << " in component " << i << " in " << filename; } } } }; TEST_F(ComponentTest, Components) { std::vector> correct_components = { { 11, 12, 13, 14, 15, 16, 17 }, { 21, 22, 23, 24, 25 } }; std::vector> result = weakly_connected_components(this->graph); ASSERT_EQ(result.size(), correct_components.size()) << "Wrong number of components"; for(size_t i = 0; i < result.size(); i++) { ASSERT_EQ(result[i].size(), correct_components[i].size()) << "Wrong number of nodes in component " << i; auto result_iter = result[i].begin(); auto correct_iter = correct_components[i].begin(); while(result_iter != result[i].end()) { EXPECT_EQ(*result_iter, *correct_iter) << "Incorrect node in component " << i; ++result_iter; ++correct_iter; } } } TEST_F(ComponentTest, HeadNodes) { std::vector> correct_heads = { { 11 }, { } }; std::vector> components = weakly_connected_components(this->graph); ASSERT_EQ(components.size(), correct_heads.size()) << "Wrong number of components"; // For both components, add a version with a nonexistent node id. components.emplace_back(components[0]); components.back().push_back(42); correct_heads.emplace_back(correct_heads[0]); components.emplace_back(components[1]); components.back().push_back(42); correct_heads.emplace_back(correct_heads[1]); for(size_t i = 0; i < components.size(); i++) { std::vector heads = is_nice_and_acyclic(this->graph, components[i]); ASSERT_EQ(heads.size(), correct_heads[i].size()) << "Wrong number of head nodes in component " << i; auto result_iter = heads.begin(); auto correct_iter = correct_heads[i].begin(); while(result_iter != heads.end()) { EXPECT_EQ(*result_iter, *correct_iter) << "Incorrect head node in component " << i; ++result_iter; ++correct_iter; } } } TEST_F(ComponentTest, ConstructionJobs) { std::vector> components = weakly_connected_components(this->graph); std::vector> bounds_and_jobs { { 0, 2 }, { 11, 2 }, { 12, 1 } }; for(auto params : bounds_and_jobs) { ConstructionJobs jobs = gbwt_construction_jobs(this->graph, params.first); ASSERT_EQ(jobs.size(), params.second) << "Invalid number of jobs with size bound " << params.first; if(params.second == 1) { ASSERT_EQ(jobs.job_size(0), this->graph.get_node_count()) << "Invalid size for job 0 with size bound " << params.first; } ASSERT_EQ(jobs.components(), components.size()) << "Invalid number of components with size bound " << params.first; ASSERT_EQ(jobs.weakly_connected_components, components) << "Invalid components with size bound " << params.first; // For each component, check that the mappings for each node in the component are correct. for(size_t i = 0; i < components.size(); i++) { for(nid_t id : components[i]) { EXPECT_EQ(jobs.component(id), i) << "Invalid component for node " << id << " with size bound " << params.first; } if(params.second == components.size()) { ASSERT_EQ(jobs.job_size(i), components[i].size()) << "Invalid size for job " << i << " with size bound " << params.first; } size_t expected_job = (params.second == 1 ? 0 : i); for(nid_t id : components[i]) { EXPECT_EQ(jobs.job(id), expected_job) << "Invalid job for node " << id << " with size bound " << params.first; } } // For each job, check that the mappings for each component in the job are correct. std::vector> components_per_job = jobs.components_per_job(); for(size_t job_id = 0; job_id < components_per_job.size(); job_id++) { for(size_t component_id : components_per_job[job_id]) { EXPECT_EQ(jobs.job_for_component(component_id), job_id) << "Invalid job for component " << component_id << " with size bound " << params.first; } } } } TEST_F(ComponentTest, ContigNames) { std::vector>> files_and_components = { { "gfas/components.gfa", { "A1", "B1" } }, { "gfas/components_walks.gfa", { "component_0", "component_1" } }, { "gfas/components_ref.gfa", { "A", "B" } }, { "gfas/default.gfa", { "A", "B" } } }; for(auto params : files_and_components) { auto gfa_parse = gfa_to_gbwt(params.first); const gbwt::GBWT& index = *(gfa_parse.first); GBWTGraph graph(index, *(gfa_parse.second)); ConstructionJobs jobs = gbwt_construction_jobs(graph, 0); std::vector names = jobs.contig_names(graph); ASSERT_EQ(names.size(), params.second.size()) << "Wrong number of contig names in " << params.first; for(size_t i = 0; i < names.size(); i++) { EXPECT_EQ(names[i], params.second[i]) << "Incorrect contig name " << i << " in " << params.first; } } } TEST_F(ComponentTest, ContigNamesWithFilter) { std::vector>> files_and_components = { { "gfas/components.gfa", { "A1", "B1" } }, { "gfas/components_walks.gfa", { "component_0", "component_1" } }, { "gfas/components_ref.gfa", { "component_0", "component_1" } }, { "gfas/default.gfa", { "A", "B" } } }; for(auto params : files_and_components) { auto gfa_parse = gfa_to_gbwt(params.first); const gbwt::GBWT& index = *(gfa_parse.first); GBWTGraph graph(index, *(gfa_parse.second)); std::function generic_filter = [&](const path_handle_t& path) -> bool { return (graph.get_sense(path) == PathSense::GENERIC); }; ConstructionJobs jobs = gbwt_construction_jobs(graph, 0); std::vector names = jobs.contig_names(graph, generic_filter); ASSERT_EQ(names.size(), params.second.size()) << "Wrong number of contig names in " << params.first; for(size_t i = 0; i < names.size(); i++) { EXPECT_EQ(names[i], params.second[i]) << "Incorrect contig name " << i << " in " << params.first; } } } TEST_F(ComponentTest, AssignPaths) { std::vector>>> files_and_paths = { { "gfas/components.gfa", { { 0, 1 }, { 2, 3 } } }, { "gfas/components_walks.gfa", { { }, { } } }, { "gfas/components_ref.gfa", { { 0 }, { 3 } } }, { "gfas/default.gfa", { { 0 }, { 3 } } } }; for(auto params : files_and_paths) { auto gfa_parse = gfa_to_gbwt(params.first); const gbwt::GBWT& index = *(gfa_parse.first); GBWTGraph graph(index, *(gfa_parse.second)); ConstructionJobs jobs = gbwt_construction_jobs(graph, 0); auto result = assign_paths(graph, jobs, nullptr, nullptr); this->check_path_assignments(graph, result, params.second, params.first); } } TEST_F(ComponentTest, AssignPathsWithFilter) { std::vector>>> files_and_paths = { { "gfas/components.gfa", { { 0, 1 }, { 2, 3 } } }, { "gfas/components_walks.gfa", { { }, { } } }, { "gfas/components_ref.gfa", { { }, { } } }, { "gfas/default.gfa", { { 0 }, { 3 } } } }; for(auto params : files_and_paths) { auto gfa_parse = gfa_to_gbwt(params.first); const gbwt::GBWT& index = *(gfa_parse.first); GBWTGraph graph(index, *(gfa_parse.second)); std::function generic_filter = [&](const path_handle_t& path) -> bool { return (graph.get_sense(path) == PathSense::GENERIC); }; ConstructionJobs jobs = gbwt_construction_jobs(graph, 0); auto result = assign_paths(graph, jobs, nullptr, &generic_filter); this->check_path_assignments(graph, result, params.second, params.first); } } TEST_F(ComponentTest, InsertPaths) { std::vector>>> files_and_paths = { { "gfas/components.gfa", { { 0, 1 }, { 2, 3 } } }, { "gfas/components_walks.gfa", { { }, { } } }, { "gfas/components_ref.gfa", { { 0 }, { 3 } } }, { "gfas/default.gfa", { { 0 }, { 3 } } } }; for(auto params : files_and_paths) { auto gfa_parse = gfa_to_gbwt(params.first); const gbwt::GBWT& index = *(gfa_parse.first); GBWTGraph graph(index, *(gfa_parse.second)); MetadataBuilder metadata_builder; // Assign paths to jobs. ConstructionJobs jobs = gbwt_construction_jobs(graph, 0); auto assigned = assign_paths(graph, jobs, &metadata_builder, nullptr); // Build a GBWT with the assigned paths. gbwt::size_type node_width = sdsl::bits::length(gbwt::Node::encode(graph.max_node_id(), true)); gbwt::GBWTBuilder builder(node_width, index.size()); for(size_t job = 0; job < assigned.size(); job++) { insert_paths(graph, assigned[job], builder, job, false); } builder.finish(); gbwt::GBWT built = builder.index; // Add metadata. built.addMetadata(); built.metadata = metadata_builder.get_metadata(); if(index.tags.contains(REFERENCE_SAMPLE_LIST_GBWT_TAG)) { built.tags.set(REFERENCE_SAMPLE_LIST_GBWT_TAG, index.tags.get(REFERENCE_SAMPLE_LIST_GBWT_TAG)); } GBWTGraph built_graph(built, graph, nullptr); // Check that the paths are correct. gbwt::size_type path_id = 0; for(size_t job = 0; job < assigned.size(); job++) { for(size_t i = 0; i < assigned[job].size(); i++) { path_handle_t handle = built_graph.path_to_handle(path_id); path_handle_t old_handle = assigned[job][i]; EXPECT_EQ(built_graph.get_path_name(handle), graph.get_path_name(old_handle)) << "Incorrect path name for path " << path_id << " in job " << job; EXPECT_EQ(built_graph.get_sense(handle), graph.get_sense(old_handle)) << "Incorrect path sense for path " << path_id << " in job " << job; gbwt::vector_type built_path = built.extract(gbwt::Path::encode(path_id, false)); gbwt::vector_type correct_path = index.extract(gbwt::Path::encode(graph.handle_to_path(old_handle), false)); EXPECT_EQ(built_path, correct_path) << "Incorrect path sequence for path " << path_id << " in job " << job; path_id++; } } } } //------------------------------------------------------------------------------ class TopologicalOrderTest : public ::testing::Test { public: gbwt::GBWT index; GBWTGraph graph; TopologicalOrderTest() { } void SetUp() override { auto gfa_parse = gfa_to_gbwt("gfas/cyclic.gfa"); this->index = *(gfa_parse.first); this->graph = GBWTGraph(this->index, *(gfa_parse.second)); } void check_subgraph(const std::unordered_set& subgraph, bool acyclic) const { std::vector order = topological_order(this->graph, subgraph); if(!acyclic) { ASSERT_TRUE(order.empty()) << "Non-empty order for a subgraph containing cycles"; return; } // Determine the node ids that do not exist in the graph. size_t missing_nodes = 0; for(nid_t node : subgraph) { if(!(this->graph.has_node(node))) { missing_nodes++; } } ASSERT_EQ(order.size(), 2 * (subgraph.size() - missing_nodes)) << "Wrong number of handles in the order"; for(nid_t node : subgraph) { if(!(this->graph.has_node(node))) { continue; } for(bool orientation : { false, true }) { handle_t from = this->graph.get_handle(node, orientation); auto from_iter = std::find(order.begin(), order.end(), from); ASSERT_NE(from_iter, order.end()) << "Node " << node << ", orientation " << orientation << " not found in the order"; bool ok = this->graph.follow_edges(from, false, [&](const handle_t& to) -> bool { if(subgraph.find(this->graph.get_id(to)) == subgraph.end()) { return true; } auto to_iter = std::find(order.begin(), order.end(), to); if(to_iter == order.end()) { return false; } return (from_iter < to_iter); }); EXPECT_TRUE(ok) << "Constraints not satisfied for node " << node << ", orientation " << orientation; } } } }; TEST_F(TopologicalOrderTest, SingleComponent) { std::unordered_set subgraph = { static_cast(1), static_cast(2), static_cast(4), static_cast(5), static_cast(6) }; this->check_subgraph(subgraph, true); } TEST_F(TopologicalOrderTest, TwoComponents) { std::unordered_set subgraph = { static_cast(1), static_cast(2), static_cast(4), static_cast(6), static_cast(7), static_cast(8), static_cast(9) }; this->check_subgraph(subgraph, true); } TEST_F(TopologicalOrderTest, CyclicComponent) { std::unordered_set subgraph = { static_cast(2), static_cast(4), static_cast(5), static_cast(6), static_cast(8) }; this->check_subgraph(subgraph, false); } TEST_F(TopologicalOrderTest, MissingNodes) { std::unordered_set subgraph = { static_cast(1), static_cast(2), static_cast(4), static_cast(5), static_cast(6), static_cast(42) }; this->check_subgraph(subgraph, true); } //------------------------------------------------------------------------------ class ChunkMergeTest : public ::testing::Test { public: // Some test graphs have reference paths as P-lines with PanSN names. GFAParsingParameters get_params() const { GFAParsingParameters params; params.path_name_formats.clear(); params.path_name_formats.emplace_back(GFAParsingParameters::PAN_SN_REGEX, GFAParsingParameters::PAN_SN_FIELDS, GFAParsingParameters::PAN_SN_SENSE); params.path_name_formats.emplace_back(GFAParsingParameters::DEFAULT_REGEX, GFAParsingParameters::DEFAULT_FIELDS, GFAParsingParameters::DEFAULT_SENSE); return params; } }; TEST_F(ChunkMergeTest, Empty) { GBZ empty; ChunkParameters params; auto chunks = chunk_graph(empty, params); ASSERT_TRUE(chunks.first.empty()) << "Non-empty chunking of an empty graph"; GBZ merged(std::move(chunks.first)); compare_gbzs(empty, merged, DEFAULT_FLAGS, ""); } TEST_F(ChunkMergeTest, SingleComponent) { GFAParsingParameters gfa_params = this->get_params(); auto gfa_parse = gfa_to_gbwt("gfas/example_reference.gfa", gfa_params); GBZ graph(gfa_parse.first, gfa_parse.second); ChunkParameters params; auto chunks = chunk_graph(graph, params); ASSERT_EQ(chunks.first.size(), size_t(1)) << "Wrong number of chunks for a single-component graph"; GBZ merged(std::move(chunks.first)); compare_gbzs(merged, graph, DEFAULT_FLAGS, ""); } TEST_F(ChunkMergeTest, TwoComponents) { GFAParsingParameters gfa_params = this->get_params(); auto gfa_parse = gfa_to_gbwt("gfas/components_ref.gfa", gfa_params); GBZ graph(gfa_parse.first, gfa_parse.second); GraphName parent = graph.graph_name(); ChunkParameters params; auto chunks = chunk_graph(graph, params); ASSERT_EQ(chunks.first.size(), size_t(2)) << "Wrong number of chunks for a two-component graph"; size_t total_nodes = 0; for(size_t i = 0; i < chunks.first.size(); i++) { GraphName chunk = chunks.first[i].graph_name(); bool is_subgraph = chunk.subgraph_of(parent); EXPECT_TRUE(is_subgraph) << "Chunk " << i << " is not a subgraph of the original graph"; size_t node_count = chunks.first[i].graph.get_node_count(); EXPECT_GT(node_count, size_t(0)) << "Chunk " << i << " is empty"; total_nodes += node_count; } EXPECT_EQ(total_nodes, graph.graph.get_node_count()) << "Total number of nodes in chunks does not match original graph"; std::vector expected_names { "A", "B" }; ASSERT_EQ(chunks.second.size(), expected_names.size()) << "Wrong number of contig names for a two-component graph"; for(size_t i = 0; i < expected_names.size(); i++) { EXPECT_EQ(chunks.second[i], expected_names[i]) << "Incorrect contig name for chunk " << i; } GBZ merged(std::move(chunks.first)); compare_gbzs(merged, graph, DEFAULT_FLAGS, ""); } TEST_F(ChunkMergeTest, WithTranslation) { GFAParsingParameters gfa_params = this->get_params(); gfa_params.max_node_length = 3; auto gfa_parse = gfa_to_gbwt("gfas/example_chopping.gfa", gfa_params); GBZ graph(gfa_parse.first, gfa_parse.second); ASSERT_TRUE(graph.graph.has_segment_names()) << "Original graph should have node-to-segment translation"; // Our original graph has a node-to-segment translation. // The corresponding GraphName tags do not survive the round-trip. graph.tags.unset(GraphName::GBZ_TRANSLATION_TAG); graph.tags.unset(GraphName::GBZ_TRANSLATION_TARGET_TAG); ChunkParameters params; auto chunks = chunk_graph(graph, params); ASSERT_EQ(chunks.first.size(), size_t(1)) << "Wrong number of chunks for a single-component graph"; ASSERT_FALSE(chunks.first[0].graph.has_segment_names()) << "Chunk should not have node-to-segment translation"; GBZ merged(std::move(chunks.first)); std::uint64_t flags = (DEFAULT_FLAGS & ~FLAG_TRANSLATION); compare_gbzs(merged, graph, flags, ""); ASSERT_FALSE(merged.graph.has_segment_names()) << "Merged graph should not have node-to-segment translation"; } TEST_F(ChunkMergeTest, ByContigName) { GFAParsingParameters gfa_params = this->get_params(); auto gfa_parse = gfa_to_gbwt("gfas/components_ref.gfa", gfa_params); GBZ graph(gfa_parse.first, gfa_parse.second); ChunkParameters params; auto chunks = chunk_graph(graph, params); for(size_t i = 0; i < chunks.first.size(); i++) { params.contig_name = chunks.second[i]; auto new_chunks = chunk_graph(graph, params); ASSERT_EQ(new_chunks.first.size(), size_t(1)) << "Wrong number of chunks for contig name " << params.contig_name; compare_gbzs(new_chunks.first[0], chunks.first[i], DEFAULT_FLAGS, "chunk " + std::to_string(i) + " with contig name " + params.contig_name); } params.contig_name = "nonexistent_contig"; auto new_chunks = chunk_graph(graph, params); ASSERT_TRUE(new_chunks.first.empty()) << "Found a chunk for a non-existent contig name"; } //------------------------------------------------------------------------------ class LCSTest : public ::testing::Test { public: gbwt::GBWT index; GBWTGraph graph; std::mt19937_64 rng; std::vector node_ids; LCSTest() : rng(0xACDCABBABABA ^ std::random_device()()) { } void SetUp() override { // This graph contains nodes 1 to 8 with variable length sequences. auto gfa_parse = gfa_to_gbwt("gfas/for_subgraph.gfa"); this->index = *(gfa_parse.first); this->graph = GBWTGraph(this->index, *(gfa_parse.second)); this->graph.for_each_handle([&](const handle_t& handle) { this->node_ids.push_back(this->graph.get_id(handle)); }); } void compare_paths(const gbwt::vector_type& left, const gbwt::vector_type& right, const std::string& name) { std::vector> result = path_lcs(this->graph, left, right); for(size_t i = 0; i < result.size(); i++) { ASSERT_LT(result[i].first, left.size()) << "Invalid left LCS position " << i << " for " << name; ASSERT_LT(result[i].second, right.size()) << "Invalid right LCS position " << i << " for " << name; if(i > 0) { ASSERT_LT(result[i - 1].first, result[i].first) << "Non-increasing left LCS position " << i << " for " << name; ASSERT_LT(result[i - 1].second, result[i].second) << "Non-increasing right LCS position " << i << " for " << name; } ASSERT_EQ(left[result[i].first], right[result[i].second]) << "Incorrect LCS element " << i << " for " << name; } } static gbwt::vector_type forward_path(const std::vector& nodes) { gbwt::vector_type result; for(nid_t node : nodes) { result.push_back(gbwt::Node::encode(node, false)); } return result; } gbwt::vector_type random_path(size_t length) { gbwt::vector_type result; std::uniform_int_distribution dist(0, this->node_ids.size() - 1); for(size_t i = 0; i < length; i++) { nid_t node = this->node_ids[dist(this->rng)]; result.push_back(gbwt::Node::encode(node, false)); } return result; } gbwt::vector_type mutate_path(const gbwt::vector_type& path, double rate) { gbwt::vector_type result = path; std::uniform_real_distribution mutate(0.0, 1.0); std::uniform_int_distribution node_dist(0, this->node_ids.size() - 1); for(size_t i = 0; i < result.size(); i++) { if(mutate(this->rng) < rate) { nid_t node = this->node_ids[node_dist(this->rng)]; result[i] = gbwt::Node::encode(node, false); } } return result; } }; TEST_F(LCSTest, EmptyLCS) { gbwt::vector_type empty; gbwt::vector_type path = this->index.extract(gbwt::Path::encode(0, false)); ASSERT_TRUE(path_lcs(this->graph, empty, empty).empty()) << "Nonempty LCS for two empty paths"; ASSERT_TRUE(path_lcs(this->graph, empty, path).empty()) << "Nonempty LCS for an empty first path"; ASSERT_TRUE(path_lcs(this->graph, path, empty).empty()) << "Nonempty LCS for an empty second path"; gbwt::vector_type reverse = this->index.extract(gbwt::Path::encode(0, true)); ASSERT_TRUE(path_lcs(this->graph, path, reverse).empty()) << "Nonempty LCS for non-overlapping paths"; } TEST_F(LCSTest, IndenticalLCS) { for(gbwt::size_type i = 0; i < this->index.sequences(); i++) { gbwt::vector_type path = this->index.extract(i); std::vector> truth; for(size_t j = 0; j < path.size(); j++) { truth.push_back(std::make_pair(j, j)); } std::vector> result = path_lcs(this->graph, path, path); gbwt::size_type id = gbwt::Path::id(i); bool orientation = gbwt::Path::is_reverse(i); ASSERT_EQ(result, truth) << "Incorrect LCS for path " << id << ", orientation " << orientation << " with itself"; } } TEST_F(LCSTest, FirstLastLCS) { gbwt::vector_type path = forward_path({ 1, 2, 3, 4, 5 }); gbwt::vector_type no_no = forward_path({ 6, 2, 4, 7 }); std::vector> no_no_left = { { 1, 1 }, { 2, 3 } }; ASSERT_EQ(path_lcs(this->graph, no_no, path), no_no_left) << "Incorrect LCS for no first, no last, left"; std::vector> no_no_right = { { 1, 1 }, { 3, 2 } }; ASSERT_EQ(path_lcs(this->graph, path, no_no), no_no_right) << "Incorrect LCS for no first, no last, right"; gbwt::vector_type no_yes = forward_path({ 6, 2, 4, 5 }); std::vector> no_yes_left = { { 1, 1 }, { 2, 3 }, { 3, 4 } }; ASSERT_EQ(path_lcs(this->graph, no_yes, path), no_yes_left) << "Incorrect LCS for no first, yes last, left"; std::vector> no_yes_right = { { 1, 1 }, { 3, 2 }, { 4, 3 } }; ASSERT_EQ(path_lcs(this->graph, path, no_yes), no_yes_right) << "Incorrect LCS for no first, yes last, right"; gbwt::vector_type yes_no = forward_path({ 1, 2, 4, 7 }); std::vector> yes_no_left = { { 0, 0 }, { 1, 1 }, { 2, 3 } }; ASSERT_EQ(path_lcs(this->graph, yes_no, path), yes_no_left) << "Incorrect LCS for yes first, no last, left"; std::vector> yes_no_right = { { 0, 0 }, { 1, 1 }, { 3, 2 } }; ASSERT_EQ(path_lcs(this->graph, path, yes_no), yes_no_right) << "Incorrect LCS for yes first, no last, right"; gbwt::vector_type yes_yes = forward_path({ 1, 2, 4, 5 }); std::vector> yes_yes_left = { { 0, 0 }, { 1, 1 }, { 2, 3 }, { 3, 4 } }; ASSERT_EQ(path_lcs(this->graph, yes_yes, path), yes_yes_left) << "Incorrect LCS for yes first, yes last, left"; std::vector> yes_yes_right = { { 0, 0 }, { 1, 1 }, { 3, 2 }, { 4, 3 } }; ASSERT_EQ(path_lcs(this->graph, path, yes_yes), yes_yes_right) << "Incorrect LCS for yes first, yes last, right"; } TEST_F(LCSTest, MinimalMaximalLCS) { gbwt::vector_type three_odd = forward_path({ 1, 3, 5 }); gbwt::vector_type four_odd = forward_path({ 1, 3, 5, 7 }); gbwt::vector_type three_even = forward_path({ 4, 6, 8 }); gbwt::vector_type four_even = forward_path({ 2, 4, 6, 8 }); ASSERT_TRUE(path_lcs(this->graph, four_odd, four_even).empty()) << "Nonempty LCS for non-overlapping paths"; std::vector> four_four { { 0, 0 }, { 1, 1 }, { 2, 2 }, { 3, 3 } }; ASSERT_EQ(path_lcs(this->graph, four_odd, four_odd), four_four) << "Incorrect LCS for identical paths"; std::vector> prefix { { 0, 0 }, { 1, 1 }, { 2, 2 } }; ASSERT_EQ(path_lcs(this->graph, three_odd, four_odd), prefix) << "Incorrect LCS for prefix / full"; ASSERT_EQ(path_lcs(this->graph, four_odd, three_odd), prefix) << "Incorrect LCS for full / prefix"; std::vector> left_suffix { { 0, 1 }, { 1, 2 }, { 2, 3 } }; ASSERT_EQ(path_lcs(this->graph, three_even, four_even), left_suffix) << "Incorrect LCS for suffix / full"; std::vector> right_suffix { { 1, 0 }, { 2, 1 }, { 3, 2 } }; ASSERT_EQ(path_lcs(this->graph, four_even, three_even), right_suffix) << "Incorrect LCS for full / suffix"; } TEST_F(LCSTest, RandomLCS) { for(size_t length : { 10, 30, 100, 300 }) { gbwt::vector_type left = this->random_path(length); gbwt::vector_type right = this->random_path(length); std::string name = "(length " + std::to_string(length) + ")"; this->compare_paths(left, right, name); } } TEST_F(LCSTest, RealLCS) { gbwt::size_type paths = this->index.metadata.paths(); for(gbwt::size_type left_id = 0; left_id < paths; left_id++) { for(gbwt::size_type right_id = 0; right_id < paths; right_id++) { for(bool orientation : { false, true }) { gbwt::vector_type left = this->index.extract(gbwt::Path::encode(left_id, orientation)); gbwt::vector_type right = this->index.extract(gbwt::Path::encode(right_id, orientation)); std::string name = "(paths " + std::to_string(left_id) + " and " + std::to_string(right_id) + " " + (orientation ? "reverse" : "forward") + ")"; this->compare_paths(left, right, name); } } } } TEST_F(LCSTest, LCSWeights) { // This is an artificial scenario where node lengths matter. // Nodes 3 and 6 are long nodes, while nodes 5, 7, and 8 are short nodes. gbwt::vector_type left = forward_path({ 2, 3, 6, 5, 7, 8 }); gbwt::vector_type right = forward_path({ 2, 5, 7, 8, 3, 6 }); std::vector> truth = { { 0, 0 }, { 1, 4 }, { 2, 5 } }; std::vector> result = path_lcs(this->graph, left, right); ASSERT_EQ(result, truth) << "Incorrect LCS weighted by node lengths"; } TEST_F(LCSTest, SimilarLCS) { std::vector> params = { { 100, 0.02 }, { 1000, 0.01 }, { 10000, 0.005 }, { 30000, 0.005 } }; for(std::pair test : params) { gbwt::vector_type left = this->random_path(test.first); gbwt::vector_type right = this->mutate_path(left, test.second); std::string name = "(length " + std::to_string(test.first) + ", mutation rate " + std::to_string(test.second) + ")"; this->compare_paths(left, right, name); } } //------------------------------------------------------------------------------ } // namespace jltsiren-gbwtgraph-e27bc43/tests/test_cached_gbwtgraph.cpp000066400000000000000000000047131524136466400241430ustar00rootroot00000000000000#include #include #include #include #include #include #include #include "shared.h" using namespace gbwtgraph; namespace { //------------------------------------------------------------------------------ class GraphOperations : public ::testing::Test { public: gbwt::GBWT index; NaiveGraph source; GBWTGraph graph; CachedGBWTGraph cached_graph; std::vector correct_nodes; std::set correct_edges; GraphOperations() { } void SetUp() override { this->index = build_gbwt_index(); this->source = build_naive_graph(false); this->graph = GBWTGraph(this->index, this->source); this->cached_graph = CachedGBWTGraph(this->graph); this->correct_nodes = { { nid_t(1), "G" }, { nid_t(2), "A" }, { nid_t(4), "GGG" }, { nid_t(5), "T" }, { nid_t(6), "A" }, { nid_t(7), "C" }, { nid_t(8), "A" }, { nid_t(9), "A" } }; this->correct_edges = { { gbwt::Node::encode(1, false), gbwt::Node::encode(2, false) }, { gbwt::Node::encode(1, false), gbwt::Node::encode(4, false) }, { gbwt::Node::encode(2, false), gbwt::Node::encode(4, false) }, { gbwt::Node::encode(4, false), gbwt::Node::encode(5, false) }, { gbwt::Node::encode(5, false), gbwt::Node::encode(6, false) }, { gbwt::Node::encode(6, false), gbwt::Node::encode(7, false) }, { gbwt::Node::encode(6, false), gbwt::Node::encode(8, false) }, { gbwt::Node::encode(7, false), gbwt::Node::encode(9, false) }, { gbwt::Node::encode(8, false), gbwt::Node::encode(9, false) } }; } }; TEST_F(GraphOperations, EmptyGraph) { gbwt::GBWT empty_index; NaiveGraph empty_source; GBWTGraph empty_graph(empty_index, empty_source); CachedGBWTGraph empty_cache(empty_graph); std::vector nodes; std::set edges; handle_graph_handles(empty_cache, nodes, true); handle_graph_nodes(empty_cache, nodes); handle_graph_edges(empty_cache, nodes, edges); } // HandleGraph interface for the default test graph. TEST_F(GraphOperations, HandleGraphTests) { handle_graph_handles(this->cached_graph, this->correct_nodes, true); handle_graph_nodes(this->cached_graph, this->correct_nodes); handle_graph_edges(this->cached_graph, this->correct_nodes, this->correct_edges); } //------------------------------------------------------------------------------ } // namespace jltsiren-gbwtgraph-e27bc43/tests/test_gbwtgraph.cpp000066400000000000000000001112611524136466400226510ustar00rootroot00000000000000#include #include #include #include #include #include #include #include #include "shared.h" using namespace gbwtgraph; namespace { //------------------------------------------------------------------------------ /* Test most GBWTGraph operations. SegmentHandleGraph / node-to-segment translation is GFA-specific functionality that is currently tested in test_gfa.cpp. */ class GraphOperations : public ::testing::Test { public: typedef std::pair gbwt_edge; gbwt::GBWT index; NaiveGraph source; GBWTGraph graph; std::vector correct_nodes; std::set correct_edges; std::set correct_paths; std::map correct_named_paths; std::map correct_reference_paths; std::map correct_haplotype_paths; std::map correct_sample_name; std::map correct_locus_name; std::map correct_haplotype_number; std::map correct_phase_block_number; std::map correct_subrange; void SetUp() override { this->index = build_gbwt_index_with_named_paths(); this->source = build_naive_graph(false); this->graph = GBWTGraph(this->index, this->source); this->correct_nodes = { { nid_t(1), "G" }, { nid_t(2), "A" }, { nid_t(4), "GGG" }, { nid_t(5), "T" }, { nid_t(6), "A" }, { nid_t(7), "C" }, { nid_t(8), "A" }, { nid_t(9), "A" } }; this->correct_edges = { { gbwt::Node::encode(1, false), gbwt::Node::encode(2, false) }, { gbwt::Node::encode(1, false), gbwt::Node::encode(4, false) }, { gbwt::Node::encode(2, false), gbwt::Node::encode(4, false) }, { gbwt::Node::encode(4, false), gbwt::Node::encode(5, false) }, { gbwt::Node::encode(5, false), gbwt::Node::encode(6, false) }, { gbwt::Node::encode(6, false), gbwt::Node::encode(7, false) }, { gbwt::Node::encode(6, false), gbwt::Node::encode(8, false) }, { gbwt::Node::encode(7, false), gbwt::Node::encode(9, false) }, { gbwt::Node::encode(8, false), gbwt::Node::encode(9, false) } }; this->correct_paths = { alt_path, short_path }; // Path order is short, alt, short, empty, empty as ref 1, ref 2, sample, empty 1, empty 2 this->correct_named_paths = {{"chr1", short_path}, {"chr2", alt_path}, {"empty1", empty_path}, {"GRCh38#empty2", empty_path}}; this->correct_reference_paths = {{"GRCh38#empty2", empty_path}}; this->correct_haplotype_paths = {{"Jouni Sirén#0#chr1#0", short_path}}; this->correct_sample_name = {{"chr1", handlegraph::PathMetadata::NO_SAMPLE_NAME}, {"chr2", handlegraph::PathMetadata::NO_SAMPLE_NAME}, {"empty1", handlegraph::PathMetadata::NO_SAMPLE_NAME}, {"GRCh38#empty2", "GRCh38"}, {"Jouni Sirén#0#chr1#0", "Jouni Sirén"}}; this->correct_locus_name = {{"chr1", "chr1"}, {"chr2", "chr2"}, {"empty1", "empty1"}, {"GRCh38#empty2", "empty2"}, {"Jouni Sirén#0#chr1#0", "chr1"}}; this->correct_haplotype_number = {{"chr1", handlegraph::PathMetadata::NO_HAPLOTYPE}, {"chr2", handlegraph::PathMetadata::NO_HAPLOTYPE}, {"empty1", handlegraph::PathMetadata::NO_HAPLOTYPE}, {"GRCh38#empty2", handlegraph::PathMetadata::NO_HAPLOTYPE}, {"Jouni Sirén#0#chr1#0", 0}}; this->correct_phase_block_number = {{"chr1", handlegraph::PathMetadata::NO_PHASE_BLOCK}, {"chr2", handlegraph::PathMetadata::NO_PHASE_BLOCK}, {"empty1", handlegraph::PathMetadata::NO_PHASE_BLOCK}, {"GRCh38#empty2", handlegraph::PathMetadata::NO_PHASE_BLOCK}, {"Jouni Sirén#0#chr1#0", 0}}; this->correct_subrange = {{"chr1", handlegraph::PathMetadata::NO_SUBRANGE}, {"chr2", handlegraph::PathMetadata::NO_SUBRANGE}, {"empty1", handlegraph::PathMetadata::NO_SUBRANGE}, {"GRCh38#empty2", handlegraph::PathMetadata::NO_SUBRANGE}, {"Jouni Sirén#0#chr1#0", handlegraph::PathMetadata::NO_SUBRANGE}}; } }; TEST_F(GraphOperations, EmptyGraph) { gbwt::GBWT empty_index; NaiveGraph empty_source; GBWTGraph empty_graph(empty_index, empty_source); EXPECT_EQ(empty_graph.get_node_count(), static_cast(0)) << "Empty graph contains nodes"; EXPECT_FALSE(empty_graph.has_segment_names()) << "Empty graph has segment names"; std::vector nodes; std::set edges; handle_graph_handles(empty_graph, nodes, true); handle_graph_nodes(empty_graph, nodes); handle_graph_edges(empty_graph, nodes, edges); } // HandleGraph interface for the default test graph. TEST_F(GraphOperations, HandleGraphTests) { handle_graph_handles(this->graph, this->correct_nodes, true); handle_graph_nodes(this->graph, this->correct_nodes); handle_graph_edges(this->graph, this->correct_nodes, this->correct_edges); } // Custom interface for the default test graph. TEST_F(GraphOperations, CustomInterface) { EXPECT_FALSE(this->graph.has_segment_names()) << "Graph has segment names"; // Check that handles are as expected. for(const node_type& node : this->correct_nodes) { handle_t fw_handle = this->graph.get_handle(node.first, false); handle_t rev_handle = this->graph.get_handle(node.first, true); gbwt::node_type fw_node = gbwt::Node::encode(node.first, false); gbwt::node_type rev_node = gbwt::Node::encode(node.first, true); handle_t fw_from_node = GBWTGraph::node_to_handle(fw_node); handle_t rev_from_node = GBWTGraph::node_to_handle(rev_node); gbwt::node_type fw_from_handle = GBWTGraph::handle_to_node(fw_handle); gbwt::node_type rev_from_handle = GBWTGraph::handle_to_node(rev_handle); EXPECT_EQ(as_integer(fw_handle), as_integer(fw_from_node)) << "Forward handle incorrect for node " << node.first; EXPECT_EQ(as_integer(rev_handle), as_integer(rev_from_node)) << "Reverse handle incorrect for node " << node.first; EXPECT_EQ(fw_node, fw_from_handle) << "Incorrect GBWT node id from forward handle for node " << node.first; EXPECT_EQ(rev_node, rev_from_handle) << "Incorrect GBWT node id from reverse handle for node " << node.first; } // Sequence views; starts/ends with. for(const node_type& node : this->correct_nodes) { handle_t fw_handle = this->graph.get_handle(node.first, false); EXPECT_EQ(this->graph.get_sequence_view(fw_handle), std::string_view(node.second)) << "Invalid sequence view for forward node " << node.first; EXPECT_TRUE(this->graph.starts_with(fw_handle, node.second.front())) << "Wrong first character for node " << node.first; EXPECT_FALSE(this->graph.starts_with(fw_handle, 'x')) << "First character is 'x' for node " << node.first; EXPECT_TRUE(this->graph.ends_with(fw_handle, node.second.back())) << "Wrong last character for node " << node.first; EXPECT_FALSE(this->graph.ends_with(fw_handle, 'x')) << "Last character is 'x' for node " << node.first; handle_t rev_handle = this->graph.get_handle(node.first, true); std::string reverse = node.second; reverse_complement_in_place(reverse); EXPECT_EQ(this->graph.get_sequence_view(rev_handle), std::string_view(reverse)) << "Invalid sequence view for reverse node " << node.first; EXPECT_TRUE(this->graph.starts_with(rev_handle, reverse.front())) << "Wrong first character for reverse node " << node.first; EXPECT_FALSE(this->graph.starts_with(rev_handle, 'x')) << "First character is 'x' for reverse node " << node.first; EXPECT_TRUE(this->graph.ends_with(rev_handle, reverse.back())) << "Wrong last character for reverse node " << node.first; EXPECT_FALSE(this->graph.ends_with(rev_handle, 'x')) << "Last character is 'x' for reverse node " << node.first; } } TEST_F(GraphOperations, FromHandleGraph) { GBWTGraph copy(this->index, this->graph, nullptr); EXPECT_EQ(copy.header, this->graph.header) << "Invalid header"; EXPECT_EQ(copy.sequences, this->graph.sequences) << "Invalid sequences"; EXPECT_EQ(copy.real_nodes, this->graph.real_nodes) << "Invalid real_nodes"; EXPECT_FALSE(copy.has_segment_names()) << "Got segment names from HandleGraph"; } TEST_F(GraphOperations, Paths) { ASSERT_EQ(this->graph.get_path_count(), this->correct_named_paths.size() + this->correct_haplotype_paths.size()) << "Wrong number of paths"; // We need to count the steps we see on each node. std::map steps_on_node; for(auto& path_set : {this->correct_named_paths, this->correct_haplotype_paths}) { for(auto& kv : path_set) { ASSERT_TRUE(this->graph.has_path(kv.first)) << "Path " << kv.first << " that should exist appears not to"; // Grab the path handlegraph::path_handle_t path_handle = this->graph.get_path_handle(kv.first); // Make sure we can recover the name ASSERT_EQ(this->graph.get_path_name(path_handle), kv.first) << "Path " << kv.first << " does not appear to have its own name"; // Trace along the truth path handlegraph::step_handle_t step_handle = this->graph.path_begin(path_handle); // Remember the previous step handlegraph::step_handle_t expected_previous; // And count our step index in the truth size_t index_in_path = 0; while (index_in_path < kv.second.size()) { // Make sure we are still on the correct path ASSERT_EQ(this->graph.get_path_handle_of_step(step_handle), path_handle) << "Step " << index_in_path << " of path " << kv.first << " does not appear to belong to the path"; // Check step node and orientation handlegraph::handle_t visited = this->graph.get_handle_of_step(step_handle); ASSERT_EQ(this->graph.get_id(visited), static_cast(gbwt::Node::id(kv.second[index_in_path]))) << "Step " << index_in_path << " of path " << kv.first << " visits the wrong node"; ASSERT_EQ(this->graph.get_is_reverse(visited), gbwt::Node::is_reverse(kv.second[index_in_path])) << "Step " << index_in_path << " of path " << kv.first << " visits the right node backward"; steps_on_node[this->graph.get_id(visited)]++; // Get the path steps on the node and make sure one of them is us. bool found_self = false; auto visit_step = [&](const handlegraph::step_handle_t other_step) -> bool { EXPECT_FALSE(found_self) << "Step iteration did not stop early"; // Node could be forward or backward in the path, but it must be this node. if (other_step == step_handle) { // Stop as soon as we see ourselves. found_self = true; return false; } return true; }; // Try forward this->graph.for_each_step_on_handle(visited, visit_step); EXPECT_TRUE(found_self) << "Step iteration on handle did not find current step"; // And try reverse found_self = false; this->graph.for_each_step_on_handle(this->graph.flip(visited), visit_step); EXPECT_TRUE(found_self) << "Step iteration on flipped handle did not find current step"; if (index_in_path == 0) { // This is the very first entry in the path EXPECT_FALSE(this->graph.has_previous_step(step_handle)) << "Initial step of path " << kv.first << " appears to have a predecessor"; // We can still look left and we expect to see path_front_end() handlegraph::step_handle_t observed_previous = this->graph.get_previous_step(step_handle); handlegraph::step_handle_t front_end_step_handle = this->graph.path_front_end(path_handle); ASSERT_EQ(observed_previous, front_end_step_handle) << "Running off start of path " << kv.first << " did not yield path_front_end for the path"; } else { // This isn't the first entry in the path, so we should have a previous entry. EXPECT_TRUE(this->graph.has_previous_step(step_handle)) << "Step " << index_in_path << " of path " << kv.first << " appears to be the first"; // Look left handlegraph::step_handle_t observed_previous = this->graph.get_previous_step(step_handle); ASSERT_EQ(observed_previous, expected_previous) << "Step " << index_in_path << " of path " << kv.first << " has wrong previous step"; } // Save the previous visited handle expected_previous = step_handle; if (index_in_path + 1 == kv.second.size()) { // This is the last entry in the path EXPECT_FALSE(this->graph.has_next_step(step_handle)) << "Final step " << index_in_path << " of path " << kv.first << " appears to have a successor"; // While here, we expect to match path_back() handlegraph::step_handle_t back_step_handle = this->graph.path_back(path_handle); ASSERT_EQ(step_handle, back_step_handle) << "Passing last step in path " << kv.first << " did not yield path_back for the path"; // We can still advance and we expect to see path_end() step_handle = this->graph.get_next_step(step_handle); handlegraph::step_handle_t end_step_handle = this->graph.path_end(path_handle); ASSERT_EQ(step_handle, end_step_handle) << "Running off end of path " << kv.first << " did not yield path_end for the path"; } else { // Advance along the path EXPECT_TRUE(this->graph.has_next_step(step_handle)) << "Step " << index_in_path << " of path " << kv.first << " appears to be the last"; step_handle = this->graph.get_next_step(step_handle); } // And advance along the true path index_in_path++; } EXPECT_EQ(this->graph.get_step_count(path_handle), kv.second.size()) << "Path " << kv.first << " appears to have the wrong number of steps"; } } this->graph.for_each_handle([&](const handlegraph::handle_t& here) { // Make sure the paths we traced agree with the per-node step counts. EXPECT_EQ(steps_on_node[this->graph.get_id(here)], this->graph.get_step_count(here)) << "Node " << this->graph.get_id(here) << " has the wrong number of steps on it"; }); } TEST_F(GraphOperations, NamedPaths) { EXPECT_FALSE(this->graph.has_path("SirNotAppearingInThisGraph")) << "Named path that shouldn't exist appears to exist"; } TEST_F(GraphOperations, PathMetadata) { // Make sure iteration of haplotype paths works. std::set haplotype_paths_seen; this->graph.for_each_path_of_sense(handlegraph::PathSense::HAPLOTYPE, [&](const handlegraph::path_handle_t& path_handle) { auto name = this->graph.get_path_name(path_handle); haplotype_paths_seen.insert(name); EXPECT_TRUE(this->correct_haplotype_paths.count(name)) << "Unexpected haplotype path " << name; }); EXPECT_EQ(haplotype_paths_seen.size(), this->correct_haplotype_paths.size()) << "Found wrong number of haplotype paths"; // Make sure iteration of reference paths works (even if we can't store any). std::set reference_paths_seen; this->graph.for_each_path_of_sense(handlegraph::PathSense::REFERENCE, [&](const handlegraph::path_handle_t& path_handle) { auto name = this->graph.get_path_name(path_handle); reference_paths_seen.insert(name); EXPECT_TRUE(this->correct_reference_paths.count(name)) << "Unexpected reference path " << name; }); EXPECT_EQ(reference_paths_seen.size(), this->correct_reference_paths.size()) << "Found wrong number of reference paths"; // Make sure iteration of generic paths works. std::set generic_paths_seen; this->graph.for_each_path_of_sense(handlegraph::PathSense::GENERIC, [&](const handlegraph::path_handle_t& path_handle) { auto name = this->graph.get_path_name(path_handle); generic_paths_seen.insert(name); EXPECT_TRUE(this->correct_named_paths.count(name)) << "Unexpected haplotype path " << name; }); EXPECT_EQ(generic_paths_seen.size(), this->correct_named_paths.size() - this->correct_reference_paths.size()) << "Found wrong number of generic paths"; for(auto& kv : this->correct_haplotype_paths) { ASSERT_TRUE(this->graph.has_path(kv.first)) << "Haplotype path " << kv.first << " that should exist appears not to"; // Grab the path handlegraph::path_handle_t path_handle = this->graph.get_path_handle(kv.first); // Make sure we can recover the name ASSERT_EQ(this->graph.get_path_name(path_handle), kv.first) << "Haplotype path " << kv.first << " does not appear to have its own name"; // Make sure we can get the right start and end. // We assume tracing works the same as for named paths. handlegraph::step_handle_t front_handle = this->graph.path_begin(path_handle); auto front_visit = this->graph.get_handle_of_step(front_handle); ASSERT_EQ(this->graph.get_id(front_visit), static_cast(gbwt::Node::id(kv.second.front()))) << "Path " << kv.first << " starts at the wrong node"; ASSERT_EQ(this->graph.get_is_reverse(front_visit), gbwt::Node::is_reverse(kv.second.front())) << "Path " << kv.first << " starts at the right node backward"; handlegraph::step_handle_t back_handle = this->graph.path_back(path_handle); auto back_visit = this->graph.get_handle_of_step(back_handle); ASSERT_EQ(this->graph.get_id(back_visit), static_cast(gbwt::Node::id(kv.second.back()))) << "Path " << kv.first << " ends at the wrong node"; ASSERT_EQ(this->graph.get_is_reverse(back_visit), gbwt::Node::is_reverse(kv.second.back())) << "Path " << kv.first << " ends at the right node backward"; // Make sure we can see steps bool found_front_step = false; this->graph.for_each_step_of_sense(front_visit, handlegraph::PathSense::HAPLOTYPE, [&](const handlegraph::step_handle_t& step) { if(step == front_handle) { found_front_step = true; return false; } return true; }); EXPECT_TRUE(found_front_step) << "Front step of " << kv.first << " not visible on node"; bool found_back_step = false; this->graph.for_each_step_of_sense(back_visit, handlegraph::PathSense::HAPLOTYPE, [&](const handlegraph::step_handle_t& step) { if(step == back_handle) { found_back_step = true; return false; } return true; }); EXPECT_TRUE(found_back_step) << "Back step of " << kv.first << " not visible on node"; // And check size. EXPECT_EQ(this->graph.get_step_count(path_handle), kv.second.size()) << "Path " << kv.first << " appears to have the wrong number of steps"; // Check sense EXPECT_EQ(this->graph.get_sense(path_handle), handlegraph::PathSense::HAPLOTYPE) << "Haplotype has wrong sense"; } for(auto& kv : this->correct_reference_paths) { handlegraph::path_handle_t path_handle = this->graph.get_path_handle(kv.first); // Check sense of reference paths EXPECT_EQ(this->graph.get_sense(path_handle), handlegraph::PathSense::REFERENCE) << "Named path has wrong sense"; } for(auto& kv : this->correct_named_paths) { handlegraph::path_handle_t path_handle = this->graph.get_path_handle(kv.first); // Check sense of named paths auto true_sense = this->correct_reference_paths.count(kv.first) ? handlegraph::PathSense::REFERENCE : handlegraph::PathSense::GENERIC; EXPECT_EQ(this->graph.get_sense(path_handle), true_sense) << "Named path has wrong sense"; } // Now check metadata across path types. for(auto& kv : this->correct_sample_name) { handlegraph::path_handle_t path_handle = this->graph.get_path_handle(kv.first); EXPECT_EQ(this->graph.get_sample_name(path_handle), kv.second) << "Path " << kv.first << " has wrong sample name"; } for(auto& kv : this->correct_locus_name) { handlegraph::path_handle_t path_handle = this->graph.get_path_handle(kv.first); EXPECT_EQ(this->graph.get_locus_name(path_handle), kv.second) << "Path " << kv.first << " has wrong sample name"; } for(auto& kv : this->correct_haplotype_number) { handlegraph::path_handle_t path_handle = this->graph.get_path_handle(kv.first); EXPECT_EQ(this->graph.get_haplotype(path_handle), kv.second) << "Path " << kv.first << " has wrong haplotype number"; } for(auto& kv : this->correct_phase_block_number) { handlegraph::path_handle_t path_handle = this->graph.get_path_handle(kv.first); EXPECT_EQ(this->graph.get_phase_block(path_handle), kv.second) << "Path " << kv.first << " has wrong phase block number"; } for(auto& kv : this->correct_subrange) { handlegraph::path_handle_t path_handle = this->graph.get_path_handle(kv.first); EXPECT_EQ(this->graph.get_subrange(path_handle), kv.second) << "Path " << kv.first << " has wrong subrange"; } } TEST_F(GraphOperations, ForwardTraversal) { typedef std::pair state_type; std::vector found_paths; std::stack states; // Extract all paths starting from node 1 in forward orientation. handle_t first_node = this->graph.get_handle(1, false); states.push({ this->graph.get_state(first_node), { static_cast(GBWTGraph::handle_to_node(first_node)) } }); while(!states.empty()) { state_type curr = states.top(); states.pop(); bool extend_success = false; this->graph.follow_paths(curr.first, [&](const gbwt::SearchState& next_search) -> bool { if(!next_search.empty()) { extend_success = true; gbwt::vector_type next_path = curr.second; next_path.push_back(next_search.node); states.push({ next_search, next_path }); } return true; }); if(!extend_success) { found_paths.push_back(curr.second); } } ASSERT_EQ(found_paths.size(), correct_paths.size()) << "Found a wrong number of paths"; for(size_t i = 0; i < found_paths.size(); i++) { const gbwt::vector_type& path = found_paths[i]; EXPECT_TRUE(correct_paths.find(path) != correct_paths.end()) << "Path " << i << " is incorrect"; } } TEST_F(GraphOperations, BidirectionalTraversal) { typedef std::pair state_type; std::vector found_paths; std::stack fw_states, rev_states; // Extract all paths visiting node 4 in forward orientation. handle_t first_node = this->graph.get_handle(4, false); fw_states.push({ this->graph.get_bd_state(first_node), { static_cast(GBWTGraph::handle_to_node(first_node)) } }); while(!fw_states.empty()) { state_type curr = fw_states.top(); fw_states.pop(); bool extend_success = false; this->graph.follow_paths(curr.first, false, [&](const gbwt::BidirectionalState& next_search) -> bool { if(!next_search.empty()) { extend_success = true; gbwt::vector_type next_path = curr.second; next_path.push_back(next_search.forward.node); fw_states.push({ next_search, next_path }); } return true; }); if (!extend_success) { rev_states.push(curr); } } while (!rev_states.empty()) { state_type curr = rev_states.top(); rev_states.pop(); bool extend_success = false; this->graph.follow_paths(curr.first, true, [&](const gbwt::BidirectionalState& next_search) -> bool { if(!next_search.empty()) { extend_success = true; gbwt::vector_type next_path { static_cast(gbwt::Node::reverse(next_search.backward.node)) }; next_path.insert(next_path.end(), curr.second.begin(), curr.second.end()); rev_states.push({ next_search, next_path }); } return true; }); if (!extend_success) { found_paths.push_back(curr.second); } } ASSERT_EQ(found_paths.size(), correct_paths.size()) << "Found a wrong number of paths"; for(size_t i = 0; i < found_paths.size(); i++) { const gbwt::vector_type& path = found_paths[i]; EXPECT_TRUE(correct_paths.find(path) != correct_paths.end()) << "Path " << i << " is incorrect"; } } //------------------------------------------------------------------------------ class GraphSerialization : public ::testing::Test { public: gbwt::GBWT index; NaiveGraph source; GBWTGraph graph; GraphSerialization() { } void SetUp() override { this->index = build_gbwt_index(); this->source = build_naive_graph(false); this->graph = GBWTGraph(this->index, this->source); } }; TEST_F(GraphSerialization, SerializeEmpty) { GBWTGraph empty_graph; std::string filename = gbwt::TempFile::getName("gbwtgraph"); empty_graph.serialize(filename); GBWTGraph duplicate_graph; duplicate_graph.deserialize(filename); compare_graphs(duplicate_graph, empty_graph, DEFAULT_FLAGS, ""); gbwt::TempFile::remove(filename); } TEST_F(GraphSerialization, CompressEmpty) { gbwt::GBWT empty_gbwt; GBWTGraph empty_graph; empty_graph.set_gbwt(empty_gbwt); size_t expected_size = empty_graph.simple_sds_size() * sizeof(sdsl::simple_sds::element_type); std::string filename = gbwt::TempFile::getName("gbwtgraph"); sdsl::simple_sds::serialize_to(empty_graph, filename); GBWTGraph duplicate_graph; std::ifstream in(filename, std::ios_base::binary); size_t bytes = gbwt::fileSize(in); ASSERT_EQ(bytes, expected_size) << "Invalid file size"; duplicate_graph.simple_sds_load(in, *(empty_graph.index)); in.close(); compare_graphs(duplicate_graph, empty_graph, DEFAULT_FLAGS, ""); gbwt::TempFile::remove(filename); } TEST_F(GraphSerialization, CompressEmptyVersion) { gbwt::GBWT empty_gbwt; GBWTGraph empty_graph; empty_graph.set_gbwt(empty_gbwt); for(std::uint32_t version = GBWTGraph::Header::MIN_SERIALIZE_VERSION; version <= GBWTGraph::Header::VERSION; version++) { std::string filename = gbwt::TempFile::getName("gbwtgraph"); sdsl::simple_sds::serialize_to(empty_graph, filename, version); GBWTGraph duplicate_graph; std::ifstream in(filename, std::ios_base::binary); duplicate_graph.simple_sds_load(in, *(empty_graph.index)); in.close(); compare_graphs(duplicate_graph, empty_graph, DEFAULT_FLAGS, "version " + std::to_string(version)); gbwt::TempFile::remove(filename); } } TEST_F(GraphSerialization, SerializeNonEmpty) { std::string filename = gbwt::TempFile::getName("gbwtgraph"); this->graph.serialize(filename); GBWTGraph duplicate_graph; duplicate_graph.deserialize(filename); duplicate_graph.set_gbwt(this->index); compare_graphs(duplicate_graph, this->graph, DEFAULT_FLAGS, ""); gbwt::TempFile::remove(filename); } TEST_F(GraphSerialization, CompressNonEmpty) { size_t expected_size = this->graph.simple_sds_size() * sizeof(sdsl::simple_sds::element_type); std::string filename = gbwt::TempFile::getName("gbwtgraph"); sdsl::simple_sds::serialize_to(this->graph, filename); GBWTGraph duplicate_graph; std::ifstream in(filename, std::ios_base::binary); size_t bytes = gbwt::fileSize(in); ASSERT_EQ(bytes, expected_size) << "Invalid file size"; duplicate_graph.simple_sds_load(in, this->index); in.close(); compare_graphs(duplicate_graph, this->graph, DEFAULT_FLAGS, ""); gbwt::TempFile::remove(filename); } TEST_F(GraphSerialization, CompressNonEmptyVersion) { for(std::uint32_t version = GBWTGraph::Header::MIN_SERIALIZE_VERSION; version <= GBWTGraph::Header::VERSION; version++) { std::string filename = gbwt::TempFile::getName("gbwtgraph"); sdsl::simple_sds::serialize_to(this->graph, filename, version); GBWTGraph duplicate_graph; std::ifstream in(filename, std::ios_base::binary); duplicate_graph.simple_sds_load(in, this->index); in.close(); compare_graphs(duplicate_graph, this->graph, DEFAULT_FLAGS, "version " + std::to_string(version)); gbwt::TempFile::remove(filename); } } TEST_F(GraphSerialization, SerializeTranslation) { NaiveGraph source = build_naive_graph(true); GBWTGraph graph(this->index, source); std::string filename = gbwt::TempFile::getName("gbwtgraph"); graph.serialize(filename); GBWTGraph duplicate_graph; duplicate_graph.deserialize(filename); duplicate_graph.set_gbwt(this->index); compare_graphs(duplicate_graph, graph, DEFAULT_FLAGS, ""); gbwt::TempFile::remove(filename); } TEST_F(GraphSerialization, CompressTranslation) { NaiveGraph source = build_naive_graph(true); GBWTGraph graph(this->index, source); size_t expected_size = graph.simple_sds_size() * sizeof(sdsl::simple_sds::element_type); std::string filename = gbwt::TempFile::getName("gbwtgraph"); sdsl::simple_sds::serialize_to(graph, filename); GBWTGraph duplicate_graph; std::ifstream in(filename, std::ios_base::binary); size_t bytes = gbwt::fileSize(in); ASSERT_EQ(bytes, expected_size) << "Invalid file size"; duplicate_graph.simple_sds_load(in, this->index); in.close(); compare_graphs(duplicate_graph, graph, DEFAULT_FLAGS, ""); gbwt::TempFile::remove(filename); } TEST_F(GraphSerialization, DecompressSerialized) { NaiveGraph source = build_naive_graph(true); GBWTGraph graph(this->index, source); std::string filename = gbwt::TempFile::getName("gbwtgraph"); graph.serialize(filename); GBWTGraph duplicate_graph; std::ifstream in(filename, std::ios_base::binary); std::uint32_t magic = 0; in.read(reinterpret_cast(&magic), sizeof(std::uint32_t)); EXPECT_EQ(ntohl(magic), graph.get_magic_number()) << "Magic number missing from serialized graph"; duplicate_graph.simple_sds_load(in, this->index); in.close(); compare_graphs(duplicate_graph, graph, DEFAULT_FLAGS, ""); gbwt::TempFile::remove(filename); } //------------------------------------------------------------------------------ class ForEachWindow : public ::testing::Test { public: typedef std::pair, std::string> kmer_type; gbwt::GBWT index; NaiveGraph source; GBWTGraph graph; std::set correct_kmers; std::set correct_nonredundant; ForEachWindow() { } void SetUp() override { this->index = build_gbwt_index(); this->source = build_naive_graph(false); this->graph = GBWTGraph(this->index, this->source); this->correct_kmers = { // alt_path forward { { this->graph.get_handle(1, false), this->graph.get_handle(2, false), this->graph.get_handle(4, false) }, "GAG" }, { { this->graph.get_handle(2, false), this->graph.get_handle(4, false) }, "AGG" }, { { this->graph.get_handle(4, false), this->graph.get_handle(5, false), this->graph.get_handle(6, false) }, "GGGTA" }, { { this->graph.get_handle(5, false), this->graph.get_handle(6, false), this->graph.get_handle(8, false) }, "TAA" }, { { this->graph.get_handle(6, false), this->graph.get_handle(8, false), this->graph.get_handle(9, false) }, "AAA" }, // alt_path reverse { { this->graph.get_handle(9, true), this->graph.get_handle(8, true), this->graph.get_handle(6, true) }, "TTT" }, { { this->graph.get_handle(8, true), this->graph.get_handle(6, true), this->graph.get_handle(5, true) }, "TTA" }, { { this->graph.get_handle(6, true), this->graph.get_handle(5, true), this->graph.get_handle(4, true) }, "TAC" }, { { this->graph.get_handle(5, true), this->graph.get_handle(4, true) }, "ACC" }, { { this->graph.get_handle(4, true), this->graph.get_handle(2, true), this->graph.get_handle(1, true) }, "CCCTC" }, // short_path forward { { this->graph.get_handle(1, false), this->graph.get_handle(4, false) }, "GGG" }, { { this->graph.get_handle(4, false), this->graph.get_handle(5, false), this->graph.get_handle(6, false) }, "GGGTA" }, { { this->graph.get_handle(5, false), this->graph.get_handle(6, false), this->graph.get_handle(7, false) }, "TAC" }, { { this->graph.get_handle(6, false), this->graph.get_handle(7, false), this->graph.get_handle(9, false) }, "ACA" }, // short_path reverse { { this->graph.get_handle(9, true), this->graph.get_handle(7, true), this->graph.get_handle(6, true) }, "TGT" }, { { this->graph.get_handle(7, true), this->graph.get_handle(6, true), this->graph.get_handle(5, true) }, "GTA" }, { { this->graph.get_handle(6, true), this->graph.get_handle(5, true), this->graph.get_handle(4, true) }, "TAC" }, { { this->graph.get_handle(5, true), this->graph.get_handle(4, true) }, "ACC" }, { { this->graph.get_handle(4, true), this->graph.get_handle(1, true) }, "CCCC" } }; this->correct_nonredundant = { // alt_path forward { { this->graph.get_handle(1, false), this->graph.get_handle(2, false), this->graph.get_handle(4, false) }, "GAG" }, { { this->graph.get_handle(2, false), this->graph.get_handle(4, false) }, "AGG" }, { { this->graph.get_handle(4, false) }, "GGG" }, { { this->graph.get_handle(4, false), this->graph.get_handle(5, false), this->graph.get_handle(6, false) }, "GGTA" }, { { this->graph.get_handle(5, false), this->graph.get_handle(6, false), this->graph.get_handle(8, false) }, "TAA" }, { { this->graph.get_handle(6, false), this->graph.get_handle(8, false), this->graph.get_handle(9, false) }, "AAA" }, // alt_path reverse { { this->graph.get_handle(9, true), this->graph.get_handle(8, true), this->graph.get_handle(6, true) }, "TTT" }, { { this->graph.get_handle(8, true), this->graph.get_handle(6, true), this->graph.get_handle(5, true) }, "TTA" }, { { this->graph.get_handle(6, true), this->graph.get_handle(5, true), this->graph.get_handle(4, true) }, "TAC" }, { { this->graph.get_handle(5, true), this->graph.get_handle(4, true) }, "ACC" }, { { this->graph.get_handle(4, true) }, "CCC" }, { { this->graph.get_handle(4, true), this->graph.get_handle(2, true), this->graph.get_handle(1, true) }, "CCTC" }, // short_path forward { { this->graph.get_handle(1, false), this->graph.get_handle(4, false) }, "GGG" }, { { this->graph.get_handle(4, false) }, "GGG" }, { { this->graph.get_handle(4, false), this->graph.get_handle(5, false), this->graph.get_handle(6, false) }, "GGTA" }, { { this->graph.get_handle(5, false), this->graph.get_handle(6, false), this->graph.get_handle(7, false) }, "TAC" }, { { this->graph.get_handle(6, false), this->graph.get_handle(7, false), this->graph.get_handle(9, false) }, "ACA" }, // short_path reverse { { this->graph.get_handle(9, true), this->graph.get_handle(7, true), this->graph.get_handle(6, true) }, "TGT" }, { { this->graph.get_handle(7, true), this->graph.get_handle(6, true), this->graph.get_handle(5, true) }, "GTA" }, { { this->graph.get_handle(6, true), this->graph.get_handle(5, true), this->graph.get_handle(4, true) }, "TAC" }, { { this->graph.get_handle(5, true), this->graph.get_handle(4, true) }, "ACC" }, { { this->graph.get_handle(4, true) }, "CCC" }, { { this->graph.get_handle(4, true), this->graph.get_handle(1, true) }, "CCC" } }; } }; TEST_F(ForEachWindow, KmerExtraction) { // Extract all haplotype-consistent windows of length 3. std::set found_kmers; for_each_haplotype_window(this->graph, 3, [&found_kmers](const std::vector& traversal, const std::string& seq) { found_kmers.insert(kmer_type(traversal, seq)); }, false); ASSERT_EQ(found_kmers.size(), this->correct_kmers.size()) << "Found a wrong number of kmers"; for(const kmer_type& kmer : found_kmers) { EXPECT_TRUE(correct_kmers.find(kmer) != correct_kmers.end()) << "Kmer " << kmer.second << " is incorrect"; } } TEST_F(ForEachWindow, NonRedundantKmers) { // Extract all haplotype-consistent non-redundant windows of length 3. std::set found_kmers; for_each_nonredundant_window(this->graph, 3, [&found_kmers](const std::vector& traversal, const std::string& seq) { found_kmers.insert(kmer_type(traversal, seq)); }, false); ASSERT_EQ(found_kmers.size(), this->correct_nonredundant.size()) << "Found a wrong number of kmers"; for(const kmer_type& kmer : found_kmers) { EXPECT_TRUE(correct_nonredundant.find(kmer) != correct_kmers.end()) << "Kmer " << kmer.second << " is incorrect"; } } //------------------------------------------------------------------------------ } // namespace jltsiren-gbwtgraph-e27bc43/tests/test_gbz.cpp000066400000000000000000000233651524136466400214550ustar00rootroot00000000000000#include #include #include #include "shared.h" using namespace gbwtgraph; namespace { // Subgraph constructor is tested in test_gfa.cpp. // Chunking and merging GBZs is tested in test_algorithms.cpp. //------------------------------------------------------------------------------ class GBZSerialization : public ::testing::Test { public: std::unique_ptr create_gbz() { NaiveGraph source = build_naive_graph(false); return std::make_unique(build_gbwt_index(), source); } }; TEST_F(GBZSerialization, Empty) { GBZ empty; size_t expected_size = empty.simple_sds_size() * sizeof(sdsl::simple_sds::element_type); std::string filename = gbwt::TempFile::getName("gbz"); sdsl::simple_sds::serialize_to(empty, filename); GBZ duplicate; std::ifstream in(filename, std::ios_base::binary); size_t bytes = gbwt::fileSize(in); ASSERT_EQ(bytes, expected_size) << "Invalid file size"; duplicate.simple_sds_load(in); in.close(); compare_gbzs(duplicate, empty, DEFAULT_FLAGS, ""); gbwt::TempFile::remove(filename); } TEST_F(GBZSerialization, EmptyVersion) { GBZ empty; for(std::uint32_t version = GBZ::Header::MIN_SERIALIZE_VERSION; version <= GBZ::Header::VERSION; version++) { std::string filename = gbwt::TempFile::getName("gbz"); sdsl::simple_sds::serialize_to(empty, filename, version); GBZ duplicate; std::ifstream in(filename, std::ios_base::binary); duplicate.simple_sds_load(in); in.close(); compare_gbzs(duplicate, empty, DEFAULT_FLAGS, "version " + std::to_string(version)); gbwt::TempFile::remove(filename); } } TEST_F(GBZSerialization, NonEmpty) { std::unique_ptr original = this->create_gbz(); size_t expected_size = original->simple_sds_size() * sizeof(sdsl::simple_sds::element_type); std::string filename = gbwt::TempFile::getName("gbz"); sdsl::simple_sds::serialize_to(*original, filename); GBZ duplicate; std::ifstream in(filename, std::ios_base::binary); size_t bytes = gbwt::fileSize(in); ASSERT_EQ(bytes, expected_size) << "Invalid file size"; duplicate.simple_sds_load(in); in.close(); compare_gbzs(duplicate, *original, DEFAULT_FLAGS, ""); gbwt::TempFile::remove(filename); } TEST_F(GBZSerialization, NonEmptyVersion) { std::unique_ptr original = this->create_gbz(); for(std::uint32_t version = GBZ::Header::MIN_SERIALIZE_VERSION; version <= GBZ::Header::VERSION; version++) { std::string filename = gbwt::TempFile::getName("gbz"); sdsl::simple_sds::serialize_to(*original, filename, version); GBZ duplicate; std::ifstream in(filename, std::ios_base::binary); duplicate.simple_sds_load(in); in.close(); compare_gbzs(duplicate, *original, DEFAULT_FLAGS, "version " + std::to_string(version)); gbwt::TempFile::remove(filename); } } TEST_F(GBZSerialization, ExternalObjects) { // Serialization into separate GBWT / GBWTGraph files does not preserve tags. // We therefore do not check file sizes. std::unique_ptr original = this->create_gbz(); std::string filename = gbwt::TempFile::getName("gbz"); std::ofstream out(filename, std::ios_base::binary); GBZ::simple_sds_serialize(original->index, original->graph, out); out.close(); GBZ duplicate; std::ifstream in(filename, std::ios_base::binary); duplicate.simple_sds_load(in); in.close(); std::uint64_t flags = (DEFAULT_FLAGS & ~FLAG_GBZ_TAGS); compare_gbzs(duplicate, *original, flags, ""); gbwt::TempFile::remove(filename); } TEST_F(GBZSerialization, CopyAndSerialize) { std::string filename = gbwt::TempFile::getName("gbz"); { std::unique_ptr original = this->create_gbz(); GBZ copied = *original; original.reset(); sdsl::simple_sds::serialize_to(copied, filename); } { std::unique_ptr truth = this->create_gbz(); GBZ loaded; sdsl::simple_sds::load_from(loaded, filename); compare_gbzs(loaded, *truth, DEFAULT_FLAGS, ""); } gbwt::TempFile::remove(filename); } TEST_F(GBZSerialization, MoveAndSerialize) { std::string filename = gbwt::TempFile::getName("gbz"); { std::unique_ptr original = this->create_gbz(); GBZ moved = std::move(*original); original.reset(); sdsl::simple_sds::serialize_to(moved, filename); } { std::unique_ptr truth = this->create_gbz(); GBZ loaded; sdsl::simple_sds::load_from(loaded, filename); compare_gbzs(loaded, *truth, DEFAULT_FLAGS, ""); } gbwt::TempFile::remove(filename); } TEST_F(GBZSerialization, SwapAndSerialize) { std::string filename = gbwt::TempFile::getName("gbz"); { std::unique_ptr original = this->create_gbz(); GBZ swapped; swapped.swap(*original); original.reset(); sdsl::simple_sds::serialize_to(swapped, filename); } { std::unique_ptr truth = this->create_gbz(); GBZ loaded; sdsl::simple_sds::load_from(loaded, filename); compare_gbzs(loaded, *truth, DEFAULT_FLAGS, ""); } gbwt::TempFile::remove(filename); } TEST_F(GBZSerialization, LoadTags) { std::unique_ptr original = this->create_gbz(); std::string filename = gbwt::TempFile::getName("gbz"); sdsl::simple_sds::serialize_to(*original, filename); gbwt::Tags loaded_tags = GBZ::simple_sds_load_tags(filename); ASSERT_EQ(loaded_tags, original->tags) << "Invalid tags loaded from file"; gbwt::TempFile::remove(filename); } //------------------------------------------------------------------------------ class GBZFunctionality : public ::testing::Test { public: GBZ build_gbz(const std::string& graph_name) { auto parse = gfa_to_gbwt(graph_name); return GBZ(parse.first, parse.second); } void set_reference_samples(GBZ& gbz, const sample_name_set& samples, size_t expected, const std::string& test) { size_t present = gbz.set_reference_samples(samples); ASSERT_EQ(present, expected) << test << ": Unexpected number of sample names present in the graph"; } void check_named_paths(const GBZ& gbz, const sample_name_set& true_samples, size_t expected_paths, const std::string& test) { ASSERT_EQ(gbz.named_paths(), expected_paths) << test << ": Invalid number of named paths"; const sample_name_set& samples = gbz.get_reference_samples(); ASSERT_EQ(samples.size(), true_samples.size()) << test << ": Invalid number of reference samples"; for(const std::string& sample : true_samples) { ASSERT_TRUE(samples.find(sample) != samples.end()) << test << ": Missing reference sample " << sample; } } // The test GFA files have their graph names computed with the reference pggname implementation. // Tests in test_gfa.cpp ensure that our implementation computes the same names. void check_graph_name(GBZ& gbz, const GraphName* parent, bool missing_name, const std::string& test_case) { EXPECT_EQ(gbz.pggname().empty(), missing_name) << "Unexpected graph name presence for " << test_case; EXPECT_EQ(gbz.translation_target(), "") << "Translation target should not be set for " << test_case; GraphName old = gbz.graph_name(); EXPECT_EQ(gbz.pggname(), old.name()) << "Graph name mismatch for " << test_case; gbz.compute_pggname(parent); GraphName recomputed = gbz.graph_name(); EXPECT_TRUE(recomputed.has_name()) << "Recomputed graph name missing for " << test_case; if(!missing_name) { EXPECT_EQ(recomputed, old) << "Graph name changed after recomputation for " << test_case; } } }; TEST_F(GBZFunctionality, ReferenceSamples) { GBZ gbz = this->build_gbz("gfas/components_ref.gfa"); sample_name_set samples { "ref" }; sample_name_set true_samples = samples; this->check_named_paths(gbz, true_samples, 2, "Initial graph"); samples.erase("ref"); samples.insert("sample"); true_samples = samples; this->set_reference_samples(gbz, samples, 1, "New sample"); this->check_named_paths(gbz, true_samples, 4, "New sample"); samples.insert("ref"); true_samples = samples; this->set_reference_samples(gbz, samples, 2, "Both samples"); this->check_named_paths(gbz, true_samples, 6, "Both samples"); samples.erase("sample"); true_samples = samples; samples.insert("missing"); this->set_reference_samples(gbz, samples, 1, "Invalid sample"); this->check_named_paths(gbz, true_samples, 2, "Invalid sample"); } TEST_F(GBZFunctionality, GraphNames) { // Additional tests with parent graph relationships are in test_gfa.cpp. // Constructor from gfa_to_gbwt() output. { std::unique_ptr index = std::make_unique(build_gbwt_index()); std::unique_ptr graph = std::make_unique(std::move(build_naive_graph(false))); GBZ gbz(index, graph); this->check_graph_name(gbz, nullptr, false, "gfa_to_gbwt() output"); } // Constructor from GBWT and NaiveGraph. { gbwt::GBWT index = build_gbwt_index(); NaiveGraph graph = build_naive_graph(false); GBZ gbz(index, graph); this->check_graph_name(gbz, nullptr, false, "GBWT and NaiveGraph"); } // Subgraph construction. { gbwt::GBWT index = build_gbwt_index(); NaiveGraph graph = build_naive_graph(false); GBZ supergraph(index, graph); GraphName parent = supergraph.graph_name(); std::vector paths { alt_path }; gbwt::GBWT sub_index = build_gbwt(paths); GBZ gbz(std::move(sub_index), supergraph); this->check_graph_name(gbz, &parent, false, "subgraph construction"); } // Constructor from GBWT and HandleGraph; graph name must be set manually. { gbwt::GBWT parent_index = build_gbwt_index(); NaiveGraph graph = build_naive_graph(false); GBWTGraph parent_graph(parent_index, graph); gbwt::GBWT index = parent_index; GBZ gbz(std::move(index), parent_graph, nullptr); this->check_graph_name(gbz, nullptr, true, "GBWT and HandleGraph"); } } //------------------------------------------------------------------------------ } // namespace jltsiren-gbwtgraph-e27bc43/tests/test_gfa.cpp000066400000000000000000001275771524136466400214420ustar00rootroot00000000000000#include #include #include #include #include #include #include "shared.h" using namespace gbwtgraph; namespace { //------------------------------------------------------------------------------ // We do not check GBWT metadata here, as we have separate tests (GBWTMetadata) for that. // We also do not check node-to-segment translation in compare_graphs(), as we have a separate check for that. class GFAConstruction : public ::testing::Test { public: gbwt::GBWT index; NaiveGraph source; GBWTGraph graph; GFAConstruction() { } void SetUp() override { this->index = build_gbwt_index(); this->source = build_naive_graph(false); this->graph = GBWTGraph(this->index, this->source); } void check_paths(const GBWTGraph& gfa_graph, const GBWTGraph* truth) const { std::unordered_map truth_paths; truth->for_each_path_matching(nullptr, nullptr, nullptr, [&](const path_handle_t truth_path) { truth_paths[truth->get_path_name(truth_path)] = truth->get_sense(truth_path); }); std::unordered_map observed_paths; gfa_graph.for_each_path_matching(nullptr, nullptr, nullptr, [&](const path_handle_t gfa_path) { observed_paths[gfa_graph.get_path_name(gfa_path)] = gfa_graph.get_sense(gfa_path); }); for (auto& kv : observed_paths) { EXPECT_TRUE(truth_paths.count(kv.first)) << "GFA has extraneous path " << kv.first; } for (auto& kv : truth_paths) { ASSERT_TRUE(observed_paths.count(kv.first)) << "GFA is missing truth path " << kv.first; EXPECT_EQ(observed_paths[kv.first], kv.second) << "GFA has wrong sense for path " << kv.first; } } void check_translation(const NaiveGraph& source, const std::vector& truth) const { ASSERT_TRUE(source.uses_translation()) << "Segment translation not in use"; ASSERT_GE(source.get_segment_count(), truth.size()) << "Too few segments in the translation"; for(const translation_type& translation : truth) { EXPECT_EQ(source.translate(translation.first), translation.second) << "Invalid translation for " << translation.first; } } void check_no_translation(const GBWTGraph& graph) const { ASSERT_FALSE(graph.has_segment_names()) << "The graph has segment names"; graph.for_each_handle([&](const handle_t& handle) { nid_t id = graph.get_id(handle); std::pair correct(std::to_string(id), 0); EXPECT_EQ(graph.get_segment_name_and_offset(handle), correct) << "Invalid null translation for node " << id; EXPECT_EQ(graph.get_segment_name(handle), correct.first) << "Invalid null segment name for node " << id; EXPECT_EQ(graph.get_segment_offset(handle), correct.second) << "Invalid null segment offset for node " << id; handle_t reverse = graph.flip(handle); EXPECT_EQ(graph.get_segment_name_and_offset(reverse), correct) << "Invalid null translation for reverse node " << id; EXPECT_EQ(graph.get_segment_name(reverse), correct.first) << "Invalid null segment name for reverse node " << id; EXPECT_EQ(graph.get_segment_offset(reverse), correct.second) << "Invalid null segment offset for reverse node " << id; auto segment = graph.get_segment(handle); std::pair range(id, id + 1); EXPECT_EQ(segment.first, correct.first) << "Invalid null segment containing node " << id; EXPECT_EQ(segment.second, range) << "Invalid node range for null segment containing node " << id; }); } void check_translation(const GBWTGraph& graph, const std::vector& truth) const { ASSERT_TRUE(graph.has_segment_names()) << "The graph has no segment names"; // Forward orientation. for(const translation_type& translation : truth) { size_t offset = 0; for(nid_t id = translation.second.first; id < translation.second.second; id++) { handle_t handle = graph.get_handle(id, false); size_t node_length = graph.get_length(handle); ASSERT_TRUE(node_length > 0) << "Node appears empty"; // Check the SegmentHandleGraph API std::pair correct(translation.first, offset); EXPECT_EQ(graph.get_segment_name_and_offset(handle), correct) << "Invalid translation for node " << id; EXPECT_EQ(graph.get_segment_name(handle), correct.first) << "Invalid segment name for node " << id; EXPECT_EQ(graph.get_segment_offset(handle), correct.second) << "Invalid segment offset for node " << id; // Check the NamedNodeBackTranslation API for (size_t start = 0; start + 1 < node_length; start++) { for (size_t length = 1; start + length <= node_length; length++) { // For each possible range on this node, make a range. oriented_node_range_t graph_range {id, false, start, length}; // And translate it back to GFA coordinates std::vector back_translated = graph.translate_back(graph_range); EXPECT_EQ(back_translated.size(), 1ul) << "GFA segments appear merged in graph"; EXPECT_EQ(graph.get_back_graph_node_name(std::get<0>(back_translated[0])), translation.first) << "Graph node belongs to wrong GFA segment"; ASSERT_FALSE(std::get<1>(back_translated[0])) << "Graph node is backward in GFA segment"; EXPECT_EQ(std::get<2>(back_translated[0]), offset + start) << "Graph node range begins at wrong offset on GFA segment"; EXPECT_EQ(std::get<3>(back_translated[0]), length) << "Graph node range translated to GFA range of wrong length"; } } offset += node_length; } } // Reverse orientation. for(const translation_type& translation : truth) { size_t offset = 0; for(nid_t i = translation.second.second; i > translation.second.first; i--) { nid_t id = i - 1; handle_t handle = graph.get_handle(id, true); size_t node_length = graph.get_length(handle); ASSERT_TRUE(node_length > 0) << "Node appears empty"; // Check the SegmentHandleGraph API std::pair correct(translation.first, offset); EXPECT_EQ(graph.get_segment_name_and_offset(handle), correct) << "Invalid translation for reverse node " << id; EXPECT_EQ(graph.get_segment_name(handle), correct.first) << "Invalid segment name for reverse node " << id; EXPECT_EQ(graph.get_segment_offset(handle), correct.second) << "Invalid segment offset for reverse node " << id; // Check the NamedNodeBackTranslation API for (size_t start = 0; start + 1 < node_length; start++) { for (size_t length = 1; start + length <= node_length; length++) { // For each possible range on this node, make a range. oriented_node_range_t graph_range {id, true, start, length}; // And translate it back to GFA coordinates std::vector back_translated = graph.translate_back(graph_range); EXPECT_EQ(back_translated.size(), 1ul) << "GFA segments appear merged in graph"; EXPECT_EQ(graph.get_back_graph_node_name(std::get<0>(back_translated[0])), translation.first) << "Graph node belongs to wrong GFA segment"; ASSERT_TRUE(std::get<1>(back_translated[0])) << "Graph node is backward in GFA segment"; EXPECT_EQ(std::get<2>(back_translated[0]), offset + start) << "Graph node range begins at wrong offset on GFA segment"; EXPECT_EQ(std::get<3>(back_translated[0]), length) << "Graph node range translated to GFA range of wrong length"; } } offset += node_length; } } // Entire segments. for(const translation_type& translation : truth) { for(nid_t id = translation.second.first; id < translation.second.second; id++) { EXPECT_EQ(graph.get_segment(graph.get_handle(id, false)), translation) << "Invalid segment for node " << id; } } // For each segment. bool ok = true; auto iter = truth.begin(); graph.for_each_segment([&](const std::string& name, std::pair nodes) -> bool { if(iter == truth.end() || name != iter->first || nodes != iter->second) { ok = false; return false; } ++iter; return true; }); ASSERT_TRUE(ok) << "for_each_segment() did not find the right translations"; EXPECT_EQ(iter, truth.end()) << "for_each_segment() did not find all translations"; } void check_links(const GBWTGraph& graph, const std::vector& edges) const { bool ok = true; auto iter = edges.begin(); graph.for_each_link([&](const edge_t& edge, const std::string& from, const std::string& to) -> bool { std::string from_segment = graph.get_segment_name(edge.first); std::string to_segment = graph.get_segment_name(edge.second); if(iter == edges.end() || edge != *iter || from != from_segment || to != to_segment) { ok = false; return false; } ++iter; return true; }); ASSERT_TRUE(ok) << "for_each_link() did not find the right links"; EXPECT_EQ(iter, edges.end()) << "for_each_links() did not find all links"; } }; TEST_F(GFAConstruction, NormalGraph) { auto gfa_parse = gfa_to_gbwt("gfas/example.gfa"); const gbwt::GBWT& index = *(gfa_parse.first); GBWTGraph graph(*(gfa_parse.first), *(gfa_parse.second)); ASSERT_FALSE(gfa_parse.second->uses_translation()) << "Unnecessary segment translation"; compare_gbwts(index, this->index, NO_FLAGS, ""); compare_graphs(graph, this->graph, NO_FLAGS, ""); this->check_no_translation(graph); } TEST_F(GFAConstruction, WithZeroSegment) { auto gfa_parse = gfa_to_gbwt("gfas/example_0-based.gfa"); const gbwt::GBWT& index = *(gfa_parse.first); GBWTGraph graph(*(gfa_parse.first), *(gfa_parse.second)); std::vector translation = { { "0", { 1, 2 } }, { "1", { 2, 3 } }, { "3", { 4, 5 } }, { "4", { 5, 6 } }, { "5", { 6, 7 } }, { "6", { 7, 8 } }, { "7", { 8, 9 } }, { "8", { 9, 10 } }, }; this->check_translation(*(gfa_parse.second), translation); compare_gbwts(index, this->index, NO_FLAGS, ""); compare_graphs(graph, this->graph, NO_FLAGS, ""); this->check_translation(graph, translation); std::vector links = { edge_t(graph.get_handle(1, false), graph.get_handle(2, false)), edge_t(graph.get_handle(1, false), graph.get_handle(4, false)), edge_t(graph.get_handle(2, false), graph.get_handle(4, false)), edge_t(graph.get_handle(4, false), graph.get_handle(5, false)), edge_t(graph.get_handle(5, false), graph.get_handle(6, false)), edge_t(graph.get_handle(6, false), graph.get_handle(7, false)), edge_t(graph.get_handle(6, false), graph.get_handle(8, false)), edge_t(graph.get_handle(7, false), graph.get_handle(9, false)), edge_t(graph.get_handle(8, false), graph.get_handle(9, false)), }; this->check_links(graph, links); } TEST_F(GFAConstruction, StringSegmentNames) { auto gfa_parse = gfa_to_gbwt("gfas/example_str-names.gfa"); const gbwt::GBWT& index = *(gfa_parse.first); GBWTGraph graph(*(gfa_parse.first), *(gfa_parse.second)); std::vector translation = { { "s1", { 1, 2 } }, { "s2", { 2, 3 } }, { "s4", { 4, 5 } }, { "s5", { 5, 6 } }, { "s6", { 6, 7 } }, { "s7", { 7, 8 } }, { "s8", { 8, 9 } }, { "s9", { 9, 10 } }, }; this->check_translation(*(gfa_parse.second), translation); compare_gbwts(index, this->index, NO_FLAGS, ""); compare_graphs(graph, this->graph, NO_FLAGS, ""); this->check_translation(graph, translation); std::vector links = { edge_t(graph.get_handle(1, false), graph.get_handle(2, false)), edge_t(graph.get_handle(1, false), graph.get_handle(4, false)), edge_t(graph.get_handle(2, false), graph.get_handle(4, false)), edge_t(graph.get_handle(4, false), graph.get_handle(5, false)), edge_t(graph.get_handle(5, false), graph.get_handle(6, false)), edge_t(graph.get_handle(6, false), graph.get_handle(7, false)), edge_t(graph.get_handle(6, false), graph.get_handle(8, false)), edge_t(graph.get_handle(7, false), graph.get_handle(9, false)), edge_t(graph.get_handle(8, false), graph.get_handle(9, false)), }; this->check_links(graph, links); } TEST_F(GFAConstruction, SegmentChopping) { GFAParsingParameters parameters; parameters.max_node_length = 3; auto gfa_parse = gfa_to_gbwt("gfas/example_chopping.gfa", parameters); const gbwt::GBWT& index = *(gfa_parse.first); GBWTGraph graph(*(gfa_parse.first), *(gfa_parse.second)); std::vector translation = { { "1", { 1, 2 } }, { "2", { 2, 3 } }, { "4", { 4, 6 } }, { "6", { 6, 7 } }, { "7", { 7, 8 } }, { "8", { 8, 9 } }, { "9", { 9, 10 } }, }; this->check_translation(*(gfa_parse.second), translation); compare_gbwts(index, this->index, NO_FLAGS, ""); compare_graphs(graph, this->graph, NO_FLAGS, ""); this->check_translation(graph, translation); std::vector links = { edge_t(graph.get_handle(1, false), graph.get_handle(2, false)), edge_t(graph.get_handle(1, false), graph.get_handle(4, false)), edge_t(graph.get_handle(2, false), graph.get_handle(4, false)), edge_t(graph.get_handle(5, false), graph.get_handle(6, false)), edge_t(graph.get_handle(6, false), graph.get_handle(7, false)), edge_t(graph.get_handle(6, false), graph.get_handle(8, false)), edge_t(graph.get_handle(7, false), graph.get_handle(9, false)), edge_t(graph.get_handle(8, false), graph.get_handle(9, false)), }; this->check_links(graph, links); } class GFAConstructionReversal : public GFAConstruction { public: void SetUp() override { // We need to parse the P line in our truth GFA as a haplotype, so we can // match the W line in the test GFA. GFAParsingParameters parameters; parameters.path_name_formats.clear(); parameters.path_name_formats.emplace_back( GFAParsingParameters::PAN_SN_REGEX, GFAParsingParameters::PAN_SN_FIELDS, GFAParsingParameters::PAN_SN_SENSE ); auto truth = gfa_to_gbwt("gfas/reversal.gfa", parameters); this->index = *truth.first; this->source = *truth.second; this->graph = GBWTGraph(this->index, this->source); } }; TEST_F(GFAConstructionReversal, ChoppingWithReversal) { GFAParsingParameters parameters; parameters.max_node_length = 3; auto gfa_parse = gfa_to_gbwt("gfas/reversal_chopping.gfa", parameters); const gbwt::GBWT& index = *(gfa_parse.first); GBWTGraph graph(*(gfa_parse.first), *(gfa_parse.second)); std::vector translation = { { "1", { 1, 2 } }, { "2", { 2, 3 } }, { "3", { 3, 4 } }, { "4", { 4, 6 } }, }; this->check_translation(*(gfa_parse.second), translation); compare_gbwts(index, this->index, NO_FLAGS, ""); compare_graphs(graph, this->graph, NO_FLAGS, ""); this->check_translation(graph, translation); std::vector links = { edge_t(graph.get_handle(1, false), graph.get_handle(2, false)), edge_t(graph.get_handle(1, false), graph.get_handle(3, false)), edge_t(graph.get_handle(2, false), graph.get_handle(5, true)), edge_t(graph.get_handle(3, false), graph.get_handle(4, false)), }; this->check_links(graph, links); } TEST_F(GFAConstructionReversal, WalksWithReversal) { auto gfa_parse = gfa_to_gbwt("gfas/reversal_walks.gfa"); const gbwt::GBWT& index = *(gfa_parse.first); GBWTGraph graph(*(gfa_parse.first), *(gfa_parse.second)); ASSERT_FALSE(gfa_parse.second->uses_translation()) << "Unnecessary segment translation"; compare_gbwts(index, this->index, NO_FLAGS, ""); compare_graphs(graph, this->graph, NO_FLAGS, ""); this->check_paths(graph, &(this->graph)); this->check_no_translation(graph); } class GFAConstructionWalks : public GFAConstruction { public: void SetUp() override { this->index = build_gbwt_example_walks(); this->source = build_naive_graph(false); this->graph = GBWTGraph(this->index, this->source); } }; TEST_F(GFAConstructionWalks, WalksAndPaths) { auto gfa_parse = gfa_to_gbwt("gfas/example_walks.gfa"); const gbwt::GBWT& index = *(gfa_parse.first); GBWTGraph graph(*(gfa_parse.first), *(gfa_parse.second)); ASSERT_FALSE(gfa_parse.second->uses_translation()) << "Unnecessary segment translation"; compare_gbwts(index, this->index, NO_FLAGS, ""); compare_graphs(graph, this->graph, NO_FLAGS, ""); this->check_paths(graph, &(this->graph)); this->check_no_translation(graph); } TEST_F(GFAConstructionWalks, WalksOnly) { auto gfa_parse = gfa_to_gbwt("gfas/example_walks-only.gfa"); const gbwt::GBWT& index = *(gfa_parse.first); GBWTGraph graph(*(gfa_parse.first), *(gfa_parse.second)); ASSERT_FALSE(gfa_parse.second->uses_translation()) << "Unnecessary segment translation"; compare_gbwts(index, this->index, NO_FLAGS, ""); compare_graphs(graph, this->graph, NO_FLAGS, ""); this->check_paths(graph, &(this->graph)); this->check_no_translation(graph); } class GFAConstructionReference : public GFAConstruction { public: void SetUp() override { this->index = build_gbwt_example_reference(); this->source = build_naive_graph(false); this->graph = GBWTGraph(this->index, this->source); } }; TEST_F(GFAConstructionReference, ReferencePaths) { GFAParsingParameters parameters; // Parse panSN paths if possible. parameters.path_name_formats.emplace_front(GFAParsingParameters::PAN_SN_REGEX, GFAParsingParameters::PAN_SN_FIELDS, GFAParsingParameters::PAN_SN_SENSE); auto gfa_parse = gfa_to_gbwt("gfas/example_reference.gfa", parameters); const gbwt::GBWT& index = *(gfa_parse.first); GBWTGraph graph(*(gfa_parse.first), *(gfa_parse.second)); ASSERT_FALSE(gfa_parse.second->uses_translation()) << "Unnecessary segment translation"; compare_gbwts(index, this->index, NO_FLAGS, ""); compare_graphs(graph, this->graph, NO_FLAGS, ""); this->check_paths(graph, &(this->graph)); this->check_no_translation(graph); } //------------------------------------------------------------------------------ class GBWTSubgraph : public ::testing::Test { public: gbwt::GBWT select_paths(const gbwt::GBWT& source, gbwt::size_type skip) const { gbwt::GBWTBuilder builder(sdsl::bits::length(source.sigma() - 1), source.size()); MetadataBuilder metadata_builder; for(gbwt::size_type path_id = 0; path_id < source.metadata.paths(); path_id += 1 + skip) { gbwt::vector_type path = source.extract(gbwt::Path::encode(path_id, false)); builder.insert(path, true); gbwt::FullPathName path_name = source.metadata.fullPath(path_id); metadata_builder.add_gbwt_path(path_name); } builder.finish(); builder.index.addMetadata(); builder.index.metadata = metadata_builder.get_metadata(); return builder.index; } void check_subgraph(const GBWTGraph& graph, const GBWTGraph& subgraph, size_t expected_ref_sample_count) const { bool nodes_ok = true; bool sequences_ok = true; bool reverse_complements_ok = true; subgraph.for_each_handle([&](const handle_t& handle) { if(!(graph.has_node(subgraph.get_id(handle)))) { nodes_ok = false; return; } if(subgraph.get_sequence(handle) != graph.get_sequence(handle)) { sequences_ok = false; } handle_t flipped = subgraph.flip(handle); if(subgraph.get_sequence(flipped) != graph.get_sequence(flipped)) { reverse_complements_ok = false; } }); ASSERT_TRUE(nodes_ok) << "Some nodes were missing from the supergraph"; ASSERT_TRUE(sequences_ok) << "Some sequences were not identical"; ASSERT_TRUE(reverse_complements_ok) << "Some reverse complement sequences were not identical"; bool edges_ok = true; subgraph.for_each_edge([&](const edge_t& edge) { if(!(graph.has_edge(edge.first, edge.second))) { edges_ok = false; } }); ASSERT_TRUE(edges_ok) << "Some edges were missing from the supergraph"; auto parent_ref_samples = parse_reference_samples_tag(*(graph.index)); auto expected_ref_samples = present_sample_names(parent_ref_samples, *(subgraph.index)); auto found_ref_samples = parse_reference_samples_tag(*(subgraph.index)); ASSERT_EQ(found_ref_samples.size(), expected_ref_sample_count) << "Wrong number of reference samples in the subgraph"; ASSERT_EQ(found_ref_samples, expected_ref_samples) << "Wrong reference samples in the subgraph"; } }; TEST_F(GBWTSubgraph, WithoutTranslation) { auto gfa_parse = gfa_to_gbwt("gfas/for_subgraph.gfa"); GBZ gbz(gfa_parse.first, gfa_parse.second); gbwt::GBWT selected = this->select_paths(gbz.index, 1); GBZ subgraph(std::move(selected), gbz); ASSERT_NO_THROW(subgraph.graph.sanity_checks()) << "The subgraph failed sanity checks"; this->check_subgraph(gbz.graph, subgraph.graph, 0); EXPECT_FALSE(subgraph.graph.has_segment_names()) << "The subgraph has segment names"; } TEST_F(GBWTSubgraph, WithTranslation) { GFAParsingParameters parameters; parameters.max_node_length = 3; auto gfa_parse = gfa_to_gbwt("gfas/for_subgraph.gfa", parameters); GBZ gbz(gfa_parse.first, gfa_parse.second); gbwt::GBWT selected = this->select_paths(gbz.index, 1); GBZ subgraph(std::move(selected), gbz); ASSERT_NO_THROW(subgraph.graph.sanity_checks()) << "The subgraph failed sanity checks"; this->check_subgraph(gbz.graph, subgraph.graph, 0); EXPECT_FALSE(subgraph.graph.has_segment_names()) << "The subgraph has segment names"; } TEST_F(GBWTSubgraph, AllPaths) { auto gfa_parse = gfa_to_gbwt("gfas/for_subgraph.gfa"); GBZ gbz(gfa_parse.first, gfa_parse.second); gbwt::GBWT selected = this->select_paths(gbz.index, 0); GBZ subgraph(std::move(selected), gbz); ASSERT_NO_THROW(subgraph.graph.sanity_checks()) << "The subgraph failed sanity checks"; this->check_subgraph(gbz.graph, subgraph.graph, 1); EXPECT_FALSE(subgraph.graph.has_segment_names()) << "The subgraph has segment names"; } //------------------------------------------------------------------------------ // GFA -> GBZ -> GFA, including GBZ construction in more complex situations. class GFAExtraction : public ::testing::Test { public: void extract_gfa(const GBZ& gbz, const std::string& filename, const GFAExtractionParameters& parameters) const { std::ofstream out(filename, std::ios_base::binary); gbwt_to_gfa(gbz, out, parameters); out.close(); } // If a header override is given, replace the header lines related to GraphName // with ones derived from it. void compare_gfas ( const std::string& test_file, const std::string& truth_file, const std::string& name, const GraphName* header_override = nullptr ) const { std::vector test_rows, truth_rows; gbwt::readRows(test_file, test_rows, false); gbwt::readRows(truth_file, truth_rows, false); if(header_override != nullptr) { std::vector modified_truth; // Keep the file header and assume that other header lines are from GraphName. modified_truth.push_back(truth_rows[0]); std::vector header_lines = header_override->gfa_header_lines(); modified_truth.insert(modified_truth.end(), header_lines.begin(), header_lines.end()); // Copy all non-header lines from the truth GFA. for(size_t i = 1; i < truth_rows.size(); i++) { if(truth_rows[i].empty() || truth_rows[i][0] != 'H') { modified_truth.push_back(truth_rows[i]); } } truth_rows = std::move(modified_truth); } ASSERT_EQ(test_rows.size(), truth_rows.size()) << name << ": Invalid number of rows"; size_t line = 0; for(auto iter = test_rows.begin(), truth = truth_rows.begin(); iter != test_rows.end(); ++iter, ++truth) { EXPECT_EQ(*iter, *truth) << name << ": Invalid line " << line; line++; } } }; TEST_F(GFAExtraction, Components) { std::string input = "gfas/components_walks.gfa"; auto gfa_parse = gfa_to_gbwt(input); GBZ gbz(gfa_parse.first, gfa_parse.second); std::string output = gbwt::TempFile::getName("gfa-extraction"); GFAExtractionParameters parameters; this->extract_gfa(gbz, output, parameters); this->compare_gfas(output, input, "Components"); gbwt::TempFile::remove(output); } TEST_F(GFAExtraction, PathsAndWalks) { std::string input = "gfas/example_walks.gfa"; auto gfa_parse = gfa_to_gbwt(input); GBZ gbz(gfa_parse.first, gfa_parse.second); std::string output = gbwt::TempFile::getName("gfa-extraction"); GFAExtractionParameters parameters; this->extract_gfa(gbz, output, parameters); this->compare_gfas(output, input, "Paths and walks"); gbwt::TempFile::remove(output); } TEST_F(GFAExtraction, CacheRecords) { std::string input = "gfas/components_walks.gfa"; auto gfa_parse = gfa_to_gbwt(input); GBZ gbz(gfa_parse.first, gfa_parse.second); std::vector cache_limits = { 0, 1, 2, 4, 8, 16 }; for(size_t limit : cache_limits) { std::string output = gbwt::TempFile::getName("gfa-extraction"); GFAExtractionParameters parameters; parameters.large_record_bytes = limit; this->extract_gfa(gbz, output, parameters); std::string name = "Cache records " + std::to_string(limit); this->compare_gfas(output, input, name); gbwt::TempFile::remove(output); } } TEST_F(GFAExtraction, Compressed) { std::string input = "gfas/components_walks_compressed.gfa"; auto gfa_parse = gfa_to_gbwt(input); GBZ gbz(gfa_parse.first, gfa_parse.second); std::string output = gbwt::TempFile::getName("gfa-extraction"); GFAExtractionParameters parameters; this->extract_gfa(gbz, output, parameters); std::string truth = "gfas/components_walks.gfa"; this->compare_gfas(output, truth, "Compressed"); gbwt::TempFile::remove(output); } TEST_F(GFAExtraction, PathModes) { std::string input = "gfas/default.gfa"; auto gfa_parse = gfa_to_gbwt(input); GBZ gbz(gfa_parse.first, gfa_parse.second); // Default mode. { std::string truth = "gfas/default.gfa"; std::string output = gbwt::TempFile::getName("gfa-modes"); GFAExtractionParameters parameters; parameters.mode = GFAExtractionParameters::mode_default; this->extract_gfa(gbz, output, parameters); this->compare_gfas(output, truth, "Default"); gbwt::TempFile::remove(output); } // PanSN mode. { std::string truth = "gfas/pan-sn.gfa"; std::string output = gbwt::TempFile::getName("gfa-modes"); GFAExtractionParameters parameters; parameters.mode = GFAExtractionParameters::mode_pan_sn; this->extract_gfa(gbz, output, parameters); this->compare_gfas(output, truth, "PanSN"); gbwt::TempFile::remove(output); } // Reference-only mode with generic paths. { std::string truth = "gfas/generic-only.gfa"; std::string output = gbwt::TempFile::getName("gfa-modes"); GFAExtractionParameters parameters; parameters.mode = GFAExtractionParameters::mode_ref_only; this->extract_gfa(gbz, output, parameters); this->compare_gfas(output, truth, "Ref-only with P-lines"); gbwt::TempFile::remove(output); } // Reference-only mode with reference paths. // components_ref is the same as default, but with the reference as W-lines. // TODO: Maybe that graph should be the default. { auto gfa_parse = gfa_to_gbwt("gfas/components_ref.gfa"); GBZ gbz(gfa_parse.first, gfa_parse.second); std::string truth = "gfas/ref-only.gfa"; std::string output = gbwt::TempFile::getName("gfa-modes"); GFAExtractionParameters parameters; parameters.mode = GFAExtractionParameters::mode_ref_only; this->extract_gfa(gbz, output, parameters); this->compare_gfas(output, truth, "Ref-only with W-lines"); gbwt::TempFile::remove(output); } } TEST_F(GFAExtraction, Translation) { GFAParsingParameters parameters; parameters.max_node_length = 3; std::string input = "gfas/example_chopping.gfa"; auto gfa_parse = gfa_to_gbwt(input, parameters); GBZ gbz(gfa_parse.first, gfa_parse.second); EXPECT_FALSE(gbz.translation_target().empty()) << "Translation target tag was not set"; // Use translation. Here we do not know if we produced the translation target or // its subgraph, and hence the resulting graph will be unnamed. { std::string truth = "gfas/example_from_chopping.gfa"; std::string output = gbwt::TempFile::getName("gfa-translation"); GFAExtractionParameters parameters; parameters.use_translation = true; this->extract_gfa(gbz, output, parameters); GraphName empty; this->compare_gfas(output, truth, "With translation", &empty); gbwt::TempFile::remove(output); } // No translation. { std::string truth = "gfas/example_chopped.gfa"; std::string output = gbwt::TempFile::getName("gfa-translation"); GFAExtractionParameters parameters; parameters.use_translation = false; this->extract_gfa(gbz, output, parameters); GraphName name = gbz.graph_name(); this->compare_gfas(output, truth, "Without translation", &name); gbwt::TempFile::remove(output); } } TEST_F(GFAExtraction, CanonicalGFA) { std::vector inputs = { "gfas/components_walks.gfa", "gfas/example_walks.gfa", "gfas/example_chopping.gfa" }; for(const std::string& input : inputs) { auto gfa_parse = gfa_to_gbwt(input); GBZ gbz(gfa_parse.first, gfa_parse.second); std::stringstream truth_stream; handlegraph::algorithms::canonical_gfa(gbz.graph, truth_stream, true); std::string truth = truth_stream.str(); std::stringstream output_stream; gbwt_to_canonical_gfa(gbz.graph, output_stream); std::string output = output_stream.str(); ASSERT_EQ(output, truth) << "Canonical GFA mismatch for " << input; } } //------------------------------------------------------------------------------ class GBWTMetadata : public ::testing::Test { public: std::string path_name_regex; std::string samples_and_haplotypes; std::string contigs_and_fragments; std::vector names; std::vector ids; GBWTMetadata() { } void SetUp() override { this->path_name_regex = "(.)(.)"; this->samples_and_haplotypes = "-SH"; this->contigs_and_fragments = "-CF"; this->names = { "A", "B" }; this->ids = { static_cast(1), static_cast(2) }; } void check_metadata(const gbwt::Metadata& metadata, const gbwt::Metadata& expected) { // Samples. ASSERT_EQ(metadata.samples(), expected.samples()) << "Wrong number of samples"; if(expected.hasSampleNames()) { ASSERT_TRUE(metadata.hasSampleNames()) << "No sample names"; bool names_ok = true; for(gbwt::size_type i = 0; i < metadata.samples(); i++) { if(metadata.sample(i) != expected.sample(i)) { names_ok = false; } } EXPECT_TRUE(names_ok) << "Invalid sample names"; } else { EXPECT_FALSE(metadata.hasSampleNames()) << "Sample names were created"; } // Haplotypes. EXPECT_EQ(metadata.haplotypes(), expected.haplotypes()) << "Wrong number of haplotypes"; // Contigs. ASSERT_EQ(metadata.contigs(), expected.contigs()) << "Wrong number of contigs"; if(expected.hasContigNames()) { ASSERT_TRUE(metadata.hasContigNames()) << "No contig names"; bool names_ok = true; for(gbwt::size_type i = 0; i < metadata.contigs(); i++) { if(metadata.contig(i) != expected.contig(i)) { names_ok = false; } } EXPECT_TRUE(names_ok) << "Invalid contig names"; } else { EXPECT_FALSE(metadata.hasContigNames()) << "Contig names were created"; } // Paths. ASSERT_EQ(metadata.paths(), expected.paths()) << "Wrong number of paths"; if(expected.hasPathNames()) { ASSERT_TRUE(metadata.hasPathNames()) << "No path names"; bool names_ok = true; for(gbwt::size_type i = 0; i < metadata.paths(); i++) { if(metadata.path(i) != expected.path(i)) { names_ok = false; } } EXPECT_TRUE(names_ok) << "Invalid path names"; } else { EXPECT_FALSE(metadata.hasPathNames()) << "Path names were created"; } } }; TEST_F(GBWTMetadata, SamplesAndHaplotypes) { GFAParsingParameters parameters; parameters.path_name_formats.clear(); parameters.path_name_formats.emplace_back(this->path_name_regex, this->samples_and_haplotypes, PathSense::HAPLOTYPE); auto gfa_parse = gfa_to_gbwt("gfas/components.gfa", parameters); const gbwt::GBWT& index = *(gfa_parse.first); gbwt::Metadata expected_metadata; expected_metadata.setSamples(this->names); expected_metadata.setHaplotypes(4); expected_metadata.setContigs(1); expected_metadata.addPath(0, 0, 1, 0); expected_metadata.addPath(0, 0, 2, 0); expected_metadata.addPath(1, 0, 1, 0); expected_metadata.addPath(1, 0, 2, 0); ASSERT_TRUE(index.hasMetadata()) << "No GBWT metadata was created"; this->check_metadata(index.metadata, expected_metadata); } TEST_F(GBWTMetadata, ContigsAndFragments) { GFAParsingParameters parameters; parameters.path_name_formats.clear(); parameters.path_name_formats.emplace_back(this->path_name_regex, this->contigs_and_fragments, PathSense::HAPLOTYPE); auto gfa_parse = gfa_to_gbwt("gfas/components.gfa", parameters); const gbwt::GBWT& index = *(gfa_parse.first); gbwt::Metadata expected_metadata; expected_metadata.setSamples(1); expected_metadata.setHaplotypes(1); expected_metadata.setContigs(this->names); expected_metadata.addPath(0, 0, GBWTGraph::NO_PHASE, 1); expected_metadata.addPath(0, 0, GBWTGraph::NO_PHASE, 2); expected_metadata.addPath(0, 1, GBWTGraph::NO_PHASE, 1); expected_metadata.addPath(0, 1, GBWTGraph::NO_PHASE, 2); ASSERT_TRUE(index.hasMetadata()) << "No GBWT metadata was created"; this->check_metadata(index.metadata, expected_metadata); } TEST_F(GBWTMetadata, Walks) { auto gfa_parse = gfa_to_gbwt("gfas/components_walks.gfa"); const gbwt::GBWT& index = *(gfa_parse.first); gbwt::Metadata expected_metadata; std::vector samples = { "sample" }; expected_metadata.setSamples(samples); expected_metadata.setHaplotypes(2); expected_metadata.setContigs(this->names); expected_metadata.addPath(0, 0, 1, 0); expected_metadata.addPath(0, 0, 2, 0); expected_metadata.addPath(0, 1, 1, 0); expected_metadata.addPath(0, 1, 2, 0); ASSERT_TRUE(index.hasMetadata()) << "No GBWT metadata was created"; this->check_metadata(index.metadata, expected_metadata); } TEST_F(GBWTMetadata, WalksNoInterval) { auto gfa_parse = gfa_to_gbwt("gfas/components_walks_no_interval.gfa"); const gbwt::GBWT& index = *(gfa_parse.first); gbwt::Metadata expected_metadata; std::vector samples = { "sample" }; expected_metadata.setSamples(samples); expected_metadata.setHaplotypes(2); expected_metadata.setContigs(this->names); expected_metadata.addPath(0, 0, 1, 0); expected_metadata.addPath(0, 0, 2, 0); expected_metadata.addPath(0, 1, 1, 0); expected_metadata.addPath(0, 1, 2, 0); ASSERT_TRUE(index.hasMetadata()) << "No GBWT metadata was created"; this->check_metadata(index.metadata, expected_metadata); } TEST_F(GBWTMetadata, WalksAndPaths) { auto gfa_parse = gfa_to_gbwt("gfas/example_walks.gfa"); const gbwt::GBWT& index = *(gfa_parse.first); gbwt::Metadata expected_metadata; std::vector samples = { GENERIC_PATH_SAMPLE_NAME, "short", "alt" }; expected_metadata.setSamples(samples); expected_metadata.setHaplotypes(4); std::vector contigs = { "short", "alt1", "alt2", "chr" }; expected_metadata.setContigs(contigs); expected_metadata.addPath(0, 0, GBWTGraph::NO_PHASE, 0); expected_metadata.addPath(0, 1, GBWTGraph::NO_PHASE, 0); expected_metadata.addPath(0, 2, GBWTGraph::NO_PHASE, 0); expected_metadata.addPath(1, 3, 1, 0); expected_metadata.addPath(2, 3, 0, 0); expected_metadata.addPath(1, 3, 2, 0); ASSERT_TRUE(index.hasMetadata()) << "No GBWT metadata was created"; this->check_metadata(index.metadata, expected_metadata); } //------------------------------------------------------------------------------ class GFAGrammarTest : public ::testing::Test { public: using expansion_t = GFAGrammar::expansion_type; expansion_t reverse(const expansion_t& path) const { std::vector> rev_path; for(auto iter = path.rbegin(); iter != path.rend(); ++iter) { rev_path.emplace_back(iter->first, !iter->second); } return rev_path; } expansion_t collect(GFAGrammarIterator& iter) const { expansion_t result; for(std::pair next = iter.next(); !next.first.empty(); next = iter.next()) { result.emplace_back(std::string(next.first), next.second); } return result; } }; TEST_F(GFAGrammarTest, Empty) { GFAGrammar grammar; EXPECT_TRUE(grammar.empty()) << "New grammar is not empty"; EXPECT_EQ(grammar.size(), size_t(0)) << "New grammar has non-zero size"; EXPECT_EQ(grammar.expand("example"), grammar.no_rule()) << "New grammar has rules"; GFAGrammarIterator iter = grammar.iter("example", false); EXPECT_EQ(iter.next(), std::make_pair(std::string_view(), false)) << "Iterator returned a value for empty grammar"; NaiveGraph source; EXPECT_NO_THROW(grammar.validate(source)) << "Empty grammar failed validation"; } TEST_F(GFAGrammarTest, NonEmpty) { GFAGrammar grammar; std::unordered_map rules { { "A", { { "1", false }, { "2", false }, { "3", false } } }, // >1>2>3 { "B", { { "4", false }, { "5", true } } }, // >4<5 { "C", { { "6", true }, { "7", true }, { "8", false } } }, // <6<7>8 { "D", { { "A", false }, { "B", true } } }, // >AB>C { "F", { { "D", false }, { "E", false } } }, // >D>E }; std::unordered_map expansions; expansions["A"] = rules["A"]; expansions["B"] = rules["B"]; expansions["C"] = rules["C"]; { auto d = expansions["A"]; auto b_rev = this->reverse(expansions["B"]); d.insert(d.end(), b_rev.begin(), b_rev.end()); expansions["D"] = d; } { auto e = expansions["B"]; auto c = expansions["C"]; e.insert(e.end(), c.begin(), c.end()); expansions["E"] = e; } { auto f = expansions["D"]; auto e = expansions["E"]; f.insert(f.end(), e.begin(), e.end()); expansions["F"] = f; } // Insert the rules. for(auto iter = rules.begin(); iter != rules.end(); ++iter) { bool success = grammar.insert(std::string(iter->first), expansion_t(iter->second)); ASSERT_TRUE(success) << "Failed to insert rule " << iter->first; } EXPECT_FALSE(grammar.empty()) << "Grammar is empty after inserting rules"; EXPECT_EQ(grammar.size(), rules.size()) << "Grammar has wrong size after inserting rules"; // Check direct expansions. for(auto iter = rules.begin(); iter != rules.end(); ++iter) { auto rule = grammar.expand(iter->first); EXPECT_EQ(rule->first, iter->first) << "Wrong name for the direct expansion of rule " << iter->first; EXPECT_EQ(rule->second, rules[iter->first]) << "Wrong direct expansion for rule " << iter->first; } { std::string nonexistent = "nonexistent"; auto no_rule = grammar.expand(nonexistent); EXPECT_EQ(no_rule, grammar.no_rule()) << "Nonexistent rule has an expansion"; } // Iterators in both directions. for(auto iter = expansions.begin(); iter != expansions.end(); ++iter) { std::string first = grammar.first_segment(iter->first); EXPECT_EQ(first, iter->second.front().first) << "Wrong first segment for rule " << iter->first; GFAGrammarIterator fwd_iter = grammar.iter(iter->first, false); EXPECT_FALSE(fwd_iter.empty()) << "Forward iterator is empty for rule " << iter->first; expansion_t forward = this->collect(fwd_iter); EXPECT_TRUE(fwd_iter.empty()) << "Forward iterator was not exhausted for rule " << iter->first; EXPECT_EQ(forward, iter->second) << "Wrong forward expansion for rule " << iter->first; expansion_t reverse_truth = this->reverse(iter->second); GFAGrammarIterator rev_iter = grammar.iter(iter->first, true); EXPECT_FALSE(rev_iter.empty()) << "Reverse iterator is empty for rule " << iter->first; expansion_t reverse = this->collect(rev_iter); EXPECT_TRUE(rev_iter.empty()) << "Reverse iterator was not exhausted for rule " << iter->first; EXPECT_EQ(reverse, reverse_truth) << "Wrong reverse expansion for rule " << iter->first; } { std::string nonexistent = "nonexistent"; GFAGrammarIterator no_iter = grammar.iter(nonexistent, false); EXPECT_TRUE(no_iter.empty()) << "Iterator is not empty for nonexistent rule"; EXPECT_EQ(no_iter.next(), std::make_pair(std::string_view(), false)) << "Iterator returned a value for nonexistent rule"; std::string first = grammar.first_segment(nonexistent); EXPECT_EQ(first, nonexistent) << "Wrong first segment for a symbol that is not a rule"; } // Validation. NaiveGraph source; for(nid_t id = 1; id <= 8; id++) { source.create_node(id, "ACGT"); } EXPECT_NO_THROW(grammar.validate(source)) << "Valid grammar failed validation"; } TEST_F(GFAGrammarTest, SpecialCases) { { expansion_t correct { { "correct", false } }; expansion_t incorrect { { "incorrect", false } }; GFAGrammar grammar; grammar.insert("A", expansion_t(correct)); bool success = grammar.insert("A", expansion_t(incorrect)); EXPECT_FALSE(success) << "Inserted duplicate rule"; auto rule = grammar.expand("A"); EXPECT_EQ(rule->second, correct) << "Duplicate rule overwrote original"; } { GFAGrammar grammar; grammar.insert("A", expansion_t{ { "A", false }, { "1", false } }); NaiveGraph source; source.create_node(1, "ACGT"); EXPECT_THROW(grammar.validate(source), std::runtime_error) << "Simple cycle passed validation"; } { std::unordered_map rules { { "A", { { "1", false }, { "2", false }, { "3", false } } }, // >1>2>3 { "B", { { "4", false }, { "5", true } } }, // >4<5 { "C", { { "6", true }, { "7", true }, { "8", false }, { "F", false } } }, // <6<7>8>F { "D", { { "A", false }, { "B", true } } }, // >AB>C { "F", { { "D", false }, { "E", false } } }, // >D>E }; GFAGrammar grammar; for(auto iter = rules.begin(); iter != rules.end(); ++iter) { grammar.insert(std::string(iter->first), expansion_t(iter->second)); } NaiveGraph source; for(nid_t id = 1; id <= 8; id++) { source.create_node(id, "ACGT"); } EXPECT_THROW(grammar.validate(source), std::runtime_error) << "Indirect cycle passed validation"; } { GFAGrammar grammar; grammar.insert("A", expansion_t{ { "B", false }, { "1", false } }); NaiveGraph source; source.create_node(1, "ACGT"); EXPECT_THROW(grammar.validate(source), std::runtime_error) << "Undefined symbol passed validation"; } { GFAGrammar grammar; grammar.insert("A", expansion_t{ { "1", false }, { "B", false } }); grammar.insert("B", expansion_t{ { "2", false }, { "3", false } }); NaiveGraph source; std::string sequence = "GATTACA"; source.translate_segment("B", sequence, 3); // Name clash; translates to nodes 1 to 3. EXPECT_THROW(grammar.validate(source), std::runtime_error) << "Name clash passed validation"; } { GFAGrammar grammar; grammar.insert("", expansion_t{ { "1", false }, { "2", false } }); NaiveGraph source; source.create_node(1, "ACGT"); source.create_node(2, "TGCA"); EXPECT_THROW(grammar.validate(source), std::runtime_error) << "Empty rule name passed validation"; } { GFAGrammar grammar; grammar.insert("A", expansion_t{ }); NaiveGraph source; EXPECT_THROW(grammar.validate(source), std::runtime_error) << "Empty rule passed validation"; } { GFAGrammar grammar; grammar.insert("A", expansion_t{ { "1", false } }); NaiveGraph source; source.create_node(1, "ACGT"); EXPECT_THROW(grammar.validate(source), std::runtime_error) << "Trivial rule passed validation"; } } //------------------------------------------------------------------------------ } // namespace jltsiren-gbwtgraph-e27bc43/tests/test_index.cpp000066400000000000000000000567521524136466400220100ustar00rootroot00000000000000#include #include #include #include #include #include #include "shared.h" using namespace gbwtgraph; namespace { //------------------------------------------------------------------------------ using KeyTypes = ::testing::Types; const std::vector payload_sizes { 0, 1, 2, 3 }; //------------------------------------------------------------------------------ template class KmerExtraction : public ::testing::Test { }; TYPED_TEST_SUITE(KmerExtraction, KeyTypes); TYPED_TEST(KmerExtraction, CanonicalKmers) { typedef TypeParam key_type; std::string sequence = "GATTACAGATTA"; std::vector> truth { { key_type::encode("ATC"), 0, 2, true }, { key_type::encode("AAT"), 0, 3, true }, { key_type::encode("TAA"), 0, 4, true }, { key_type::encode("GTA"), 0, 5, true }, { key_type::encode("ACA"), 0, 4, false }, { key_type::encode("CAG"), 0, 5, false }, { key_type::encode("AGA"), 0, 6, false }, { key_type::encode("ATC"), 0, 9, true }, { key_type::encode("AAT"), 0, 10, true }, { key_type::encode("TAA"), 0, 11, true }, }; std::sort(truth.begin(), truth.end()); auto kmers = canonical_kmers(sequence, 3); ASSERT_EQ(kmers.size(), truth.size()) << "Invalid number of kmers"; for(size_t i = 0; i < truth.size(); i++) { EXPECT_EQ(kmers[i], truth[i]) << "Kmer " << i << ": expected " << truth[i] << ", got " << kmers[i]; } } TYPED_TEST(KmerExtraction, InvalidCharacters) { typedef TypeParam key_type; std::string sequence = "GATTAxAGATTA"; std::vector> truth { { key_type::encode("ATC"), 0, 2, true }, { key_type::encode("AAT"), 0, 3, true }, { key_type::encode("TAA"), 0, 4, true }, { key_type::encode("AGA"), 0, 6, false }, { key_type::encode("ATC"), 0, 9, true }, { key_type::encode("AAT"), 0, 10, true }, { key_type::encode("TAA"), 0, 11, true }, }; std::sort(truth.begin(), truth.end()); auto kmers = canonical_kmers(sequence, 3); ASSERT_EQ(kmers.size(), truth.size()) << "Invalid number of kmers"; for(size_t i = 0; i < truth.size(); i++) { EXPECT_EQ(kmers[i], truth[i]) << "Kmer " << i << ": expected " << truth[i] << ", got " << kmers[i]; } } //------------------------------------------------------------------------------ template class IndexConstruction : public ::testing::Test { public: GBZ gbz; void SetUp() override { gbwt::GBWT index = build_gbwt_index(); NaiveGraph graph = build_naive_graph(false); this->gbz = GBZ(index, graph); } static std::vector> get_kmers(const KmerIndex&, const std::string& str, size_t k) { return canonical_kmers(str, k); } static std::vector> get_kmers(const MinimizerIndex& index, const std::string& str, size_t) { return index.minimizers(str); } template void insert_values(const IndexType& index, const gbwt::vector_type& path, std::map>& result, size_t k) const { // Convert the path to a string and find the minimizers. std::string str = path_to_string(this->gbz.graph, path); auto kmers = get_kmers(index, str, k); // Insert the minimizers into the result. auto iter = path.begin(); size_t node_start = 0; for(auto kmer : kmers) { if(kmer.empty()) { continue; } handle_t handle = GBWTGraph::node_to_handle(*iter); size_t node_length = this->gbz.graph.get_length(handle); while(node_start + node_length <= kmer.offset) { node_start += node_length; ++iter; handle = GBWTGraph::node_to_handle(*iter); node_length = this->gbz.graph.get_length(handle); } pos_t pos { this->gbz.graph.get_id(handle), this->gbz.graph.get_is_reverse(handle), kmer.offset - node_start }; if(kmer.is_reverse) { pos = reverse_base_pos(pos, node_length); } owned_value_type value = create_value(pos, index.payload_size(), hash(pos)); result[kmer.key].insert(value); } } // This version stores path information in the last word of payload. template void insert_values(const IndexType& index, const std::vector& paths, std::map>& result, size_t k) const { ASSERT_GE(index.payload_size(), 1) << "Index must have payload to store path information"; for(size_t path_id = 0; path_id < paths.size(); path_id++) { // Convert the path to a string and find the minimizers. const gbwt::vector_type& path = paths[path_id]; std::string str = path_to_string(this->gbz.graph, path); auto kmers = get_kmers(index, str, k); // Insert the minimizers into the result. auto iter = path.begin(); size_t node_start = 0; for(auto kmer : kmers) { if(kmer.empty()) { continue; } handle_t handle = GBWTGraph::node_to_handle(*iter); size_t node_length = this->gbz.graph.get_length(handle); while(node_start + node_length <= kmer.offset) { node_start += node_length; ++iter; handle = GBWTGraph::node_to_handle(*iter); node_length = this->gbz.graph.get_length(handle); } pos_t pos { this->gbz.graph.get_id(handle), this->gbz.graph.get_is_reverse(handle), kmer.offset - node_start }; if(kmer.is_reverse) { pos = reverse_base_pos(pos, node_length); } // This value will be inserted/updated in the set. owned_value_type value = create_value(pos, index.payload_size(), hash(pos)); value.second.back() = 1 << path_id; auto found = result.find(kmer.key); if(found == result.end()) { // Case 1: first occurrence of the kmer. result[kmer.key].insert(value); continue; } bool same_hit = false; owned_value_type old_value; for(auto& existing_value : found->second) { if(existing_value.first == value.first) { // Case 2: same position already exists; merge the sets of paths. value.second.back() |= existing_value.second.back(); same_hit = true; old_value = existing_value; break; } } if(same_hit) { // Update the existing value with the new path information. found->second.erase(old_value); found->second.insert(value); } else { // Case 3: new position for the kmer. result[kmer.key].insert(value); } } } } std::map> filter_values(const std::map>& source, char middle_base) const { // NOTE: This assumes 3-mers. std::map> result; for(auto iter = source.begin(); iter != source.end(); ++iter) { if(iter->first.access(3, 1) == middle_base) { result[iter->first] = iter->second; } } return result; } static KmerEncoding::multi_value_type find_values(const KmerIndex& index, KeyType key) { return index.find(key); } static KmerEncoding::multi_value_type find_values(const MinimizerIndex& index, KeyType key) { return index.find(get_minimizer(key)); } template void check_index(const IndexType& index, const std::map>& correct_values, bool with_paths) const { using code_type = KmerEncoding::code_type; std::string payload_msg = " with payload size " + std::to_string(index.payload_size()); size_t values = 0; for(auto iter = correct_values.begin(); iter != correct_values.end(); ++iter) { values += iter->second.size(); } ASSERT_EQ(index.size(), correct_values.size()) << "Wrong number of keys" << payload_msg; ASSERT_EQ(index.number_of_values(), values) << "Wrong number of values" << payload_msg; for(auto iter = correct_values.begin(); iter != correct_values.end(); ++iter) { KmerEncoding::multi_value_type values = find_values(index, iter->first); EXPECT_TRUE(same_values(values, iter->second, index.payload_size(), with_paths)) << "Wrong values for key " << iter->first << payload_msg; } } void check_graph_names(const MinimizerIndex& index) const { GraphName from_graph = this->gbz.graph_name(); GraphName from_index = index.graph_name(); EXPECT_EQ(from_graph.name(), from_index.name()) << "Graph names do not match"; EXPECT_EQ(from_graph, from_index) << "Graph relationships do not match"; } }; TYPED_TEST_SUITE(IndexConstruction, KeyTypes); TYPED_TEST(IndexConstruction, WithoutPayload) { // Determine the correct minimizer occurrences. MinimizerIndex index(3, 2, 0); std::map> correct_values; this->insert_values(index, alt_path, correct_values, index.k()); this->insert_values(index, short_path, correct_values, index.k()); // Check that we managed to index them. index_haplotypes(this->gbz, index, [](const pos_t&) { return nullptr; }); this->check_index(index, correct_values, false); this->check_graph_names(index); } TYPED_TEST(IndexConstruction, WithPayload) { using key_type = TypeParam; using index_type = MinimizerIndex; for(size_t payload_size : payload_sizes) { // Determine the correct minimizer occurrences. index_type index(3, 2, payload_size); std::map> correct_values; this->insert_values(index, alt_path, correct_values, index.k()); this->insert_values(index, short_path, correct_values, index.k()); // Extract the payloads. std::map> payloads; for(auto& pair : correct_values) { for(auto& value : pair.second) { payloads[value.first.decode()] = value.second; } } // Check that we managed to index them. index_haplotypes(this->gbz, index, [&](const pos_t& pos) { return payloads[pos].data(); }); this->check_index(index, correct_values, false); this->check_graph_names(index); } } TYPED_TEST(IndexConstruction, WithPaths) { using key_type = TypeParam; using index_type = MinimizerIndex; // We need path metadata for indexing with paths. ASSERT_EQ(this->gbz.index.sequences(), 6) << "Metadata generation code does not match test index"; this->gbz.index.addMetadata(); this->gbz.index.metadata.setSamples(3); this->gbz.index.metadata.setHaplotypes(3); this->gbz.index.metadata.setContigs(1); for(size_t path_id = 0; path_id < 3; path_id++) { this->gbz.index.metadata.addPath(path_id, 0, 0, 0); } for(size_t payload_size : payload_sizes) { if(payload_size == 0) { index_type index(3, 2, payload_size); EXPECT_THROW ( index_haplotypes_with_paths(this->gbz, index, [](const pos_t&) { return nullptr; }), std::runtime_error ) << "Expected exception when indexing with paths and zero payload size"; continue; } // Determine the correct minimizer occurrences. index_type index(3, 2, payload_size); std::map> correct_values; std::vector paths { short_path, alt_path, short_path }; this->insert_values(index, paths, correct_values, index.k()); // Extract the payloads. std::map> payloads; for(auto& pair : correct_values) { for(auto& value : pair.second) { // We may have different payloads for different extensions of the same position. // But because we only use the shared prefix (of length payload_size() - 1), // we can overwrite the existing value. payloads[value.first.decode()] = value.second; } } // Check that we managed to index them. index_haplotypes_with_paths(this->gbz, index, [&](const pos_t& pos) { return payloads[pos].data(); }); std::string path_name_fields = index.get_tag(PATH_NAME_FIELDS_TAG); EXPECT_NE(path_name_fields, "") << "Path name fields tag was not set with payload size " << payload_size; this->check_index(index, correct_values, true); this->check_graph_names(index); } } TYPED_TEST(IndexConstruction, CanonicalKmers) { using key_type = TypeParam; using index_type = KmerIndex; // Determine the correct canonical kmers. index_type index(0); std::map> correct_values; this->insert_values(index, alt_path, correct_values, 3); this->insert_values(index, short_path, correct_values, 3); // Check that we managed to index them. std::function include = [](key_type) -> bool { return true; }; build_kmer_index(this->gbz.graph, index, 3, include); this->check_index(index, correct_values, false); } TYPED_TEST(IndexConstruction, CanonicalKmersByMiddleBase) { using key_type = TypeParam; using index_type = KmerIndex; // Determine the correct canonical kmers with C as the middle base. index_type index(0); std::map> all_values; this->insert_values(index, alt_path, all_values, 3); this->insert_values(index, short_path, all_values, 3); std::map> correct_values = this->filter_values(all_values, 'C'); // Check that we managed to index them. std::function include = [&](key_type key) -> bool { return (key.access(3, 1) == 'C'); }; build_kmer_index(this->gbz.graph, index, 3, include); this->check_index(index, correct_values, false); } TYPED_TEST(IndexConstruction, MultipleKmerIndexes) { using key_type = TypeParam; using index_type = KmerIndex; // Determine the correct canonical kmers. std::array indexes { index_type(0), index_type(0), index_type(0), index_type(0) }; std::map> all_values; this->insert_values(indexes[0], alt_path, all_values, 3); this->insert_values(indexes[0], short_path, all_values, 3); // Build the indexes. build_kmer_indexes(this->gbz.graph, indexes, 3); // Check that the kmers were partitioned correctly. std::string bases = "ACGT"; for(size_t i = 0; i < bases.length(); i++) { std::map> correct_values = this->filter_values(all_values, bases[i]); this->check_index(indexes[i], correct_values, false); } } //------------------------------------------------------------------------------ template class KmerCounting : public ::testing::Test { public: gbwt::GBWT index; NaiveGraph source; GBWTGraph graph; void SetUp() override { this->index = build_gbwt_index(); this->source = build_naive_graph(false); this->graph = GBWTGraph(this->index, this->source); } }; TYPED_TEST_SUITE(KmerCounting, KeyTypes); TYPED_TEST(KmerCounting, FrequentKmers) { // alt_path: G A GGG T A - A A // short_path: G - GGG T A C - A // CCC: two reverse occurrences // GTA: one forward and one reverse occurrence size_t k = 3; std::vector correct; correct.push_back(TypeParam::encode("CCC")); correct.push_back(TypeParam::encode("GTA")); auto kmers = frequent_kmers(this->graph, k, 1, false); ASSERT_EQ(kmers.size(), correct.size()) << "Invalid number of frequent kmers"; for(size_t i = 0; i < kmers.size(); i++) { EXPECT_EQ(kmers[i], correct[i]) << "Invalid kmer " << i << ": expected " << correct[i].decode(k) << ", got " << kmers[i].decode(k); } auto space_efficient = frequent_kmers(this->graph, k, 1, true); ASSERT_EQ(space_efficient, kmers) << "Invalid frequent kmers using the space-efficient algorithm"; } //------------------------------------------------------------------------------ class PathIdTest : public ::testing::Test { static void shuffle(std::vector& paths, size_t seed = 0xACDCABBACAFEBEEF) { if(paths.empty()) { return; } std::mt19937_64 rng(seed); for(size_t i = paths.size() - 1; i > 0; i--) { size_t j = rng() % (i + 1); std::swap(paths[i], paths[j]); } } gbwt::Metadata generate_test_case(size_t samples, size_t haplotypes, size_t contigs, size_t fragments) const { gbwt::Metadata metadata; metadata.setSamples(samples); metadata.setHaplotypes(samples * haplotypes); metadata.setContigs(contigs); std::vector paths; for(size_t s = 0; s < samples; s++) { for(size_t c = 0; c < contigs; c++) { for(size_t h = 0; h < haplotypes; h++) { for(size_t f = 0; f < fragments; f++) { gbwt::PathName path(s, c, h, f); paths.push_back(path); } } } } shuffle(paths); // Shuffle to test that order does not matter. for(const gbwt::PathName& path : paths) { metadata.addPath(path); } return metadata; } std::unordered_map generate_truth(size_t samples, size_t haplotypes, size_t contigs, size_t fragments) const { std::unordered_map truth; bool use_contigs = (samples * haplotypes * contigs <= PathIdMap::MAX_HAPLOTYPES); bool use_haplotypes = (samples * haplotypes <= PathIdMap::MAX_HAPLOTYPES) & !use_contigs; bool use_samples = (samples <= PathIdMap::MAX_HAPLOTYPES) & !use_haplotypes & !use_contigs; // Note that the iteration order does not match the order of the fields. // Haplotypes are more significant than contigs. size_t next = 0; for(size_t s = 0; s < samples; s++) { for(size_t h = 0; h < haplotypes; h++) { for(size_t c = 0; c < contigs; c++) { for(size_t f = 0; f < fragments; f++) { gbwt::PathName path(s, c, h, f); truth[path] = next; } if(use_contigs) { next++; } } if(use_haplotypes) { next++; } } if(use_samples) { next++; } } return truth; } public: void path_id_map_test(size_t samples, size_t haplotypes, size_t contigs, size_t fragments, PathIdMap::KeyType key_type) const { std::string test_case = " with (" + std::to_string(samples) + " samples, " + std::to_string(haplotypes) + " haplotypes/sample, " + std::to_string(contigs) + " contigs, " + std::to_string(fragments) + " fragments/sequence)"; gbwt::Metadata metadata = this->generate_test_case(samples, haplotypes, contigs, fragments); PathIdMap path_id_map(metadata); ASSERT_EQ(path_id_map.key_type(), key_type) << "Expected key type " << PathIdMap::key_type_str(key_type) << ", got " << PathIdMap::key_type_str(path_id_map.key_type()) << test_case; auto truth = this->generate_truth(samples, haplotypes, contigs, fragments); for(gbwt::size_type i = 0; i < metadata.paths(); i++) { const gbwt::PathName& path = metadata.path(i); size_t expected_id = truth[path]; size_t actual_id = path_id_map.id(path); EXPECT_LT(actual_id, PathIdMap::MAX_HAPLOTYPES) << "Too large path id for path " << path_name_to_string(path) << test_case; EXPECT_EQ(actual_id, expected_id) << "Wrong path id for path " << path_name_to_string(path) << test_case; } gbwt::PathName missing(samples, 0, 0, 0); size_t missing_id = path_id_map.id(missing); EXPECT_EQ(missing_id, size_t(0)) << "Non-zero path id for missing path " << path_name_to_string(missing) << test_case; } std::vector get_path(const gbwt::GBWT& index, gbwt::size_type seq_id) const { gbwt::vector_type gbwt_path = index.extract(seq_id); std::vector path; for(size_t i = 0; i < gbwt_path.size(); i++) { path.push_back(GBWTGraph::node_to_handle(gbwt_path[i])); } return path; } std::vector path_offset_by_seq_offset(const GBWTGraph& graph, const std::vector& path) const { std::vector result; for(size_t i = 0; i < path.size(); i++) { handle_t handle = path[i]; size_t node_length = graph.get_length(handle); for(size_t j = 0; j < node_length; j++) { result.push_back(i); } } return result; } std::vector true_kmer_path(const std::vector& path, const std::vector& expanded, size_t seq_offset, size_t k, bool is_reverse) const { std::vector result; if(is_reverse) { size_t prev = path.size(); for(size_t i = 0; i < k && i <= seq_offset; i++) { if(expanded[seq_offset - i] == prev) { continue; } prev = expanded[seq_offset - i]; result.push_back(gbwt::Node::reverse(GBWTGraph::handle_to_node(path[prev]))); } } else { size_t prev = path.size(); for(size_t i = 0; i < k && seq_offset + i < expanded.size(); i++) { if(expanded[seq_offset + i] == prev) { continue; } prev = expanded[seq_offset + i]; result.push_back(GBWTGraph::handle_to_node(path[prev])); } } return result; } }; TEST_F(PathIdTest, PathIdMap) { this->path_id_map_test(0, 0, 0, 0, PathIdMap::KeyType::SAMPLE_CONTIG_HAPLOTYPE); // Empty metadata. this->path_id_map_test(1, 1, 1, 1, PathIdMap::KeyType::SAMPLE_CONTIG_HAPLOTYPE); // Single path. this->path_id_map_test(2, 2, 2, 2, PathIdMap::KeyType::SAMPLE_CONTIG_HAPLOTYPE); // Multiple paths. this->path_id_map_test(4, 4, 4, 1, PathIdMap::KeyType::SAMPLE_CONTIG_HAPLOTYPE); // Should still use (sample, haplotype, contig). this->path_id_map_test(5, 1, 13, 1, PathIdMap::KeyType::SAMPLE_HAPLOTYPE); // Fall back to (sample, haplotype). this->path_id_map_test(32, 2, 2, 1, PathIdMap::KeyType::SAMPLE_HAPLOTYPE); // Should still use (sample, haplotype). this->path_id_map_test(13, 5, 1, 1, PathIdMap::KeyType::SAMPLE); // Fall back to (sample). this->path_id_map_test(PathIdMap::MAX_HAPLOTYPES, 2, 24, 1, PathIdMap::KeyType::SAMPLE); // Largest case where we can still use (sample) this->path_id_map_test(PathIdMap::MAX_HAPLOTYPES + 1, 2, 24, 1, PathIdMap::KeyType::NONE); // Smallest case where we can no longer use (sample) this->path_id_map_test(230, 2, 24, 3, PathIdMap::KeyType::NONE); // Large test case, where everything should map to 0. } TEST_F(PathIdTest, ExtractKmerPath) { gbwt::GBWT index = build_gbwt_index(); NaiveGraph source = build_naive_graph(false); GBWTGraph graph(index, source); size_t k = 3; for(gbwt::size_type seq_id = 0; seq_id < index.sequences(); seq_id++) { std::vector path = this->get_path(index, seq_id); std::vector expanded = this->path_offset_by_seq_offset(graph, path); size_t seq_offset = 0; for(size_t path_offset = 0; path_offset < path.size(); path_offset++) { size_t node_length = graph.get_length(path[path_offset]); for(size_t node_offset = 0; node_offset < node_length; node_offset++, seq_offset++) { for(bool is_reverse : { false, true }) { std::string test_case = " for sequence " + std::to_string(seq_id) + ", path offset " + std::to_string(path_offset) + ", node offset " + std::to_string(node_offset) + (is_reverse ? " (reverse)" : " (forward)"); std::vector kmer_path = extract_kmer_path(graph, path, path_offset, node_offset, k, is_reverse); std::vector truth = this->true_kmer_path(path, expanded, seq_offset, k, is_reverse); EXPECT_EQ(kmer_path, truth) << "Wrong kmer path" << test_case; } } } } } //------------------------------------------------------------------------------ } // namespace jltsiren-gbwtgraph-e27bc43/tests/test_minimizer.cpp000066400000000000000000001355441524136466400227010ustar00rootroot00000000000000#include #include #include #include #include #include #include #include #include #include #include "shared.h" using namespace gbwtgraph; namespace { //------------------------------------------------------------------------------ using KeyTypes = ::testing::Types; const std::vector payload_sizes { 0, 1, 2, 3 }; //------------------------------------------------------------------------------ template class KmerIndexManipulation : public ::testing::Test { }; TYPED_TEST_SUITE(KmerIndexManipulation, KeyTypes); TYPED_TEST(KmerIndexManipulation, EmptyIndex) { using key_type = TypeParam; using index_type = KmerIndex; for(size_t payload_size : payload_sizes) { index_type index(payload_size); std::string msg = "New index with payload size " + std::to_string(payload_size); EXPECT_EQ(index.size(), 0) << msg << " contains keys"; EXPECT_TRUE(index.empty()) << msg << " is not empty"; EXPECT_EQ(index.number_of_values(), 0) << msg << " has values"; EXPECT_EQ(index.unique_keys(), 0) << msg << " has unique keys"; EXPECT_EQ(index.payload_size(), payload_size) << msg << " has wrong payload size"; index_type copy(index); EXPECT_EQ(index, copy) << "A copy of a new index with payload size " << payload_size << " is not identical to the original"; } } TYPED_TEST(KmerIndexManipulation, Contents) { using key_type = TypeParam; using index_type = KmerIndex; for(size_t payload_size : payload_sizes) { index_type index(payload_size); index_type copy(index); // Different contents. { pos_t pos = make_pos_t(1, false, 3); auto value = create_value(pos, payload_size, hash(pos)); insert_value(index, key_type(1), value); EXPECT_NE(index, copy) << "Empty index is identical to nonempty index with payload size " << payload_size; } // Same key, different value. { pos_t pos = make_pos_t(2, false, 3); auto value = create_value(pos, payload_size, hash(pos)); insert_value(copy, key_type(1), value); EXPECT_NE(index, copy) << "Indexes with different values are identical with payload size " << payload_size; } // Same contents. { copy = index; EXPECT_EQ(index, copy) << "A copy of a nonempty index is not identical to the original with payload size " << payload_size; } } } TYPED_TEST(KmerIndexManipulation, Swap) { using key_type = TypeParam; using index_type = KmerIndex; for(size_t payload_size : payload_sizes) { index_type first(payload_size), second(payload_size); pos_t first_pos = make_pos_t(1, false, 3); auto first_value = create_value(first_pos, payload_size, hash(first_pos)); pos_t second_pos = make_pos_t(2, false, 3); auto second_value = create_value(second_pos, payload_size, hash(second_pos)); insert_value(first, key_type(1), first_value); insert_value(second, key_type(2), second_value); index_type first_copy(first), second_copy(second); first.swap(second); EXPECT_NE(first, first_copy) << "Swapping did not change the first index with payload size " << payload_size; EXPECT_EQ(first, second_copy) << "The first index was not swapped correctly with payload size " << payload_size; EXPECT_EQ(second, first_copy) << "The second index was not swapped correctly with payload size " << payload_size; EXPECT_NE(second, second_copy) << "Swapping did not change the second index with payload size " << payload_size; } } //------------------------------------------------------------------------------ template class CorrectKmers : public ::testing::Test { public: using key_type = KeyType; using index_type = KmerIndex; using code_type = KmerEncoding::code_type; using multi_value_type = KmerEncoding::multi_value_type; using result_type = std::map>; size_t total_keys; virtual void SetUp() { this->total_keys = 16; } void check_kmer_index ( const index_type& index, const result_type& correct_values, size_t keys, size_t values, size_t unique ) { std::string payload_msg = " with payload size " + std::to_string(index.payload_size()); ASSERT_EQ(index.size(), keys) << "Wrong number of keys" << payload_msg; ASSERT_EQ(index.number_of_values(), values) << "Wrong number of values" << payload_msg; EXPECT_EQ(index.unique_keys(), unique) << "Wrong number of unique keys" << payload_msg; for(auto iter = correct_values.begin(); iter != correct_values.end(); ++iter) { size_t count = index.count(iter->first); EXPECT_EQ(count, iter->second.size()) << "Wrong number of occurrences for key " << iter->first << payload_msg; if(count != iter->second.size()) { continue; } multi_value_type values = index.find(iter->first); EXPECT_TRUE(same_values(values, iter->second, index.payload_size(), false)) << "Wrong values for key " << iter->first << payload_msg; } } }; TYPED_TEST_SUITE(CorrectKmers, KeyTypes); TYPED_TEST(CorrectKmers, GetValue) { using key_type = TypeParam; using index_type = KmerIndex; using value_type = KmerEncoding::value_type; using multi_value_type = KmerEncoding::multi_value_type; std::mt19937_64 rng(0xACDCABBA); for(size_t payload_size : payload_sizes) { index_type index(payload_size); for(size_t values = 0; values <= 10; values++) { owned_multi_value_type owned_values; std::vector correct_values; for(size_t i = 0; i < values; i++) { pos_t pos = make_pos_t(rng() % 100 + 1, rng() & 1, rng() % 100); owned_value_type value = create_value(pos, payload_size, hash(pos)); append_value(owned_values, value, payload_size); correct_values.push_back(value); } std::string test_case = " with payload size " + std::to_string(payload_size) + " and " + std::to_string(values) + " values"; ASSERT_EQ(owned_values.size() / index.value_size(), values) << "Test error" << test_case; multi_value_type encoded_values(owned_values.data(), values); for(size_t i = 0; i < values; i++) { value_type decoded = index.get_value(encoded_values, i); EXPECT_TRUE(same_value(decoded, correct_values[i], payload_size, false)) << "Wrong value " << i << test_case; } value_type no_value(Position::no_pos(), nullptr); value_type past_end = index.get_value(encoded_values, values); EXPECT_EQ(past_end, no_value) << "Wrong past-the-end value" << test_case; } } } TYPED_TEST(CorrectKmers, UniqueKeys) { using key_type = TypeParam; using index_type = KmerIndex; for(size_t payload_size : payload_sizes) { index_type index(payload_size); size_t keys = 0, values = 0, unique = 0; typename TestFixture::result_type correct_values; for(size_t i = 1; i <= this->total_keys; i++) { key_type key(i); pos_t pos = make_pos_t(i, i & 1, i & Position::OFF_MASK); owned_value_type value = create_value(pos, payload_size, hash(pos)); insert_value(index, key, value); correct_values[key].insert(value); keys++; values++; unique++; } this->check_kmer_index(index, correct_values, keys, values, unique); } } TYPED_TEST(CorrectKmers, MissingKeys) { using key_type = TypeParam; using index_type = KmerIndex; for(size_t payload_size : payload_sizes) { index_type index(payload_size); for(size_t i = 1; i <= this->total_keys; i++) { key_type key(i); pos_t pos = make_pos_t(i, i & 1, i & Position::OFF_MASK); owned_value_type value = create_value(pos, payload_size, hash(pos)); insert_value(index, key, value); } std::string payload_msg = " with payload size " + std::to_string(index.payload_size()); for(size_t i = this->total_keys + 1; i <= 2 * this->total_keys; i++) { key_type key(i); EXPECT_EQ(index.count(key), size_t(0)) << "Non-zero occurrences for key " << i << payload_msg; EXPECT_EQ(index.find(key).second, size_t(0)) << "Non-empty occurrences for key " << i << payload_msg; } } } TYPED_TEST(CorrectKmers, EmptyKeysValues) { using key_type = TypeParam; using index_type = KmerIndex; for(size_t payload_size : payload_sizes) { index_type index(payload_size); std::string payload_msg = " with payload size " + std::to_string(index.payload_size()); key_type no_key = key_type::no_key(); insert_value(index, no_key, create_value(make_pos_t(1, false, 0), payload_size, 0)); EXPECT_EQ(index.count(no_key), size_t(0)) << "Non-zero occurrences for empty key" << payload_msg; EXPECT_EQ(index.find(no_key).second, size_t(0)) << "Non-empty value for empty key" << payload_msg; key_type key(this->total_keys + 1); insert_value(index, key, create_value(Position::no_pos().decode(), payload_size, 0)); EXPECT_EQ(index.count(key), size_t(0)) << "Non-zero occurrences after inserting empty value" << payload_msg; EXPECT_EQ(index.find(key).second, size_t(0)) << "Non-empty value after inserting empty value" << payload_msg; } } TYPED_TEST(CorrectKmers, MultipleOccurrences) { using key_type = TypeParam; using index_type = KmerIndex; for(size_t payload_size : payload_sizes) { index_type index(payload_size); size_t keys = 0, values = 0, unique = 0; typename TestFixture::result_type correct_values; for(size_t i = 1; i <= this->total_keys; i++) { key_type key(i); pos_t pos = make_pos_t(i, i & 1, i & Position::OFF_MASK); owned_value_type value = create_value(pos, payload_size, hash(pos)); insert_value(index, key, value); correct_values[key].insert(value); keys++; values++; unique++; } for(size_t i = 1; i <= this->total_keys; i += 2) { key_type key(i); pos_t pos = make_pos_t(i + 1, i & 1, (i + 1) & Position::OFF_MASK); owned_value_type value = create_value(pos, payload_size, hash(pos)); insert_value(index, key, value); correct_values[key].insert(value); values++; unique--; } for(size_t i = 1; i <= this->total_keys; i += 4) { key_type key(i); pos_t pos = make_pos_t(i + 2, i & 1, (i + 2) & Position::OFF_MASK); owned_value_type value = create_value(pos, payload_size, hash(pos)); insert_value(index, key, value); correct_values[key].insert(value); values++; } this->check_kmer_index(index, correct_values, keys, values, unique); } } TYPED_TEST(CorrectKmers, DuplicateValues) { using key_type = TypeParam; using index_type = KmerIndex; for(size_t payload_size : payload_sizes) { index_type index(payload_size); size_t keys = 0, values = 0, unique = 0; typename TestFixture::result_type correct_values; for(size_t i = 1; i <= this->total_keys; i++) { key_type key(i); pos_t pos = make_pos_t(i, i & 1, i & Position::OFF_MASK); owned_value_type value = create_value(pos, payload_size, hash(pos)); insert_value(index, key, value); correct_values[key].insert(value); keys++; values++; unique++; } for(size_t i = 1; i <= this->total_keys; i += 2) { key_type key(i); pos_t pos = make_pos_t(i + 1, i & 1, (i + 1) & Position::OFF_MASK); owned_value_type value = create_value(pos, payload_size, hash(pos)); insert_value(index, key, value); correct_values[key].insert(value); values++; unique--; } for(size_t i = 1; i <= this->total_keys; i += 4) { // Also check that inserting duplicates does not change the payload. key_type key(i); pos_t pos = make_pos_t(i + 1, i & 1, (i + 1) & Position::OFF_MASK); owned_value_type value = create_value(pos, payload_size, hash(pos) + 1); insert_value(index, key, value); } this->check_kmer_index(index, correct_values, keys, values, unique); } } TYPED_TEST(CorrectKmers, Rehashing) { using key_type = TypeParam; using index_type = KmerIndex; for(size_t payload_size : payload_sizes) { index_type index(payload_size); std::string payload_msg = " with payload size " + std::to_string(index.payload_size()); size_t keys = 0, values = 0, unique = 0; typename TestFixture::result_type correct_values; size_t threshold = index.capacity(); for(size_t i = 1; i <= threshold; i++) { key_type key(i); pos_t pos = make_pos_t(i, i & 1, i & Position::OFF_MASK); owned_value_type value = create_value(pos, payload_size, hash(pos)); insert_value(index, key, value); correct_values[key].insert(value); keys++; values++; unique++; } ASSERT_EQ(index.capacity(), threshold) << "Index capacity changed at threshold" << payload_msg; { size_t i = threshold + 1; key_type key(i); pos_t pos = make_pos_t(i, i & 1, i & Position::OFF_MASK); owned_value_type value = create_value(pos, payload_size, hash(pos)); insert_value(index, key, value); correct_values[key].insert(value); keys++; values++; unique++; } EXPECT_GT(index.capacity(), threshold) << "Index capacity did not increase after threshold" << payload_msg; this->check_kmer_index(index, correct_values, keys, values, unique); } } //------------------------------------------------------------------------------ template class ObjectManipulation : public ::testing::Test { }; TYPED_TEST_SUITE(ObjectManipulation, KeyTypes); TYPED_TEST(ObjectManipulation, EmptyIndex) { using key_type = TypeParam; using index_type = MinimizerIndex; for(size_t payload_size : payload_sizes) { index_type default_index(payload_size); index_type default_copy(default_index); index_type alt_index(15, 6, payload_size); index_type alt_copy(alt_index); std::string payload_msg = " with payload size " + std::to_string(payload_size); EXPECT_EQ(default_index, default_copy) << "A copy of the default index is not identical to the original" << payload_msg; EXPECT_EQ(alt_index, alt_copy) << "A copy of a parameterized index is not identical to the original" << payload_msg; EXPECT_NE(default_index, alt_index) << "Default and parameterized indexes are identical" << payload_msg; } } TYPED_TEST(ObjectManipulation, SyncmerIndex) { using key_type = TypeParam; using index_type = MinimizerIndex; for(size_t payload_size : payload_sizes) { index_type default_minimizer(payload_size); index_type default_syncmer(payload_size, true); index_type short_minimizer(15, 6, payload_size); index_type short_syncmer(15, 6, payload_size, true); std::string payload_msg = " with payload size " + std::to_string(payload_size); EXPECT_FALSE(default_minimizer.uses_syncmers()) << "Default minimizer index uses syncmers" << payload_msg; EXPECT_TRUE(default_syncmer.uses_syncmers()) << "Default syncmer index uses minimizers" << payload_msg; EXPECT_FALSE(short_minimizer.uses_syncmers()) << "Parameterized minimizer index uses syncmers" << payload_msg; EXPECT_TRUE(short_syncmer.uses_syncmers()) << "Parameterized syncmer index uses minimizers" << payload_msg; } } TYPED_TEST(ObjectManipulation, Contents) { using key_type = TypeParam; using index_type = MinimizerIndex; for(size_t payload_size : payload_sizes) { index_type default_index(payload_size); index_type default_copy(default_index); std::string payload_msg = " with payload size " + std::to_string(payload_size); // Different contents. { auto minimizer = get_minimizer(key_type(1)); pos_t pos = make_pos_t(1, false, 3); owned_value_type value = create_value(pos, payload_size, hash(pos)); insert_value(default_index, minimizer, value); EXPECT_NE(default_index, default_copy) << "Empty index is identical to nonempty index" << payload_msg; } // Same key, different value. { auto minimizer = get_minimizer(key_type(1)); pos_t pos = make_pos_t(2, false, 3); owned_value_type value = create_value(pos, payload_size, hash(pos)); insert_value(default_copy, minimizer, value); EXPECT_NE(default_index, default_copy) << "Indexes with different values are identical" << payload_msg; } // Same contents. default_copy = default_index; EXPECT_EQ(default_index, default_copy) << "A copy of a nonempty index is not identical to the original" << payload_msg; } } TYPED_TEST(ObjectManipulation, Swap) { using key_type = TypeParam; using index_type = MinimizerIndex; for(size_t payload_size : payload_sizes) { index_type first(payload_size), second(payload_size); auto first_minimizer = get_minimizer(key_type(1)); pos_t first_pos = make_pos_t(1, false, 3); owned_value_type first_value = create_value(first_pos, payload_size, hash(first_pos)); insert_value(first, first_minimizer, first_value); auto second_minimizer = get_minimizer(key_type(2)); pos_t second_pos = make_pos_t(2, false, 3); owned_value_type second_value = create_value(second_pos, payload_size, hash(second_pos)); insert_value(second, second_minimizer, second_value); index_type first_copy(first), second_copy(second); first.swap(second); std::string payload_msg = " with payload size " + std::to_string(payload_size); EXPECT_NE(first, first_copy) << "Swapping did not change the first index" << payload_msg; EXPECT_EQ(first, second_copy) << "The first index was not swapped correctly" << payload_msg; EXPECT_EQ(second, first_copy) << "The second index was not swapped correctly" << payload_msg; EXPECT_NE(second, second_copy) << "Swapping did not change the second index" << payload_msg; } } TYPED_TEST(ObjectManipulation, Tags) { using key_type = TypeParam; using index_type = MinimizerIndex; // Keys and values we are going to use. std::string lower_case = "tag"; std::string upper_case = "TAG"; std::string another = "another"; std::string first_value = "value1"; std::string second_value = "value2"; // Empty tags. index_type index(0); EXPECT_EQ(index.get_tag(lower_case), "") << "New index has tag " << lower_case; EXPECT_EQ(index.tags_begin(), index.tags_end()) << "New index has tags"; // Setting a new tag and retrieving its value with case insensitive keys. index.set_tag(lower_case, first_value); EXPECT_EQ(index.get_tag(lower_case), first_value) << "Could not retrieve tag " << lower_case; EXPECT_EQ(index.get_tag(upper_case), first_value) << "Could not retrieve tag " << lower_case << " as " << upper_case; // Updating a tag. index.set_tag(upper_case, second_value); EXPECT_EQ(index.get_tag(upper_case), second_value) << "Could not retrieve the updated value for tag " << upper_case; // Iterator over the tags in sorted order. index.set_tag(another, first_value); std::vector> correct_tags { { another, first_value }, { lower_case, second_value } }; auto iter = index.tags_begin(); auto truth_iter = correct_tags.begin(); for(size_t i = 0; iter != index.tags_end() && truth_iter != correct_tags.end(); ++iter, ++truth_iter, i++) { EXPECT_EQ(*iter, *truth_iter) << "Tag " << i << " is incorrect"; } EXPECT_EQ(iter, index.tags_end()) << "Index has too many tags"; EXPECT_EQ(truth_iter, correct_tags.end()) << "Index has too few tags"; // Copying and swapping. index_type second(0); EXPECT_NE(index, second) << "Index with tags is identical to index without tags"; index_type first_copy(index); EXPECT_EQ(index, first_copy) << "A copy of an index with tags is not identical to the original"; second.swap(first_copy); EXPECT_EQ(index, second) << "Swapping did not preserve tags"; EXPECT_NE(index, first_copy) << "Swapping did not change the index"; // Removing a tag. second.remove_tag(lower_case); EXPECT_EQ(second.get_tag(lower_case), "") << "Could still retrieve removed tag " << lower_case; EXPECT_EQ(second.get_tag(another), first_value) << "Could not retrieve tag " << another << " after removing " << lower_case; // Serialization is tested below. } //------------------------------------------------------------------------------ template class Serialization : public ::testing::Test { }; TYPED_TEST_SUITE(Serialization, KeyTypes); TYPED_TEST(Serialization, Serialize) { typedef TypeParam key_type; typedef MinimizerIndex index_type; for(size_t payload_size : payload_sizes) { index_type index(15, 6, payload_size); auto first_minimizer = get_minimizer(key_type(1)); pos_t first_pos = make_pos_t(1, false, 3); owned_value_type first_value = create_value(first_pos, payload_size, hash(first_pos)); insert_value(index, first_minimizer, first_value); auto second_minimizer = get_minimizer(key_type(2)); insert_value(index, second_minimizer, first_value); pos_t second_pos = make_pos_t(2, false, 3); owned_value_type second_value = create_value(second_pos, payload_size, hash(second_pos)); insert_value(index, second_minimizer, second_value); index.set_tag("tag", "value"); std::string filename = gbwt::TempFile::getName("minimizer"); std::ofstream out(filename, std::ios_base::binary); index.serialize(out); out.close(); index_type copy(payload_size + 1); // Payload size should be overwritten. std::ifstream in(filename, std::ios_base::binary); copy.deserialize(in); in.close(); gbwt::TempFile::remove(filename); EXPECT_EQ(index, copy) << "Loaded index is not identical to the original with payload size " << payload_size; } } TYPED_TEST(Serialization, WeightedMinimizers) { typedef TypeParam key_type; typedef MinimizerIndex index_type; for(size_t payload_size : payload_sizes) { index_type index(15, 6, payload_size); std::string payload_msg = " with payload size " + std::to_string(payload_size); ASSERT_FALSE(index.uses_weighted_minimizers()) << "Weighted minimizers are in use by default" << payload_msg; index.add_frequent_kmers({ key_type::encode("GATTACACATGATTA"), key_type::encode("TATTAGATTACATTA") }, 3); ASSERT_TRUE(index.uses_weighted_minimizers()) << "Weighted minimizers could not be enabled" << payload_msg; auto first_minimizer = get_minimizer(key_type(1)); pos_t first_pos = make_pos_t(1, false, 3); owned_value_type first_value = create_value(first_pos, payload_size, hash(first_pos)); insert_value(index, first_minimizer, first_value); auto second_minimizer = get_minimizer(key_type(2)); insert_value(index, second_minimizer, first_value); pos_t second_pos = make_pos_t(2, false, 3); owned_value_type second_value = create_value(second_pos, payload_size, hash(second_pos)); insert_value(index, second_minimizer, second_value); std::string filename = gbwt::TempFile::getName("minimizer"); std::ofstream out(filename, std::ios_base::binary); index.serialize(out); out.close(); index_type copy(payload_size + 1); // Payload size should be overwritten. std::ifstream in(filename, std::ios_base::binary); copy.deserialize(in); in.close(); gbwt::TempFile::remove(filename); EXPECT_EQ(copy, index) << "Loaded index is not identical to the original" << payload_msg; } } //------------------------------------------------------------------------------ template class KeyEncodeDecode : public ::testing::Test { public: std::string get_string(size_t k) const { std::string result; for(size_t i = 0; i < k; i += 7) { result += "GATTACA"; } return result.substr(0, k); } }; TYPED_TEST_SUITE(KeyEncodeDecode, KeyTypes); TYPED_TEST(KeyEncodeDecode, SimpleSequence) { for(size_t k = 1; k <= TypeParam::KMER_MAX_LENGTH; k++) { for(auto base : {'A', 'C', 'G', 'T'}) { std::string kmer(k, base); TypeParam encoded = TypeParam::encode(kmer); std::string decoded = encoded.decode(k); EXPECT_EQ(decoded, kmer) << "Decoded kmer is not identical to original"; } } } TYPED_TEST(KeyEncodeDecode, ComplexSequence) { for(size_t k = 1; k <= TypeParam::KMER_MAX_LENGTH; k++) { std::string kmer = this->get_string(k); TypeParam encoded = TypeParam::encode(kmer); std::string decoded = encoded.decode(k); EXPECT_EQ(decoded, kmer) << "Decoded kmer is not identical to original"; } } TYPED_TEST(KeyEncodeDecode, RandomAccess) { for(size_t k = 1; k <= TypeParam::KMER_MAX_LENGTH; k++) { std::string kmer = this->get_string(k); TypeParam encoded = TypeParam::encode(kmer); for(size_t i = 0; i < k; i++) { ASSERT_EQ(encoded.access(k, i), kmer[i]) << "Invalid access(" << k << ", " << i << ")"; } } } TYPED_TEST(KeyEncodeDecode, ReverseComplement) { for(size_t k = 1; k <= TypeParam::KMER_MAX_LENGTH; k++) { std::string kmer = this->get_string(k); TypeParam encoded = TypeParam::encode(kmer); TypeParam rc = encoded.reverse_complement(k); std::string decoded = rc.decode(k); std::string truth = reverse_complement(kmer); EXPECT_EQ(decoded, truth) << "Incorrect reverse complement"; } } //------------------------------------------------------------------------------ /* Order of 3-mers using Key64: AAT < TGT < TTG < TAT < ATA < TCG < ATT < ACA < GAA < ACT < TAC < CGA < CAA < GTA < TTC < AGT Merged with reverse complements: AAT < TGT < TTG < TAT < TCG < GAA < ACT < TAC ATT ACA CAA ATA CGA TTC AGT GTA Order of 3-mers using Key128: ATT < TCG < ATA < TAT < TTG < TGT < AAT < AGT < TTC < GTA < CAA < TAC < CGA < ACT < ACA < GAA Merged with reverse complements: ATT < TCG < ATA < TTG < TGT < AGT < TTC < GTA AAT CGA TAT CAA ACA ACT GAA TAC */ template class MinimizerExtraction : public ::testing::Test { public: std::string str, rev, repetitive; virtual void SetUp() { this->str = "CGAATACAATACT"; this->rev = reverse_complement(this->str); this->repetitive = "TATATA"; } }; TYPED_TEST_SUITE(MinimizerExtraction, KeyTypes); TYPED_TEST(MinimizerExtraction, KeyEncoding) { typedef TypeParam key_type; size_t k = key_type::KMER_MAX_LENGTH - 1; std::string bases = "ACGT"; key_type forward_key, reverse_key; size_t valid_chars = 0; std::string correct; for(size_t i = 0; i < 2 * k + 1; i++) { char c = bases[i % bases.length()]; forward_key.forward(k, c, valid_chars); reverse_key.reverse(k, c); correct += c; } correct = correct.substr(correct.length() - k); std::string reverse = reverse_complement(correct); EXPECT_EQ(forward_key.decode(k), correct) << "Invalid forward key " << forward_key; EXPECT_EQ(reverse_key.decode(k), reverse) << "Invalid reverse key " << reverse_key; } TYPED_TEST(MinimizerExtraction, AllMinimizers) { typedef TypeParam key_type; typedef MinimizerIndex index_type; typedef Kmer minimizer_type; index_type index(3, 2, 0); std::vector correct; if(key_type::KEY_BITS == 128) { correct = { get_minimizer("TCG", 2, true), get_minimizer("ATA", 3, false), get_minimizer("ATT", 4, true), get_minimizer("TGT", 7, true), get_minimizer("TTG", 8, true), get_minimizer("ATA", 8, false), get_minimizer("ATT", 9, true), get_minimizer("AGT", 12, true) }; } else { correct = { get_minimizer("TCG", 2, true), get_minimizer("AAT", 2, false), get_minimizer("TAT", 5, true), get_minimizer("TGT", 7, true), get_minimizer("AAT", 7, false), get_minimizer("TAT", 10, true), get_minimizer("ACT", 10, false) }; } std::vector result = index.minimizers(this->str.begin(), this->str.end()); ASSERT_EQ(result, correct) << "Did not find the correct minimizers"; std::vector fallback = index.syncmers(this->str.begin(), this->str.end()); EXPECT_EQ(fallback, correct) << "Did not find the correct minimizers using syncmers()"; } TYPED_TEST(MinimizerExtraction, WeightedMinimizers) { // This is the same test as above, but we want to avoid kmer ATA. With Key64, // we get the lowest-priority kmer TAC as the minimizer in both windows. With // Key128, we get its reverse complement GTA instead. typedef TypeParam key_type; typedef MinimizerIndex index_type; typedef Kmer minimizer_type; index_type index(3, 2, 0); index.add_frequent_kmers({ key_type::encode("ATA") }, 3); std::vector correct; if(key_type::KEY_BITS == 128) { correct = { get_minimizer("TCG", 2, true), get_minimizer("ATT", 4, true), get_minimizer("GTA", 6, true), get_minimizer("TGT", 7, true), get_minimizer("TTG", 8, true), get_minimizer("ATT", 9, true), get_minimizer("GTA", 11, true), get_minimizer("AGT", 12, true) }; } else { correct = { get_minimizer("TCG", 2, true), get_minimizer("AAT", 2, false), get_minimizer("TAC", 4, false), get_minimizer("TGT", 7, true), get_minimizer("AAT", 7, false), get_minimizer("TAC", 9, false), get_minimizer("ACT", 10, false) }; } std::vector result = index.minimizers(this->str.begin(), this->str.end()); ASSERT_EQ(result, correct) << "Did not find the correct minimizers"; } TYPED_TEST(MinimizerExtraction, ClosedSyncmers) { typedef TypeParam key_type; typedef MinimizerIndex index_type; typedef Kmer minimizer_type; index_type index(5, 3, 0, true); std::vector correct; if(key_type::KEY_BITS == 128) { correct = { get_minimizer("ATACA", 3, false), get_minimizer("ATTCG", 4, true), get_minimizer("GTATT", 6, true), get_minimizer("TTGTA", 8, true), get_minimizer("ATACT", 8, false), get_minimizer("ATTGT", 9, true), get_minimizer("GTATT", 11, true) }; } else { correct = { get_minimizer("AATAC", 2, false), get_minimizer("ATACA", 3, false), get_minimizer("ATTCG", 4, true), get_minimizer("ACAAT", 5, false), get_minimizer("AATAC", 7, false), get_minimizer("AGTAT", 12, true) }; } std::vector result = index.syncmers(this->str.begin(), this->str.end()); ASSERT_EQ(result, correct) << "Did not find the correct syncmers"; std::vector fallback = index.minimizers(this->str.begin(), this->str.end()); EXPECT_EQ(fallback, correct) << "Did not find the correct syncmers using minimizers()"; } TYPED_TEST(MinimizerExtraction, AllMinimizersWithRegions) { typedef TypeParam key_type; typedef MinimizerIndex index_type; typedef Kmer minimizer_type; index_type index(3, 2, 0); std::vector> correct; if(key_type::KEY_BITS == 128) { // 0000000000111 // 0123456789012 // CGAATACAATACT // CGA- // *--* // ATA+ // *--* // AAT- // *---* // ACA- // *--* // CAA- // *--* // ATA+ // *--* // AAT- // *---* // ACT- // *--* // Each window length will be at least k + w - 1 = 4 bases. correct = { std::make_tuple(get_minimizer("TCG", 2, true), 0, 4), std::make_tuple(get_minimizer("ATA", 3, false), 3, 4), std::make_tuple(get_minimizer("ATT", 4, true), 1, 5), std::make_tuple(get_minimizer("TGT", 7, true), 4, 4), std::make_tuple(get_minimizer("TTG", 8, true), 5, 4), std::make_tuple(get_minimizer("ATA", 8, false), 8, 4), std::make_tuple(get_minimizer("ATT", 9, true), 6, 5), std::make_tuple(get_minimizer("AGT", 12, true), 9, 4) }; } else { // 0000000000111 // 0123456789012 // CGAATACAATACT // CGA- // *--* // AAT+ // *---* // ATA- // *--* // ACA- // *---* // AAT+ // *---* // ATA- // *--* // ACT+ // *--* // Note that no minimizers actually get replaced in this example; they all take their second possible window. correct = { std::make_tuple(get_minimizer("TCG", 2, true), 0, 4), std::make_tuple(get_minimizer("AAT", 2, false), 1, 5), std::make_tuple(get_minimizer("TAT", 5, true), 3, 4), std::make_tuple(get_minimizer("TGT", 7, true), 4, 5), std::make_tuple(get_minimizer("AAT", 7, false), 6, 5), std::make_tuple(get_minimizer("TAT", 10, true), 8, 4), std::make_tuple(get_minimizer("ACT", 10, false), 9, 4) }; } std::vector> result = index.minimizer_regions(this->str.begin(), this->str.end()); EXPECT_EQ(result, correct) << "Did not find the correct minimizers"; } TEST(MinimizerExtraction, HardMinimizersWithRegion) { using TypeParam = Key128; typedef TypeParam key_type; typedef MinimizerIndex index_type; typedef Kmer minimizer_type; // Here's a case I caught not working correctly. index_type index(29, 11, 0); std::string seq("ACCAGTTTTTTACACAAGCTGCTCTTTCCCTCAATTGTTCATTTGTCTCCTTGTCCAGGT"); // No replacements happen // 000000000011111111112222222222333333333344444444445555555555 // 012345678901234567890123456789012345678901234567890123456789 // ACCAGTTTTTTACACAAGCTGCTCTTTCCCTCAATTGTTCATTTGTCTCCTTGTCCAGGT // TTTTTACACAAGCTGCTCTTTCCCTCAAT- (#1) // *-------------------------------------------* // CACAAGCTGCTCTTTCCCTCAATTGTTCA+ (#2) // *------------------------------------------* // GCTGCTCTTTCCCTCAATTGTTCATTTGT- (#3) // *-----------------------------------------* // TTTCCCTCAATTGTTCATTTGTCTCCTTG+ (#4) // *----------------------------------------* // These then get sorted by minimizer offset, at the end for reverse strand minimizers. std::vector> correct; correct = { std::make_tuple(get_minimizer("CACAAGCTGCTCTTTCCCTCAATTGTTCA", 12, false), 7, 44), // #2 std::make_tuple(get_minimizer("TTTCCCTCAATTGTTCATTTGTCTCCTTG", 24, false), 18, 42), // #4 std::make_tuple(get_minimizer("ATTGAGGGAAAGAGCAGCTTGTGTAAAAA", 34, true), 0, 45), // #1 std::make_tuple(get_minimizer("ACAAATGAACAATTGAGGGAAAGAGCAGC", 45, true), 13, 43) // #3 }; std::vector> result = index.minimizer_regions(seq.begin(), seq.end()); EXPECT_EQ(result, correct) << "Did not find the correct minimizers"; } TYPED_TEST(MinimizerExtraction, WindowLength) { typedef TypeParam key_type; typedef MinimizerIndex index_type; typedef Kmer minimizer_type; index_type index(3, 3, 0); std::vector correct; if(key_type::KEY_BITS == 128) { correct = { get_minimizer("ATA", 3, false), get_minimizer("ATT", 4, true), get_minimizer("TTG", 8, true), get_minimizer("ATA", 8, false), get_minimizer("ATT", 9, true) }; } else { correct = { get_minimizer("AAT", 2, false), get_minimizer("TGT", 7, true), get_minimizer("AAT", 7, false), get_minimizer("TAT", 10, true) }; } std::vector result = index.minimizers(this->str.begin(), this->str.end()); EXPECT_EQ(result, correct) << "Did not find the correct minimizers"; } // We have multiple occurrences of the same minimizer in the initial window and after it. // When we find the first occurrence after the window, one of the original occurrences is // still in the buffer. TYPED_TEST(MinimizerExtraction, AllOccurrences) { typedef TypeParam key_type; typedef MinimizerIndex index_type; typedef Kmer minimizer_type; index_type index(3, 3, 0); std::vector correct; if(key_type::KEY_BITS == 128) { correct = { get_minimizer("ATA", 1, false), get_minimizer("ATA", 2, true), get_minimizer("ATA", 3, false), get_minimizer("ATA", 4, true) }; } else { correct = { get_minimizer("TAT", 0, false), get_minimizer("TAT", 2, false), get_minimizer("TAT", 3, true), get_minimizer("TAT", 5, true) }; } std::vector result = index.minimizers(this->repetitive.begin(), this->repetitive.end()); EXPECT_EQ(result, correct) << "Did not find the correct minimizers"; } TYPED_TEST(MinimizerExtraction, WeirdSyncmers) { typedef TypeParam key_type; typedef MinimizerIndex index_type; typedef Kmer minimizer_type; // The string is the reverse complement of itself. The middle smers AAT and ATT // are the smallest ones using both key types. std::string weird = "CAATTG"; index_type index(5, 3, 0, true); std::vector correct; if(key_type::KEY_BITS == 128) { correct = { get_minimizer("AATTG", 1, false), get_minimizer("AATTG", 4, true) }; } else { correct = { get_minimizer("CAATT", 0, false), get_minimizer("CAATT", 5, true) }; } std::vector result = index.syncmers(weird); EXPECT_EQ(result, correct) << "Did not find the correct syncmers"; } TYPED_TEST(MinimizerExtraction, InvalidMinimizerCharacters) { typedef TypeParam key_type; typedef MinimizerIndex index_type; typedef Kmer minimizer_type; std::string weird = "CGAATAxAATACT"; index_type index(3, 2, 0); std::vector correct; if(key_type::KEY_BITS == 128) { correct = { get_minimizer("TCG", 2, true), get_minimizer("ATA", 3, false), get_minimizer("ATT", 4, true), get_minimizer("ATA", 8, false), get_minimizer("ATT", 9, true), get_minimizer("AGT", 12, true) }; } else { correct = { get_minimizer("TCG", 2, true), get_minimizer("AAT", 2, false), get_minimizer("TAT", 5, true), get_minimizer("AAT", 7, false), get_minimizer("TAT", 10, true), get_minimizer("ACT", 10, false) }; } std::vector result = index.minimizers(weird.begin(), weird.end()); EXPECT_EQ(result, correct) << "Did not find the correct minimizers"; } TYPED_TEST(MinimizerExtraction, InvalidSyncmerCharacters) { typedef TypeParam key_type; typedef MinimizerIndex index_type; typedef Kmer minimizer_type; std::string weird = "CGAATAxAATACT"; index_type index(5, 3, 0, true); std::vector correct; if(key_type::KEY_BITS == 128) { correct = { get_minimizer("ATTCG", 4, true), get_minimizer("ATACT", 8, false), get_minimizer("GTATT", 11, true) }; } else { correct = { get_minimizer("ATTCG", 4, true), get_minimizer("AATAC", 7, false), get_minimizer("AGTAT", 12, true) }; } std::vector result = index.minimizers(weird.begin(), weird.end()); EXPECT_EQ(result, correct) << "Did not find the correct syncmers"; } TYPED_TEST(MinimizerExtraction, BothOrientations) { typedef TypeParam key_type; typedef MinimizerIndex index_type; typedef Kmer minimizer_type; index_type index(3, 2, 0); std::vector forward_minimizers = index.minimizers(this->str.begin(), this->str.end()); std::vector reverse_minimizers = index.minimizers(this->rev.begin(), this->rev.end()); ASSERT_EQ(forward_minimizers.size(), reverse_minimizers.size()) << "Different number of minimizers in forward and reverse orientations"; for(size_t i = 0; i < forward_minimizers.size(); i++) { minimizer_type& f = forward_minimizers[i]; minimizer_type& r = reverse_minimizers[forward_minimizers.size() - 1 - i]; EXPECT_EQ(f.key, r.key) << "Wrong key for minimizer " << i; EXPECT_EQ(f.offset, this->str.length() - 1 - r.offset) << "Wrong offset for minimizer " << i; EXPECT_NE(f.is_reverse, r.is_reverse) << "Wrong orientation for minimizer " << i; } } //------------------------------------------------------------------------------ class HitsInSubgraphTest : public ::testing::Test { public: using value_type = KmerEncoding::value_type; using multi_value_type = KmerEncoding::multi_value_type; using result_type = std::vector; void check_results ( const MinimizerIndex& index, const std::unordered_set& subgraph, const owned_multi_value_type& hits, const result_type& expected_result, const std::string& test_case ) const { std::vector sorted_subgraph(subgraph.begin(), subgraph.end()); std::sort(sorted_subgraph.begin(), sorted_subgraph.end()); result_type result; auto report_hit = [&](value_type value) { result.push_back(create_value(value, index.payload_size())); }; multi_value_type h(hits.data(), hits.size() / index.value_size()); hits_in_subgraph(index, h, subgraph, report_hit); ASSERT_EQ(result.size(), expected_result.size()) << test_case << ": Incorrect number of results with the naive algorithm"; ASSERT_EQ(result, expected_result) << test_case << ": Incorrect results with the naive algorithm"; result.clear(); hits_in_subgraph(index, h, sorted_subgraph, report_hit); ASSERT_EQ(result.size(), expected_result.size()) << test_case << ": Incorrect number of results with exponential search"; ASSERT_EQ(result, expected_result) << test_case << ": Incorrect results with exponential search"; } std::tuple, owned_multi_value_type, result_type> create_test_case ( const MinimizerIndex& index, size_t universe_size, size_t begin, size_t end, double interval_prob, double outlier_prob, double hit_prob, size_t random_seed ) const { std::unordered_set subgraph; owned_multi_value_type hits; result_type expected_result; std::mt19937_64 rng(random_seed); for(size_t i = 0; i < universe_size; i++) { // Is node i in the subgraph? double random_value = rng() / static_cast(std::numeric_limits::max()); bool in_subgraph = false; if(i >= begin && i < end) { if(random_value <= interval_prob) { subgraph.insert(i); in_subgraph = true; } } else if(random_value <= outlier_prob) { subgraph.insert(i); in_subgraph = true; } // Are there hits for node i? random_value = rng() / static_cast(std::numeric_limits::max()); size_t hit_count = 0; while(random_value <= hit_prob) { pos_t pos = make_pos_t(i, hit_count & 1, hit_count & Position::OFF_MASK); owned_value_type value = create_value(pos, index.payload_size(), hash(pos)); append_value(hits, value, index.payload_size()); if(in_subgraph) { expected_result.push_back(value); } hit_count++; random_value = rng() / static_cast(std::numeric_limits::max()); } } return std::make_tuple(subgraph, hits, expected_result); } }; TEST_F(HitsInSubgraphTest, EmptySets) { for(size_t payload_size : payload_sizes) { std::string payload_msg = " with payload size " + std::to_string(payload_size); MinimizerIndex index(payload_size); std::unordered_set subgraph; owned_multi_value_type hits; result_type expected_result; std::string test_case = "Empty subgraph and hits" + payload_msg; this->check_results(index, subgraph, hits, expected_result, test_case); subgraph.insert(42); test_case = "Nonempty subgraph, empty hits" + payload_msg; this->check_results(index, subgraph, hits, expected_result, test_case); subgraph.clear(); pos_t pos = make_pos_t(42, false, 0); owned_value_type value = create_value(pos, index.payload_size(), hash(pos)); append_value(hits, value, index.payload_size()); test_case = "Empty subgraph, nonempty hits" + payload_msg; this->check_results(index, subgraph, hits, expected_result, test_case); } } TEST_F(HitsInSubgraphTest, SmallSets) { constexpr size_t UNIVERSE_SIZE = 1024; constexpr size_t INTERVALS = 10; constexpr size_t INTERVAL_START = 90; constexpr size_t INTERVAL_LENGTH = 30; constexpr double INTERVAL_PROB = 0.9; constexpr double OUTLIER_PROB = 0.01; constexpr double HIT_PROB = 0.1; for(size_t payload_size : payload_sizes) { MinimizerIndex index(payload_size); for(size_t i = 1; i <= INTERVALS; i++) { std::unordered_set subgraph; owned_multi_value_type hits; result_type expected_result; size_t start = i * INTERVAL_START; size_t random_seed = i * 0xDEADBEEF; std::tie(subgraph, hits, expected_result) = this->create_test_case ( index, UNIVERSE_SIZE, start, start + INTERVAL_LENGTH, INTERVAL_PROB, OUTLIER_PROB, HIT_PROB, random_seed ); std::string test_case = "Payload size " + std::to_string(payload_size) + ", set " + std::to_string(i); this->check_results(index, subgraph, hits, expected_result, test_case); } } } TEST_F(HitsInSubgraphTest, LargeSets) { constexpr size_t UNIVERSE_SIZE = 1048576; constexpr size_t INTERVALS = 3; constexpr size_t INTERVAL_START = 256 * 1024; constexpr size_t INTERVAL_LENGTH = 120; constexpr double INTERVAL_PROB = 0.9; constexpr double OUTLIER_PROB = 0.0001; constexpr double HIT_PROB = 0.05; for(size_t payload_size : payload_sizes) { MinimizerIndex index(payload_size); for(size_t i = 1; i <= INTERVALS; i++) { std::unordered_set subgraph; owned_multi_value_type hits; result_type expected_result; size_t start = i * INTERVAL_START; size_t random_seed = i * 0xDEADBEEF; std::tie(subgraph, hits, expected_result) = this->create_test_case ( index, UNIVERSE_SIZE, start, start + INTERVAL_LENGTH, INTERVAL_PROB, OUTLIER_PROB, HIT_PROB, random_seed ); std::string test_case = "Payload size " + std::to_string(payload_size) + ", set " + std::to_string(i); this->check_results(index, subgraph, hits, expected_result, test_case); } } } //------------------------------------------------------------------------------ } // namespace jltsiren-gbwtgraph-e27bc43/tests/test_naive_graph.cpp000066400000000000000000000203001524136466400231400ustar00rootroot00000000000000#include #include #include "shared.h" #include using namespace gbwtgraph; namespace { //------------------------------------------------------------------------------ class NaiveGraphTest : public ::testing::Test { public: std::vector correct_nodes; std::vector correct_translation; std::set correct_edges; void SetUp() override { this->correct_nodes = { { nid_t(1), "G" }, { nid_t(2), "A" }, { nid_t(3), "T" }, { nid_t(4), "GGG" }, { nid_t(5), "T" }, { nid_t(6), "A" }, { nid_t(7), "C" }, { nid_t(8), "A" }, { nid_t(9), "A" } }; this->correct_translation = { { "s1", { 1, 2 } }, { "s2", { 2, 3 } }, { "s3", { 3, 4 } }, { "s4", { 4, 6 } }, { "s5", { 6, 7 } }, { "s6", { 7, 8 } }, { "s7", { 8, 9 } }, { "s8", { 9, 10 } }, }; this->correct_edges = { { gbwt::Node::encode(1, false), gbwt::Node::encode(2, false) }, { gbwt::Node::encode(2, false), gbwt::Node::encode(4, false) }, { gbwt::Node::encode(4, false), gbwt::Node::encode(5, false) }, { gbwt::Node::encode(5, false), gbwt::Node::encode(6, false) }, { gbwt::Node::encode(6, false), gbwt::Node::encode(7, false) }, { gbwt::Node::encode(6, false), gbwt::Node::encode(8, false) }, { gbwt::Node::encode(7, false), gbwt::Node::encode(9, false) }, { gbwt::Node::encode(8, false), gbwt::Node::encode(9, false) } }; } void check_sequence_views(const NaiveGraph& graph, const std::vector& nodes) const { for(const node_type& node : nodes) { std::string_view view = graph.get_sequence_view(node.first); EXPECT_EQ(std::string(view), node.second) << "Invalid sequence view for node " << node.first; } } void check_translation(const NaiveGraph& graph, bool expect_translation) const { ASSERT_EQ(graph.uses_translation(), expect_translation) << "Segment translation is not used as expected"; if(graph.uses_translation()) { ASSERT_EQ(graph.get_segment_count(), correct_translation.size()) << "Invalid number of segments"; for(const translation_type& translation : correct_translation) { EXPECT_TRUE(graph.has_segment(translation.first)) << "Segment " << translation.first << " is missing from the sequence source"; EXPECT_TRUE(graph.has_node_or_segment(translation.first)) << "Node/segment " << translation.first << " is missing from the sequence source"; EXPECT_EQ(graph.translate(translation.first), translation.second) << "Invalid translation for " << translation.first; // Here we assume that segment names are not numeric in the tests. std::string node_str = std::to_string(translation.second.first); EXPECT_FALSE(graph.has_segment(node_str)) << "Node " << node_str << " of segment " << translation.first << " is identified as a segment"; EXPECT_FALSE(graph.has_node_or_segment(node_str)) << "Node " << node_str << " of segment " << translation.first << " is identified as a node/segment"; } { std::string missing = "missing_segment"; EXPECT_FALSE(graph.has_segment(missing)) << "Missing segment is reported present"; EXPECT_FALSE(graph.has_node_or_segment(missing)) << "Missing node/segment is reported present"; EXPECT_EQ(graph.translate(missing), NaiveGraph::no_translation()) << "A translation is returned for a missing segment"; } } else { for(nid_t id = graph.min_node_id(); id <= graph.max_node_id(); id++) { std::string segment = std::to_string(id); std::pair translation(id, id + 1); EXPECT_EQ(graph.translate(segment), translation) << "Invalid translation for " << segment; EXPECT_FALSE(graph.has_segment(segment)) << "Node " << segment << " is identified as a segment"; EXPECT_TRUE(graph.has_node_or_segment(segment)) << "Node " << segment << " is not identified as a node/segment"; } { std::string missing = "0"; EXPECT_EQ(graph.translate(missing), NaiveGraph::no_translation()) << "A translation is returned for a missing segment"; } } } void check_inverse_translation(const NaiveGraph& graph, const std::function)>& is_present) const { auto result = graph.invert_translation(is_present); ASSERT_EQ(result.first.size(), graph.get_segment_count()) << "Invalid number of segments in inverse translation"; ASSERT_EQ(result.second.size(), size_t(graph.max_node_id() + 1)) << "Invalid number of nodes in inverse translation"; ASSERT_EQ(result.first.size(), result.second.ones()) << "Inconsistent number of segments in inverse translation"; auto iter = result.second.one_begin(); nid_t start = iter->second; for(size_t i = 0; i < result.first.size(); i++) { ++iter; nid_t limit = iter->second; std::string segment = result.first.str(i); if(is_present(std::make_pair(start, limit))) { std::pair translation = graph.translate(segment); EXPECT_EQ(start, translation.first) << "Invalid start for segment " << segment; EXPECT_EQ(limit, translation.second) << "Invalid limit for segment " << segment; } else { EXPECT_TRUE(segment.empty()) << "Got a name for an unused segment from " << start << " to " << limit; } start = limit; } } }; TEST_F(NaiveGraphTest, EmptyGraph) { NaiveGraph graph; std::vector nodes; std::set edges; handle_graph_handles(graph, nodes, false); handle_graph_nodes(graph, nodes); handle_graph_edges(graph, nodes, edges); this->check_sequence_views(graph, nodes); this->check_translation(graph, false); this->check_inverse_translation(graph, [](std::pair) -> bool { return true; }); } TEST_F(NaiveGraphTest, AddNodes) { NaiveGraph graph = build_naive_graph(false); handle_graph_handles(graph, this->correct_nodes, false); handle_graph_nodes(graph, this->correct_nodes); handle_graph_edges(graph, this->correct_nodes, this->correct_edges); this->check_sequence_views(graph, this->correct_nodes); this->check_translation(graph, false); this->check_inverse_translation(graph, [](std::pair) -> bool { return true; }); } TEST_F(NaiveGraphTest, AddNodesLowerCase) { NaiveGraph graph = build_naive_graph(false, true); handle_graph_handles(graph, this->correct_nodes, false); handle_graph_nodes(graph, this->correct_nodes); handle_graph_edges(graph, this->correct_nodes, this->correct_edges); this->check_sequence_views(graph, this->correct_nodes); this->check_translation(graph, false); this->check_inverse_translation(graph, [](std::pair) -> bool { return true; }); } TEST_F(NaiveGraphTest, TranslateSegments) { NaiveGraph graph = build_naive_graph(true); handle_graph_handles(graph, this->correct_nodes, false); handle_graph_nodes(graph, this->correct_nodes); handle_graph_edges(graph, this->correct_nodes, this->correct_edges); this->check_sequence_views(graph, this->correct_nodes); this->check_translation(graph, true); this->check_inverse_translation(graph, [](std::pair) -> bool { return true; }); // Check an inverse translation without the names for s4 and s5. this->check_inverse_translation(graph, [](std::pair nodes) -> bool { return (nodes.first < 4 || nodes.first > 6); }); } TEST_F(NaiveGraphTest, TranslateSegmentsLowerCase) { NaiveGraph graph = build_naive_graph(true, true); handle_graph_handles(graph, this->correct_nodes, false); handle_graph_nodes(graph, this->correct_nodes); handle_graph_edges(graph, this->correct_nodes, this->correct_edges); this->check_sequence_views(graph, this->correct_nodes); this->check_translation(graph, true); this->check_inverse_translation(graph, [](std::pair) -> bool { return true; }); // Check an inverse translation without the names for s4 and s5. this->check_inverse_translation(graph, [](std::pair nodes) -> bool { return (nodes.first < 4 || nodes.first > 6); }); } //------------------------------------------------------------------------------ } // namespace jltsiren-gbwtgraph-e27bc43/tests/test_path_cover.cpp000066400000000000000000001137601524136466400230240ustar00rootroot00000000000000#include #include #include #include #include #include #include #include #include #include #include "shared.h" using namespace gbwtgraph; namespace { //------------------------------------------------------------------------------ // Included reference paths for each component from default.gfa. std::vector> generic_paths = { { { static_cast(gbwt::Node::encode(11, false)), static_cast(gbwt::Node::encode(12, false)), static_cast(gbwt::Node::encode(14, false)), static_cast(gbwt::Node::encode(15, false)), static_cast(gbwt::Node::encode(17, false)) } }, { { static_cast(gbwt::Node::encode(21, false)), static_cast(gbwt::Node::encode(22, false)), static_cast(gbwt::Node::encode(24, false)), static_cast(gbwt::Node::encode(25, false)) } } }; // Included reference paths for each component from components_ref.gfa. std::vector> reference_paths = { { { static_cast(gbwt::Node::encode(11, false)), static_cast(gbwt::Node::encode(13, false)), static_cast(gbwt::Node::encode(14, false)), static_cast(gbwt::Node::encode(15, false)), static_cast(gbwt::Node::encode(17, false)) } }, { { static_cast(gbwt::Node::encode(21, false)), static_cast(gbwt::Node::encode(23, false)), static_cast(gbwt::Node::encode(24, true)), static_cast(gbwt::Node::encode(22, true)), static_cast(gbwt::Node::encode(21, true)) } } }; // (n = 4, k = 3)-paths generated for components*.gfa / default.gfa. std::vector> correct_paths = { { { static_cast(gbwt::Node::encode(11, false)), static_cast(gbwt::Node::encode(12, false)), static_cast(gbwt::Node::encode(14, false)), static_cast(gbwt::Node::encode(15, false)), static_cast(gbwt::Node::encode(17, false)) }, { static_cast(gbwt::Node::encode(11, false)), static_cast(gbwt::Node::encode(12, false)), static_cast(gbwt::Node::encode(14, false)), static_cast(gbwt::Node::encode(16, false)), static_cast(gbwt::Node::encode(17, false)) }, { static_cast(gbwt::Node::encode(11, false)), static_cast(gbwt::Node::encode(13, false)), static_cast(gbwt::Node::encode(14, false)), static_cast(gbwt::Node::encode(15, false)), static_cast(gbwt::Node::encode(17, false)) }, { static_cast(gbwt::Node::encode(11, false)), static_cast(gbwt::Node::encode(13, false)), static_cast(gbwt::Node::encode(14, false)), static_cast(gbwt::Node::encode(16, false)), static_cast(gbwt::Node::encode(17, false)) } }, { { static_cast(gbwt::Node::encode(21, false)), static_cast(gbwt::Node::encode(22, false)), static_cast(gbwt::Node::encode(24, false)), static_cast(gbwt::Node::encode(25, false)) }, { static_cast(gbwt::Node::encode(21, false)), static_cast(gbwt::Node::encode(22, false)), static_cast(gbwt::Node::encode(24, false)), static_cast(gbwt::Node::encode(23, true)), static_cast(gbwt::Node::encode(21, true)) } } }; std::string sample_name(size_t i) { return "path_cover_" + std::to_string(i); } void check_metadata( const gbwt::GBWT& cover, const PathCoverParameters& params, const std::vector& ref_samples, const std::vector& contig_names, const std::vector>& included_paths) { bool include_paths = !(included_paths.empty()); ASSERT_TRUE(cover.hasMetadata()) << "Path cover GBWT contains no metadata"; gbwt::size_type expected_samples = params.num_paths + ref_samples.size(); EXPECT_EQ(cover.metadata.samples(), expected_samples) << "Wrong number of samples in the metadata"; ASSERT_TRUE(cover.metadata.hasSampleNames()) << "No sample names in the metadata"; for(size_t i = 0; i < ref_samples.size(); i++) { EXPECT_EQ(cover.metadata.sample(i), ref_samples[i]) << "Wrong reference sample name " << i << " in the metadata"; } for(size_t i = 0; i < params.num_paths; i++) { EXPECT_EQ(cover.metadata.sample(i + ref_samples.size()), sample_name(i)) << "Wrong path cover sample name " << i << " in the metadata"; } EXPECT_EQ(cover.metadata.contigs(), contig_names.size()) << "Wrong number of contigs in the metadata"; ASSERT_TRUE(cover.metadata.hasContigNames()) << "No contig names in the metadata"; for(size_t i = 0; i < contig_names.size(); i++) { EXPECT_EQ(cover.metadata.contig(i), contig_names[i]) << "Wrong contig name " << i << " in the metadata"; } EXPECT_EQ(cover.metadata.haplotypes(), params.num_paths + ref_samples.size()) << "Wrong number of haplotypes in the metadata"; gbwt::size_type expected_paths = contig_names.size() * params.num_paths; if(include_paths) { for(size_t i = 0; i < included_paths.size(); i++) { expected_paths += included_paths[i].size(); } } EXPECT_EQ(cover.metadata.paths(), expected_paths) << "Wrong number of path names in the metadata"; ASSERT_TRUE(cover.metadata.hasPathNames()) << "No path names in the metadata"; size_t path_offset = 0; for(size_t i = 0; i < contig_names.size(); i++) { if(include_paths) { for(size_t j = 0; j < included_paths[i].size(); j++) { size_t path_id = path_offset + j; gbwt::PathName path_name = cover.metadata.path(path_id); size_t phase = (ref_samples[j] == GENERIC_PATH_SAMPLE_NAME ? GBWTGraph::NO_PHASE : 0); gbwt::PathName correct = { gbwt::PathName::path_name_type(j), gbwt::PathName::path_name_type(i), gbwt::PathName::path_name_type(phase), 0 }; EXPECT_EQ(path_name, correct) << "Wrong included path name " << path_id << " in the metadata"; } path_offset += included_paths[i].size(); } for(size_t j = 0; j < params.num_paths; j++) { size_t path_id = path_offset + j; gbwt::PathName path_name = cover.metadata.path(path_id); gbwt::PathName correct = { gbwt::PathName::path_name_type(ref_samples.size() + j), gbwt::PathName::path_name_type(i), 0, 0 }; EXPECT_EQ(path_name, correct) << "Wrong path cover path name " << path_id << " in the metadata"; } path_offset += params.num_paths; } } //------------------------------------------------------------------------------ class PathStorageTest : public ::testing::Test { public: gbwt::GBWT index1; GBWTGraph graph1; gbwt::GBWT index2; GBWTGraph graph2; NaiveGraph source; size_t node_width; std::unordered_map paths1 { {"GRCh38#0#chr1", PathSense::REFERENCE}, {"GRCh37#0#chr1", PathSense::REFERENCE}, {"coolgene", PathSense::GENERIC}, {"sample1#1#chr1#0", PathSense::HAPLOTYPE}, {"sample1#2#chr1#0", PathSense::HAPLOTYPE}, {"CHM13#0#chr1#0", PathSense::HAPLOTYPE} }; std::unordered_map paths2 { {"GRCh38#1#chr1", PathSense::REFERENCE}, {"GRCh37#2#chr1", PathSense::REFERENCE}, {"coolergene", PathSense::GENERIC}, {"sample1#1#chr2#0", PathSense::HAPLOTYPE}, {"sample1#2#chr2#0", PathSense::HAPLOTYPE}, {"CHM13v2#0#chr1#0", PathSense::HAPLOTYPE} }; std::unordered_set all_senses {PathSense::GENERIC, PathSense::REFERENCE, PathSense::HAPLOTYPE}; std::unordered_set named_senses {PathSense::GENERIC, PathSense::REFERENCE}; PathStorageTest() { } void SetUp() override { // Need to parse PanSN GFAParsingParameters parameters; parameters.path_name_formats.emplace_front( GFAParsingParameters::PAN_SN_REGEX, GFAParsingParameters::PAN_SN_FIELDS, GFAParsingParameters::PAN_SN_SENSE ); auto gfa_parse1 = gfa_to_gbwt("gfas/example_reference.gfa", parameters); this->index1 = *(gfa_parse1.first); this->source = *gfa_parse1.second; this->graph1 = GBWTGraph(this->index1, this->source); this->node_width = gbwt::bit_length(this->index1.sigma() - 1); // Grab another graph with different paths but (we assume) the same node ID space. auto gfa_parse2 = gfa_to_gbwt("gfas/example_more_reference.gfa", parameters); this->index2 = *(gfa_parse2.first); this->graph2 = GBWTGraph(this->index2, this->source); } }; // Make sure the right senses of the right paths are present. void check_stored_paths( const GBWTGraph& constructed, const std::vector*>& path_lists, const std::unordered_set& wanted_senses, const std::unordered_set& unwanted_names ) { for(auto path_list : path_lists) { for(auto& kv : *path_list) { if(wanted_senses.count(kv.second) && !unwanted_names.count(kv.first)) { // Should have been copied over EXPECT_TRUE(constructed.has_path(kv.first)) << "Wanted path not copied: " << kv.first; if(constructed.has_path(kv.first)) { EXPECT_EQ(constructed.get_sense(constructed.get_path_handle(kv.first)), kv.second) << "Wrong path sense: " << kv.first; } } else { // Should not have been copied over EXPECT_FALSE(constructed.has_path(kv.first)) << "Unwanted path copied: " << kv.first; } } } } TEST_F(PathStorageTest, StoreNamedPathsOneGraph) { gbwt::GBWTBuilder builder(this->node_width); builder.index.addMetadata(); store_named_paths(builder, this->graph1, nullptr); builder.finish(); // Static-ify the GBWT so it doesn't happen in a temporary that GBWTGraph will keep a pointer to. gbwt::GBWT built(builder.index); GBWTGraph constructed(built, this->source); ASSERT_TRUE(constructed.index->hasMetadata()) << "Index missing metadata"; ASSERT_TRUE(constructed.index->metadata.hasPathNames()) << "Index missing path names"; ASSERT_TRUE(constructed.index->metadata.hasSampleNames()) << "Index missing sample names"; ASSERT_TRUE(constructed.index->metadata.hasContigNames()) << "Index missing contig names"; EXPECT_EQ(constructed.index->metadata.sample_names.size(), (gbwt::size_type) 3) << "Index has wrong number of samples"; EXPECT_EQ(constructed.index->metadata.contig_names.size(), (gbwt::size_type) 2) << "Index has wrong number of contigs"; EXPECT_LT(constructed.index->metadata.sample(GENERIC_PATH_SAMPLE_NAME), constructed.index->metadata.sample_names.size()) << "Index is missing generic path sample"; EXPECT_LT(constructed.index->metadata.contig("chr1"), constructed.index->metadata.contig_names.size()) << "Index is missing chr1 contig"; check_stored_paths(constructed, {&this->paths1}, this->named_senses, {}); } TEST_F(PathStorageTest, StoreNamedPathsTwoGraphs) { gbwt::GBWTBuilder builder(this->node_width); builder.index.addMetadata(); store_named_paths(builder, this->graph1, nullptr); store_named_paths(builder, this->graph2, nullptr); builder.finish(); // Static-ify the GBWT so it doesn't happen in a temporary that GBWTGraph will keep a pointer to. gbwt::GBWT built(builder.index); GBWTGraph constructed(built, this->source); EXPECT_EQ(constructed.index->metadata.sample_names.size(), (gbwt::size_type) 3) << "Index has wrong number of samples"; EXPECT_EQ(constructed.index->metadata.contig_names.size(), (gbwt::size_type) 3) << "Index has wrong number of contigs"; EXPECT_LT(constructed.index->metadata.contig("coolergene"), constructed.index->metadata.contig_names.size()) << "Index is missing coolergene contig"; check_stored_paths(constructed, {&this->paths1, &this->paths2}, this->named_senses, {}); } TEST_F(PathStorageTest, StoreAllPathsTwoGraphs) { gbwt::GBWTBuilder builder(this->node_width); builder.index.addMetadata(); store_paths(builder, this->graph1, this->all_senses, nullptr); store_paths(builder, this->graph2, this->all_senses, nullptr); builder.finish(); // Static-ify the GBWT so it doesn't happen in a temporary that GBWTGraph will keep a pointer to. gbwt::GBWT built(builder.index); GBWTGraph constructed(built, this->source); EXPECT_EQ(constructed.index->metadata.sample_names.size(), (gbwt::size_type) 6) << "Index has wrong number of samples"; EXPECT_EQ(constructed.index->metadata.contig_names.size(), (gbwt::size_type) 4) << "Index has wrong number of contigs"; EXPECT_LT(constructed.index->metadata.sample("CHM13v2"), constructed.index->metadata.sample_names.size()) << "Index is missing CHM13v2 sample"; EXPECT_LT(constructed.index->metadata.contig("chr2"), constructed.index->metadata.contig_names.size()) << "Index is missing chr2 contig"; check_stored_paths(constructed, {&this->paths1, &this->paths2}, this->all_senses, {}); } TEST_F(PathStorageTest, StoreAllPathsExceptTwoGraphs) { std::unordered_set unwanted_names {"coolgene", "GRCh38#0#chr1"}; std::function filter1 = [&](const path_handle_t& path_handle) { return !unwanted_names.count(this->graph1.get_path_name(path_handle)); }; gbwt::GBWTBuilder builder(this->node_width); builder.index.addMetadata(); store_paths(builder, this->graph1, this->all_senses, &filter1); store_paths(builder, this->graph2, this->all_senses, nullptr); builder.finish(); // Static-ify the GBWT so it doesn't happen in a temporary that GBWTGraph will keep a pointer to. gbwt::GBWT built(builder.index); GBWTGraph constructed(built, this->source); check_stored_paths(constructed, {&this->paths1, &this->paths2}, this->all_senses, unwanted_names); } //------------------------------------------------------------------------------ class PathCoverTest : public ::testing::Test { public: size_t components; PathCoverTest() { } void SetUp() override { this->components = correct_paths.size(); } void check_paths( const gbwt::GBWT& cover, const PathCoverParameters& params, const std::vector>& included_paths) { bool include_paths = !(included_paths.empty()); gbwt::size_type expected_sequences = this->components * params.num_paths * 2; if(include_paths) { for(size_t i = 0; i < included_paths.size(); i++) { expected_sequences += 2 * included_paths[i].size(); } } ASSERT_EQ(cover.sequences(), expected_sequences) << "Wrong number of sequences in the path cover GBWT"; // Check that the included paths are there. size_t path_offset = 0; if(include_paths) { for(size_t i = 0; i < this->components; i++) { for(size_t j = 0; j < included_paths[i].size(); j++) { gbwt::size_type seq_id = gbwt::Path::encode(path_offset + j, false); gbwt::vector_type path = cover.extract(seq_id); EXPECT_EQ(path, included_paths[i][j]) << "Wrong included path " << j << " in component " << i; } path_offset += included_paths[i].size() + params.num_paths; } } // Check the paths in the path cover. We insert the smaller of a path and // its reverse complement to handle paths that flip the orientation. std::vector> result(this->components); path_offset = 0; for(size_t i = 0; i < this->components; i++) { if(include_paths) { path_offset += included_paths[i].size(); } for(size_t j = 0; j < params.num_paths; j++) { size_t seq_id = gbwt::Path::encode(path_offset + j, false); gbwt::vector_type forward = cover.extract(seq_id), reverse; gbwt::reversePath(forward, reverse); result[i].insert(std::min(forward, reverse)); } path_offset += params.num_paths; } for(size_t i = 0; i < this->components; i++) { ASSERT_EQ(result[i].size(), correct_paths[i].size()) << "Wrong number of distinct paths for component " << i; auto result_iter = result[i].begin(); auto correct_iter = correct_paths[i].begin(); while(result_iter != result[i].end()) { EXPECT_EQ(*result_iter, *correct_iter) << "Wrong path in component " << i; ++result_iter; ++correct_iter; } } } }; TEST_F(PathCoverTest, SingleThreaded) { auto gfa_parse = gfa_to_gbwt("gfas/components.gfa"); const gbwt::GBWT& index = *(gfa_parse.first); GBWTGraph graph(index, *(gfa_parse.second)); PathCoverParameters params; params.num_paths = 4; params.context = 3; gbwt::GBWT cover = path_cover_gbwt(graph, params); this->check_paths(cover, params, {}); check_metadata(cover, params, {}, { "A1", "B1" }, {}); } TEST_F(PathCoverTest, MultiThreaded) { auto gfa_parse = gfa_to_gbwt("gfas/components.gfa"); const gbwt::GBWT& index = *(gfa_parse.first); GBWTGraph graph(index, *(gfa_parse.second)); PathCoverParameters params; params.num_paths = 4; params.context = 3; params.parallel_jobs = 2; gbwt::GBWT cover = path_cover_gbwt(graph, params); this->check_paths(cover, params, {}); check_metadata(cover, params, {}, { "A1", "B1" }, {}); } TEST_F(PathCoverTest, IncludeGenericSingleThreaded) { auto gfa_parse = gfa_to_gbwt("gfas/default.gfa"); const gbwt::GBWT& index = *(gfa_parse.first); GBWTGraph graph(index, *(gfa_parse.second)); PathCoverParameters params; params.num_paths = 4; params.context = 3; gbwt::GBWT cover = path_cover_gbwt(graph, params, true); this->check_paths(cover, params, generic_paths); check_metadata(cover, params, { GENERIC_PATH_SAMPLE_NAME }, { "A", "B" }, generic_paths); } TEST_F(PathCoverTest, IncludeGenericMultiThreaded) { auto gfa_parse = gfa_to_gbwt("gfas/default.gfa"); const gbwt::GBWT& index = *(gfa_parse.first); GBWTGraph graph(index, *(gfa_parse.second)); PathCoverParameters params; params.num_paths = 4; params.context = 3; params.parallel_jobs = 2; gbwt::GBWT cover = path_cover_gbwt(graph, params, true); this->check_paths(cover, params, generic_paths); check_metadata(cover, params, { GENERIC_PATH_SAMPLE_NAME }, { "A", "B" }, generic_paths); } TEST_F(PathCoverTest, IncludeReferenceSingleThreaded) { auto gfa_parse = gfa_to_gbwt("gfas/components_ref.gfa"); const gbwt::GBWT& index = *(gfa_parse.first); GBWTGraph graph(index, *(gfa_parse.second)); PathCoverParameters params; params.num_paths = 4; params.context = 3; gbwt::GBWT cover = path_cover_gbwt(graph, params, true); this->check_paths(cover, params, reference_paths); check_metadata(cover, params, { "ref" }, { "A", "B" }, reference_paths); } TEST_F(PathCoverTest, IncludeReferenceMultiThreaded) { auto gfa_parse = gfa_to_gbwt("gfas/components_ref.gfa"); const gbwt::GBWT& index = *(gfa_parse.first); GBWTGraph graph(index, *(gfa_parse.second)); PathCoverParameters params; params.num_paths = 4; params.context = 3; params.parallel_jobs = 2; gbwt::GBWT cover = path_cover_gbwt(graph, params, true); this->check_paths(cover, params, reference_paths); check_metadata(cover, params, { "ref" }, { "A", "B" }, reference_paths); } //------------------------------------------------------------------------------ class LocalHaplotypesTest : public ::testing::Test { public: size_t components; LocalHaplotypesTest() { } void SetUp() override { this->components = correct_paths.size(); } struct SearchState { gbwt::SearchState left, right; size_t depth; SearchState successor(const gbwt::GBWT& left_index, const gbwt::GBWT& right_index, gbwt::node_type to) { return { left_index.extend(this->left, to), right_index.extend(this->right, to), static_cast(this->depth + 1) }; } bool ok() const { return (this->left.empty() == this->right.empty()); } gbwt::node_type node() const { return this->left.node; } }; // Does the second index have the same depth-k extensions as the first? bool same_extensions(const gbwt::GBWT& baseline, const gbwt::GBWT& candidate, size_t k) { for(gbwt::node_type node = baseline.firstNode(); node < baseline.sigma(); node++) { std::stack states; states.push({ baseline.find(node), candidate.find(node), static_cast(1)}); while(!(states.empty())) { SearchState state = states.top(); states.pop(); if(!(state.ok())) { return false; } if(state.depth >= k) { continue; } for(auto edge : baseline.edges(state.node())) { if(edge.first == gbwt::ENDMARKER) { continue; } states.push(state.successor(baseline, candidate, edge.first)); } } } return true; } void check_cover( const gbwt::GBWT& index, const gbwt::GBWT& cover, const PathCoverParameters& params, const std::vector>& included_paths) { gbwt::size_type expected_sequences = this->components * params.num_paths * 2; for(size_t i = 0; i < included_paths.size(); i++) { expected_sequences += 2 * included_paths[i].size(); } ASSERT_EQ(cover.sequences(), expected_sequences) << "Wrong number of sequences in the local haplotype GBWT"; ASSERT_EQ(cover.sigma(), index.sigma()) << "Wrong alphabet size in the local haplotype GBWT"; ASSERT_EQ(cover.effective(), index.effective()) << "Wrong effective alphabet size in the local haplotype GBWT"; bool all_correct_subpaths = this->same_extensions(index, cover, params.context); EXPECT_TRUE(all_correct_subpaths) << "Missing " << params.context << "-subpaths in the local haplotype GBWT"; bool no_extra_subpaths = this->same_extensions(cover, index, params.context); EXPECT_TRUE(no_extra_subpaths) << "Additional " << params.context << "-subpaths in the local haplotype GBWT"; } }; TEST_F(LocalHaplotypesTest, SingleThreaded) { auto gfa_parse = gfa_to_gbwt("gfas/components.gfa"); const gbwt::GBWT& index = *(gfa_parse.first); GBWTGraph graph(index, *(gfa_parse.second)); PathCoverParameters params; params.num_paths = 4; params.context = 3; gbwt::GBWT cover = local_haplotypes(graph, index, params); this->check_cover(index, cover, params, {}); check_metadata(cover, params, {}, { "A1", "B1" }, {}); } TEST_F(LocalHaplotypesTest, MultiThreaded) { auto gfa_parse = gfa_to_gbwt("gfas/components.gfa"); const gbwt::GBWT& index = *(gfa_parse.first); GBWTGraph graph(index, *(gfa_parse.second)); PathCoverParameters params; params.num_paths = 4; params.context = 3; params.parallel_jobs = 2; gbwt::GBWT cover = local_haplotypes(graph, index, params); this->check_cover(index, cover, params, {}); check_metadata(cover, params, {}, { "A1", "B1" }, {}); } TEST_F(LocalHaplotypesTest, IncludeGenericSingleThreaded) { auto gfa_parse = gfa_to_gbwt("gfas/default.gfa"); const gbwt::GBWT& index = *(gfa_parse.first); GBWTGraph graph(index, *(gfa_parse.second)); PathCoverParameters params; params.num_paths = 4; params.context = 3; gbwt::GBWT cover = local_haplotypes(graph, index, params, true); this->check_cover(index, cover, params, generic_paths); check_metadata(cover, params, { GENERIC_PATH_SAMPLE_NAME }, { "A", "B" }, generic_paths); } TEST_F(LocalHaplotypesTest, IncludeGenericMultiThreaded) { auto gfa_parse = gfa_to_gbwt("gfas/default.gfa"); const gbwt::GBWT& index = *(gfa_parse.first); GBWTGraph graph(index, *(gfa_parse.second)); PathCoverParameters params; params.num_paths = 4; params.context = 3; params.parallel_jobs = 2; gbwt::GBWT cover = local_haplotypes(graph, index, params, true); this->check_cover(index, cover, params, generic_paths); check_metadata(cover, params, { GENERIC_PATH_SAMPLE_NAME }, { "A", "B" }, generic_paths); } TEST_F(LocalHaplotypesTest, IncludeReferenceSingleThreaded) { auto gfa_parse = gfa_to_gbwt("gfas/components_ref.gfa"); const gbwt::GBWT& index = *(gfa_parse.first); GBWTGraph graph(index, *(gfa_parse.second)); PathCoverParameters params; params.num_paths = 4; params.context = 3; gbwt::GBWT cover = local_haplotypes(graph, index, params, true); this->check_cover(index, cover, params, reference_paths); check_metadata(cover, params, { "ref" }, { "A", "B" }, reference_paths); } TEST_F(LocalHaplotypesTest, IncludeReferenceMultiThreaded) { auto gfa_parse = gfa_to_gbwt("gfas/components_ref.gfa"); const gbwt::GBWT& index = *(gfa_parse.first); GBWTGraph graph(index, *(gfa_parse.second)); PathCoverParameters params; params.num_paths = 4; params.context = 3; params.parallel_jobs = 2; gbwt::GBWT cover = local_haplotypes(graph, index, params, true); this->check_cover(index, cover, params, reference_paths); check_metadata(cover, params, { "ref" }, { "A", "B" }, reference_paths); } TEST_F(LocalHaplotypesTest, Frequencies) { auto gfa_parse = gfa_to_gbwt("gfas/components.gfa"); const gbwt::GBWT& index = *(gfa_parse.first); GBWTGraph graph(index, *(gfa_parse.second)); PathCoverParameters params; params.num_paths = 4; params.context = 3; std::vector frequent_path { gbwt::Node::encode(21, false), gbwt::Node::encode(22, false), gbwt::Node::encode(24, false) }; std::vector rare_path { gbwt::Node::encode(22, false), gbwt::Node::encode(24, false), gbwt::Node::encode(25, false) }; gbwt::GBWT cover = local_haplotypes(graph, index, params); gbwt::SearchState frequent_state = cover.find(frequent_path.begin(), frequent_path.end()); gbwt::SearchState rare_state = cover.find(rare_path.begin(), rare_path.end()); EXPECT_GE(frequent_state.size(), rare_state.size()) << "Local haplotype frequencies do not reflect true frequencies"; } TEST_F(LocalHaplotypesTest, RevertToPathCover) { auto gfa_parse = gfa_to_gbwt("gfas/components.gfa"); const gbwt::GBWT& index = *(gfa_parse.first); GBWTGraph graph(index, *(gfa_parse.second)); PathCoverParameters params; params.num_paths = 4; params.context = 3; gbwt::size_type expected_sequences = this->components * params.num_paths * 2; gbwt::GBWT haplotype_cover = local_haplotypes(graph, index, params); ASSERT_EQ(haplotype_cover.sequences(), expected_sequences) << "Wrong number of sequences in the local haplotype GBWT"; gbwt::GBWT path_cover = path_cover_gbwt(graph, params); ASSERT_EQ(path_cover.sequences(), expected_sequences) << "Wrong number of sequences in the path cover GBWT"; auto first_only = gfa_to_gbwt("gfas/components_first.gfa"); gbwt::GBWT mixed_cover = local_haplotypes(graph, *(first_only.first), params); ASSERT_EQ(mixed_cover.sequences(), expected_sequences) << "Wrong number of sequences in the mixed cover GBWT"; // For the first component, we should have the same paths as with local haplotypes. for(size_t i = 0; i < params.num_paths; i++) { gbwt::vector_type path = mixed_cover.extract(gbwt::Path::encode(i, false)); gbwt::vector_type correct_path = haplotype_cover.extract(gbwt::Path::encode(i, false)); EXPECT_EQ(path, correct_path) << "Wrong path " << i << " in the first component"; } // For the second component, we should have the same paths as with path cover. for(size_t i = 0; i < params.num_paths; i++) { gbwt::vector_type path = mixed_cover.extract(gbwt::Path::encode(params.num_paths + i, false)); gbwt::vector_type correct_path = path_cover.extract(gbwt::Path::encode(params.num_paths + i, false)); EXPECT_EQ(path, correct_path) << "Wrong path " << i << " in the second component"; } } //------------------------------------------------------------------------------ class AugmentTest : public ::testing::Test { public: gbwt::GBWT index; GBWTGraph graph; std::vector> components; size_t samples; AugmentTest() { } void SetUp() override { auto gfa_parse = gfa_to_gbwt("gfas/components.gfa"); this->index = *(gfa_parse.first); this->graph = GBWTGraph(this->index, *(gfa_parse.second)); this->components = weakly_connected_components(this->graph); this->samples = 2; } gbwt::DynamicGBWT create_gbwt(const std::set& components_present) const { size_t node_width = gbwt::bit_length(this->index.sigma() - 1); size_t total_length = this->index.size(); gbwt::GBWTBuilder builder(node_width, total_length); builder.index.addMetadata(); std::vector samples_per_component(this->components.size(), 0); std::vector component_to_rank(this->components.size(), 0); for(size_t i = 0, rank = 0; i < this->components.size(); i++) { component_to_rank[i] = rank; if(components_present.find(i) != components_present.end()) { rank++; } } for(gbwt::size_type i = 0; i < this->index.sequences(); i += 2) { gbwt::vector_type sequence = this->index.extract(i); nid_t start_node = gbwt::Node::id(sequence.front()); size_t component = 0; while(component < this->components.size()) { const std::vector& curr = this->components[component]; if(std::find(curr.begin(), curr.end(), start_node) != curr.end()) { break; } component++; } if(components_present.find(component) != components_present.end()) { builder.insert(sequence, true); builder.index.metadata.addPath(samples_per_component[component], component_to_rank[component], 0, 0); samples_per_component[component]++; } } builder.finish(); if(!(components_present.empty())) { std::vector sample_names, contig_names; for(size_t i = 0; i < this->samples; i++) { sample_names.push_back("sample_" + std::to_string(i)); } builder.index.metadata.setSamples(sample_names); for(size_t contig : components_present) { contig_names.push_back("contig_" + std::to_string(contig)); } builder.index.metadata.setContigs(contig_names); builder.index.metadata.setHaplotypes(this->samples); } return builder.index; } std::string set_name(const std::set& components) const { std::stringstream ss; ss << "{"; for(size_t i : components) { ss << " " << i; } ss << " }"; return ss.str(); } }; TEST_F(AugmentTest, CorrectPaths) { PathCoverParameters params; params.num_paths = 4; params.context = 3; std::vector> component_sets { { }, { 0 }, { 1 }, { 0, 1 } }; for(const std::set& components_present : component_sets) { gbwt::DynamicGBWT original = this->create_gbwt(components_present); gbwt::DynamicGBWT augmented = original; std::string name = this->set_name(components_present); // Augment the GBWT. size_t expected_components = this->components.size() - components_present.size(); size_t covered_components = augment_gbwt(this->graph, augmented, params); ASSERT_EQ(covered_components, expected_components) << "Wrong number of covered components for components " << name; size_t expected_sequences = original.sequences() + 2 * expected_components * params.num_paths; ASSERT_EQ(augmented.sequences(), expected_sequences) << "Wrong number of sequences for components " << name; // Check original paths. size_t path_id = 0; while(path_id < original.metadata.paths()) { size_t i = gbwt::Path::encode(path_id, false); bool same_paths = (original.extract(i) == augmented.extract(i)); EXPECT_TRUE(same_paths) << "Wrong original path " << path_id << " for components " << name; path_id++; } // Check generated paths. for(size_t component = 0; component < this->components.size(); component++) { if(components_present.find(component) != components_present.end()) { continue; } for(size_t i = 0; i < params.num_paths; i++) { gbwt::vector_type forward = augmented.extract(gbwt::Path::encode(path_id, false)), reverse; gbwt::reversePath(forward, reverse); gbwt::vector_type path = std::min(forward, reverse); bool correct_path = (correct_paths[component].find(path) != correct_paths[component].end()); EXPECT_TRUE(correct_path) << "Wrong augmented path " << i << " in component " << component << " for components " << name; path_id++; } } } } TEST_F(AugmentTest, PathNames) { PathCoverParameters params; params.num_paths = 4; params.context = 3; std::vector> component_sets { { }, { 0 }, { 1 }, { 0, 1 } }; for(const std::set& components_present : component_sets) { gbwt::DynamicGBWT original = this->create_gbwt(components_present); gbwt::DynamicGBWT augmented = original; std::string name = this->set_name(components_present); // Augment the GBWT. size_t expected_components = this->components.size() - components_present.size(); augment_gbwt(this->graph, augmented, params); // Check metadata. ASSERT_TRUE(augmented.hasMetadata()) << "No metadata for components " << name; ASSERT_TRUE(augmented.metadata.hasPathNames()) << "No path names for components " << name; size_t expected_paths = original.metadata.paths() + expected_components * params.num_paths; ASSERT_EQ(augmented.metadata.paths(), expected_paths) << "Wrong number of path names for components " << name; // Check original paths. size_t path_id = 0; while(path_id < original.metadata.paths()) { bool same_names = (original.metadata.path(path_id) == augmented.metadata.path(path_id)); EXPECT_TRUE(same_names) << "Wrong original path name " << path_id << " for components " << name; path_id++; } // Check generated paths. size_t rank = 0; for(size_t component = 0; component < this->components.size(); component++) { if(components_present.find(component) != components_present.end()) { continue; } for(size_t i = 0; i < params.num_paths; i++) { gbwt::PathName path_name = augmented.metadata.path(path_id); EXPECT_EQ(path_name.sample, original.metadata.samples() + i) << "Wrong sample for augmented path " << i << " in component " << component << " for components " << name; EXPECT_EQ(path_name.contig, components_present.size() + rank) << "Wrong contig for augmented path " << i << " in component " << component << " for components " << name; path_id++; } rank++; } } } TEST_F(AugmentTest, SamplesAndContigs) { PathCoverParameters params; params.num_paths = 4; params.context = 3; std::vector> component_sets { { }, { 0 }, { 1 }, { 0, 1 } }; for(const std::set& components_present : component_sets) { gbwt::DynamicGBWT original = this->create_gbwt(components_present); gbwt::DynamicGBWT augmented = original; std::string name = this->set_name(components_present); // Augment the GBWT. size_t expected_components = this->components.size() - components_present.size(); augment_gbwt(this->graph, augmented, params); // Check metadata. ASSERT_TRUE(augmented.hasMetadata()) << "No metadata for components " << name; ASSERT_TRUE(augmented.metadata.hasSampleNames()) << "No sample names for components " << name; size_t expected_samples = original.metadata.samples() + (expected_components == 0 ? 0 : params.num_paths); ASSERT_EQ(augmented.metadata.samples(), expected_samples) << "Wrong number of samples for components " << name; ASSERT_TRUE(augmented.metadata.hasContigNames()) << "No contig names for components " << name; ASSERT_EQ(augmented.metadata.contigs(), this->components.size()) << "Wrong number of contigs for components " << name; // Check sample names. size_t sample_id = 0; while(sample_id < original.metadata.samples()) { std::string expected_name = original.metadata.sample(sample_id); EXPECT_EQ(augmented.metadata.sample(sample_id), expected_name) << "Wrong name for original sample " << sample_id << " for components " << name; sample_id++; } if(expected_components > 0) { for(size_t i = 0; i < params.num_paths; i++) { std::string expected_name = "path_cover_" + std::to_string(i); EXPECT_EQ(augmented.metadata.sample(sample_id), expected_name) << "Wrong name for augmented sample " << i << " for components " << name; sample_id++; } } // Check contig names. size_t contig_id = 0; for(size_t i = 0; i < this->components.size(); i++) { if(components_present.find(i) != components_present.end()) { std::string expected_name = "contig_" + std::to_string(i); EXPECT_EQ(augmented.metadata.contig(contig_id), expected_name) << "Wrong name for original contig " << i << " for components " << name; contig_id++; } } if(expected_components > 0) { for(size_t i = 0; i < this->components.size(); i++) { if(components_present.find(i) == components_present.end()) { std::string expected_name = "component_" + std::to_string(i); EXPECT_EQ(augmented.metadata.contig(contig_id), expected_name) << "Wrong name for augmented contig " << i << " for components " << name; contig_id++; } } } } } //------------------------------------------------------------------------------ } // namespace jltsiren-gbwtgraph-e27bc43/tests/test_subgraph.cpp000066400000000000000000000576221524136466400225110ustar00rootroot00000000000000#include #include #include #include #include "shared.h" using namespace gbwtgraph; namespace { //------------------------------------------------------------------------------ GBZ build_gbz(const std::string& graph_name) { auto parse = gfa_to_gbwt(graph_name); return GBZ(parse.first, parse.second); } //------------------------------------------------------------------------------ class PathIndexTest : public ::testing::Test { public: std::vector graphs; std::vector> indexed_path_lengths; PathIndexTest() { this->graphs = { "gfas/default.gfa", "gfas/components_ref.gfa" }; this->indexed_path_lengths = { { 5, 4 }, { 5, 5 } }; } }; TEST_F(PathIndexTest, BuildIndex) { for(size_t i = 0; i < this->graphs.size(); i++) { GBZ gbz = build_gbz(this->graphs[i]); PathIndex index(gbz); ASSERT_EQ(index.paths(), this->indexed_path_lengths[i].size()) << "Invalid number of indexed paths for graph " << this->graphs[i]; for(size_t j = 0; j < this->indexed_path_lengths[i].size(); j++) { path_handle_t handle = handlegraph::as_path_handle(j); EXPECT_EQ(index.path_length(handle), this->indexed_path_lengths[i][j]) << "Invalid length of path " << j << " for graph " << this->graphs[i]; } } } TEST_F(PathIndexTest, IndexedPaths) { for(auto& graph_name : this->graphs) { GBZ gbz = build_gbz(graph_name); for(size_t interval = 0; interval < 10; interval++) { PathIndex index(gbz, interval); for(size_t path_id = 0; path_id < index.paths(); path_id++) { size_t length = 0; path_handle_t handle = handlegraph::as_path_handle(path_id); std::vector> reference_positions = sample_path_positions(gbz, handle, interval, &length); size_t pos_offset = 0; for(size_t seq_offset = 0; seq_offset <= length; seq_offset++) { auto position = index.sampled_position(handle, seq_offset); if(pos_offset + 1 < reference_positions.size() && reference_positions[pos_offset + 1].first <= seq_offset) { pos_offset++; } ASSERT_EQ(position, reference_positions[pos_offset]) << "Invalid sampled position for path " << path_id << ", offset " << seq_offset << " in graph " << graph_name; } } } } } TEST_F(PathIndexTest, NonIndexedPaths) { for(auto& graph_name : this->graphs) { GBZ gbz = build_gbz(graph_name); for(size_t interval = 0; interval < 10; interval++) { PathIndex index(gbz, interval); for(size_t path_id = index.paths(); path_id < gbz.index.metadata.paths(); path_id++) { path_handle_t handle = handlegraph::as_path_handle(path_id); for(size_t seq_offset = 0; seq_offset <= 10; seq_offset++) { auto position = index.sampled_position(handle, seq_offset); ASSERT_EQ(position.second, gbwt::invalid_edge()) << "Found a sampled position for non-indexed path " << path_id << ", offset " << seq_offset << " in graph " << graph_name; } } } } } //------------------------------------------------------------------------------ class AlignDivergingTest : public ::testing::Test { public: gbwt::GBWT index; GBWTGraph graph; void SetUp() override { std::vector paths { forward_path({ 1, 2, 3, 4, 5, 6 }) }; this->index = build_gbwt(paths); std::vector nodes { { 1, "A" }, { 2, "C" }, { 3, "AA" }, { 4, "CC" }, { 5, "ACA" }, { 6, "CAC" } }; NaiveGraph source; for(const auto& node : nodes) { source.create_node(node.first, node.second); } this->graph = GBWTGraph(this->index, source); } static gbwt::vector_type forward_path(const std::vector& nodes) { gbwt::vector_type result; for(nid_t node : nodes) { result.push_back(gbwt::Node::encode(node, false)); } return result; } void check_edits( const gbwt::vector_type& path, const gbwt::vector_type& reference, const std::vector>& truth, const std::string& name ) const { std::vector> edits; align_diverging( this->graph, get_subpath(path, 0, path.size()), get_subpath(reference, 0, reference.size()), edits ); ASSERT_EQ(edits.size(), truth.size()) << "Invalid number of edits for " << name; for(size_t i = 0; i < edits.size(); i++) { EXPECT_EQ(edits[i], truth[i]) << "Invalid edit " << i << " for " << name; } } }; TEST_F(AlignDivergingTest, SpecialCases) { gbwt::vector_type empty; gbwt::vector_type non_empty = forward_path({ 5, 6 }); // (empty, empty) { std::vector> truth; this->check_edits(empty, empty, truth, "empty paths"); } // (empty, non-empty) { std::vector> truth { { 'D', 6 } }; this->check_edits(empty, non_empty, truth, "empty vs. non-empty paths"); } // (non-empty, empty) { std::vector> truth { { 'I', 6 } }; this->check_edits(non_empty, empty, truth, "non-empty vs. empty paths"); } // (non-empty, non-empty) { std::vector> truth { { 'M', 6 } }; this->check_edits(non_empty, non_empty, truth, "identical paths"); } // Identical bases, different paths. { gbwt::vector_type path = forward_path({ 1, 5, 3 }); // AACAAA gbwt::vector_type reference = forward_path({ 3, 2, 1, 3 }); // AACAAA std::vector> truth { { 'M', 6 } }; this->check_edits(path, reference, truth, "identical bases, different paths"); } // Prefix + suffix length exceeds the length of the shorter path. { gbwt::vector_type short_path = forward_path({ 5, 5 }); // ACAACA gbwt::vector_type long_path = forward_path({ 1, 2, 3, 5, 3, 2, 1 }); // ACAAACAAACA std::vector> path_is_short { { 'M', 4 }, { 'D', 5 }, { 'M', 2 } }; std::vector> path_is_long { { 'M', 4 }, { 'I', 5 }, { 'M', 2 } }; this->check_edits(short_path, long_path, path_is_short, "prefix + suffix length exceeds path length"); this->check_edits(long_path, short_path, path_is_long, "prefix + suffix length exceeds reference length"); } } TEST_F(AlignDivergingTest, NoPrefixNoSuffix) { // Insertion and deletion are in SpecialCases. // Mismatch + insertion. { gbwt::vector_type path = forward_path({ 1, 2, 5 }); // ACACA gbwt::vector_type reference = forward_path({ 4 }); // CC std::vector> truth { { 'M', 2 }, { 'I', 3 } }; this->check_edits(path, reference, truth, "mismatch + insertion"); } // Mismatch + deletion. { gbwt::vector_type path = forward_path({ 3 }); // AA gbwt::vector_type reference = forward_path({ 2, 1, 6 }); // CACAC std::vector> truth { { 'M', 2 }, { 'D', 3 } }; this->check_edits(path, reference, truth, "mismatch + deletion"); } // Insertion + deletion. { gbwt::vector_type path = forward_path({ 1, 2, 5, 3 }); // ACACAAA gbwt::vector_type reference = forward_path({ 4, 2, 1, 6 }); // CCCACAC std::vector> truth { { 'I', 7 }, { 'D', 7 } }; this->check_edits(path, reference, truth, "insertion + deletion"); } } TEST_F(AlignDivergingTest, NoPrefixWithSuffix) { // Insertion. { gbwt::vector_type path = forward_path({ 1, 2, 5 }); // ACACA gbwt::vector_type reference = forward_path({ 2, 1 }); // CA std::vector> truth { { 'I', 3 }, { 'M', 2 } }; this->check_edits(path, reference, truth, "insertion"); } // Deletion. { gbwt::vector_type path = forward_path({ 1, 2 }); // AC gbwt::vector_type reference = forward_path({ 2, 1, 6 }); // CACAC std::vector> truth { { 'D', 3 }, { 'M', 2 } }; this->check_edits(path, reference, truth, "deletion"); } // Mismatch + insertion. { gbwt::vector_type path = forward_path({ 5, 2, 5 }); // ACACACA gbwt::vector_type reference = forward_path({ 4, 2, 1 }); // CCCA std::vector> truth { { 'M', 2 }, { 'I', 3 }, { 'M', 2 } }; this->check_edits(path, reference, truth, "mismatch + insertion"); } // Mismatch + deletion. { gbwt::vector_type path = forward_path({ 3, 1, 2 }); // AAAC gbwt::vector_type reference = forward_path({ 6, 1, 6 }); // CACACAC std::vector> truth { { 'M', 2 }, { 'D', 3 }, { 'M', 2 } }; this->check_edits(path, reference, truth, "mismatch + deletion"); } // Insertion + deletion. { gbwt::vector_type path = forward_path({ 3, 2, 5, 5 }); // AACACAACA gbwt::vector_type reference = forward_path({ 4, 2, 1, 6, 1 }); // CCCACACA std::vector> truth { { 'I', 6 }, { 'D', 5 }, { 'M', 3 } }; this->check_edits(path, reference, truth, "insertion + deletion"); } } TEST_F(AlignDivergingTest, WithPrefixNoSuffix) { // Insertion. { gbwt::vector_type path = forward_path({ 5, 2, 1 }); // ACACA gbwt::vector_type reference = forward_path({ 1, 2 }); // AC std::vector> truth { { 'M', 2 }, { 'I', 3 } }; this->check_edits(path, reference, truth, "insertion"); } // Deletion. { gbwt::vector_type path = forward_path({ 2, 1 }); // CA gbwt::vector_type reference = forward_path({ 6, 1, 2 }); // CACAC std::vector> truth { { 'M', 2 }, { 'D', 3 } }; this->check_edits(path, reference, truth, "deletion"); } // Mismatch + insertion. { gbwt::vector_type path = forward_path({ 5, 2, 5 }); // ACACACA gbwt::vector_type reference = forward_path({ 1, 2, 4 }); // ACCC std::vector> truth { { 'M', 4 }, { 'I', 3 } }; this->check_edits(path, reference, truth, "mismatch + insertion"); } // Mismatch + deletion. { gbwt::vector_type path = forward_path({ 2, 1, 3 }); // CAAA gbwt::vector_type reference = forward_path({ 6, 1, 6 }); // CACACAC std::vector> truth { { 'M', 4 }, { 'D', 3 } }; this->check_edits(path, reference, truth, "mismatch + deletion"); } // Insertion + deletion. { gbwt::vector_type path = forward_path({ 3, 2, 5, 5 }); // AACACAACA gbwt::vector_type reference = forward_path({ 1, 1, 4, 2, 1, 6, 2 }); // AACCCACACCC std::vector> truth { { 'M', 3 }, { 'I', 6 }, { 'D', 7 } }; this->check_edits(path, reference, truth, "insertion + deletion"); } } TEST_F(AlignDivergingTest, WithPrefixWithSuffix) { // Insertion. { gbwt::vector_type path = forward_path({ 5, 2, 5 }); // ACACACA gbwt::vector_type reference = forward_path({ 1, 4, 1 }); // ACCA std::vector> truth { { 'M', 2 }, { 'I', 3 }, { 'M', 2 } }; this->check_edits(path, reference, truth, "insertion"); } // Deletion. { gbwt::vector_type path = forward_path({ 2, 3, 2 }); // CAAC gbwt::vector_type reference = forward_path({ 6, 1, 6 }); // CACACAC std::vector> truth { { 'M', 2 }, { 'D', 3 }, { 'M', 2 } }; this->check_edits(path, reference, truth, "deletion"); } // Mismatch + insertion. { gbwt::vector_type path = forward_path({ 1, 2, 4, 6, 2, 1 }); // ACCCCACCA gbwt::vector_type reference = forward_path({ 5, 5 }); // ACAACA std::vector> truth { { 'M', 4 }, { 'I', 3 }, { 'M', 2 } }; this->check_edits(path, reference, truth, "mismatch + insertion"); } // Mismatch + deletion. { gbwt::vector_type path = forward_path({ 2, 3, 3, 2 }); // CAAAAC gbwt::vector_type reference = forward_path({ 2, 5, 3, 6 }); // CACAAACAC std::vector> truth { { 'M', 4 }, { 'D', 3 }, { 'M', 2 } }; this->check_edits(path, reference, truth, "mismatch + deletion"); } // Insertion + deletion. { gbwt::vector_type path = forward_path({ 1, 5, 4, 3, 4 }); // AACACCAACC gbwt::vector_type reference = forward_path({ 3, 1, 5, 1, 4, 2 }); // AAAACAACCC std::vector> truth { { 'M', 2 }, { 'I', 6 }, { 'D', 6 }, { 'M', 2 } }; this->check_edits(path, reference, truth, "insertion + deletion"); } } //------------------------------------------------------------------------------ class SubgraphQueryTest : public ::testing::Test { public: std::vector graphs; std::vector reference_samples; std::vector reference_haplotypes; SubgraphQueryTest() { this->graphs = { "gfas/default.gfa", "gfas/components_ref.gfa" }; this->reference_samples = { GENERIC_PATH_SAMPLE_NAME, "ref" }; this->reference_haplotypes = { GBWTGraph::NO_PHASE, 0 }; } void find_path(const GBZ& gbz, size_t graph_number, const std::string& contig_name, path_handle_t& out) const { const gbwt::Metadata& metadata = gbz.index.metadata; std::vector path_ids = metadata.findPaths(metadata.sample(this->reference_samples[graph_number]), metadata.contig(contig_name)); ASSERT_EQ(path_ids.size(), size_t(1)) << "Found " << path_ids.size() << " reference paths for contig " << contig_name << " in graph " << this->graphs[graph_number]; out = gbz.graph.path_to_handle(path_ids.front()); } Subgraph find_subgraph(const GBZ& gbz, const SubgraphQuery& query) const { PathIndex index(gbz); return Subgraph(gbz, &index, query); } void check_subgraph(size_t graph_number, const Subgraph& subgraph, const SubgraphQuery& query, const std::set& nodes, size_t path_count) const { ASSERT_EQ(subgraph.nodes.size(), nodes.size()) << "Invalid number of nodes for query " << query.to_string() << " in graph " << this->graphs[graph_number]; for(nid_t node : nodes) { EXPECT_TRUE(subgraph.nodes.find(node) != subgraph.nodes.end()) << "Node " << node << " not found by query " << query.to_string() << " in graph " << this->graphs[graph_number]; } ASSERT_EQ(subgraph.paths.size(), path_count) << "Invalid number of paths for query " << query.to_string() << " in graph " << this->graphs[graph_number]; } void check_gfa(const GBZ& gbz, size_t graph_number, const Subgraph& subgraph, const SubgraphQuery& query, const std::vector& gfa) const { std::ostringstream out; subgraph.to_gfa(gbz, out); std::string result = out.str(); std::istringstream stream(result); std::vector lines; std::string line; while(std::getline(stream, line)) { lines.push_back(line); } ASSERT_EQ(lines.size(), gfa.size()) << "Invalid number of lines in GFA output for query " << query.to_string() << " in graph " << this->graphs[graph_number]; for(size_t i = 0; i < lines.size(); i++) { EXPECT_EQ(lines[i], gfa[i]) << "Invalid GFA line " << i << " for query " << query.to_string() << " in graph " << this->graphs[graph_number]; } } }; TEST_F(SubgraphQueryTest, AllHaplotypes) { // All haplotypes within 1 bp of contig A, offset 2. std::string contig_name = "A"; size_t offset = 2, context = 1; std::set nodes { 12, 13, 14, 15, 16 }; size_t path_count = 3; std::vector gfa { "H\tVN:Z:1.1\tRS:Z:", // Append reference sample to line 0. "S\t12\tA", "S\t13\tT", "S\t14\tT", "S\t15\tA", "S\t16\tC", "L\t12\t+\t14\t+\t0M", "L\t13\t+\t14\t+\t0M", "L\t14\t+\t15\t+\t0M", "L\t14\t+\t16\t+\t0M", "", // Set reference path manually on line 10. "W\tunknown\t1\tA\t0\t3\t>12>14>15\tCG:Z:3M", "W\tunknown\t2\tA\t0\t3\t>13>14>16\tCG:Z:3M", }; std::vector reference_paths { "W\t" + this->reference_samples[0] + "\t0\tA\t1\t4\t>12>14>15", "W\t" + this->reference_samples[1] + "\t0\tA\t1\t4\t>13>14>15", }; for(size_t i = 0; i < this->graphs.size(); i++) { GBZ gbz = build_gbz(this->graphs[i]); gbwt::FullPathName path_name { this->reference_samples[i], contig_name, this->reference_haplotypes[i], 0 }; SubgraphQuery query = SubgraphQuery::path_offset(path_name, offset, context, SubgraphQuery::all_haplotypes); Subgraph subgraph = this->find_subgraph(gbz, query); this->check_subgraph(i, subgraph, query, nodes, path_count); std::vector true_gfa = gfa; true_gfa[0] += this->reference_samples[i]; true_gfa[10] = reference_paths[i]; this->check_gfa(gbz, i, subgraph, query, true_gfa); } } TEST_F(SubgraphQueryTest, DistinctHaplotypes) { // Distinct haplotypes within 1 bp of contig A, offset 2. std::string contig_name = "A"; size_t offset = 2, context = 1; std::set nodes { 12, 13, 14, 15, 16 }; std::vector path_counts { 2, 3 }; std::vector gfa { "H\tVN:Z:1.1\tRS:Z:", // Append reference sample to line 0. "S\t12\tA", "S\t13\tT", "S\t14\tT", "S\t15\tA", "S\t16\tC", "L\t12\t+\t14\t+\t0M", "L\t13\t+\t14\t+\t0M", "L\t14\t+\t15\t+\t0M", "L\t14\t+\t16\t+\t0M", // Append paths manually. }; std::vector> gfa_paths { { "W\t" + this->reference_samples[0] + "\t0\tA\t1\t4\t>12>14>15\tWT:i:2", "W\tunknown\t1\tA\t0\t3\t>13>14>16\tWT:i:1\tCG:Z:3M", }, { "W\t" + this->reference_samples[1] + "\t0\tA\t1\t4\t>13>14>15\tWT:i:1", "W\tunknown\t1\tA\t0\t3\t>12>14>15\tWT:i:1\tCG:Z:3M", "W\tunknown\t2\tA\t0\t3\t>13>14>16\tWT:i:1\tCG:Z:3M", } }; for(size_t i = 0; i < this->graphs.size(); i++) { GBZ gbz = build_gbz(this->graphs[i]); gbwt::FullPathName path_name { this->reference_samples[i], contig_name, this->reference_haplotypes[i], 0 }; SubgraphQuery query = SubgraphQuery::path_offset(path_name, offset, context, SubgraphQuery::distinct_haplotypes); Subgraph subgraph = this->find_subgraph(gbz, query); this->check_subgraph(i, subgraph, query, nodes, path_counts[i]); std::vector true_gfa = gfa; true_gfa[0] += this->reference_samples[i]; true_gfa.insert(true_gfa.end(), gfa_paths[i].begin(), gfa_paths[i].end()); this->check_gfa(gbz, i, subgraph, query, true_gfa); } } TEST_F(SubgraphQueryTest, NodeBased) { // Distinct haplotypes within 2 bp of node 13. nid_t node_id = 13; size_t context = 2; std::set nodes { 11, 12, 13, 14, 15, 16 }; std::vector path_counts { 2, 3 }; std::vector gfa { "H\tVN:Z:1.1", "S\t11\tG", "S\t12\tA", "S\t13\tT", "S\t14\tT", "S\t15\tA", "S\t16\tC", "L\t11\t+\t12\t+\t0M", "L\t11\t+\t13\t+\t0M", "L\t12\t+\t14\t+\t0M", "L\t13\t+\t14\t+\t0M", "L\t14\t+\t15\t+\t0M", "L\t14\t+\t16\t+\t0M", // Append paths manually. }; std::vector> gfa_paths { { "W\tunknown\t1\tunknown\t0\t4\t>11>12>14>15\tWT:i:2", "W\tunknown\t2\tunknown\t0\t4\t>11>13>14>16\tWT:i:1", }, { "W\tunknown\t1\tunknown\t0\t4\t>11>12>14>15\tWT:i:1", "W\tunknown\t2\tunknown\t0\t4\t>11>13>14>15\tWT:i:1", "W\tunknown\t3\tunknown\t0\t4\t>11>13>14>16\tWT:i:1", } }; for(size_t i = 0; i < this->graphs.size(); i++) { GBZ gbz = build_gbz(this->graphs[i]); SubgraphQuery query = SubgraphQuery::node(node_id, context, SubgraphQuery::distinct_haplotypes); Subgraph subgraph = this->find_subgraph(gbz, query); this->check_subgraph(i, subgraph, query, nodes, path_counts[i]); std::vector true_gfa = gfa; true_gfa.insert(true_gfa.end(), gfa_paths[i].begin(), gfa_paths[i].end()); this->check_gfa(gbz, i, subgraph, query, true_gfa); } } TEST_F(SubgraphQueryTest, ReferenceOnly) { // Reference haplotype within 1 bp of contig A, offset 2. std::string contig_name = "A"; size_t offset = 2, context = 1; std::set nodes { 12, 13, 14, 15, 16 }; size_t path_count = 1; std::vector gfa { "H\tVN:Z:1.1\tRS:Z:", // Append reference sample to line 0. "S\t12\tA", "S\t13\tT", "S\t14\tT", "S\t15\tA", "S\t16\tC", "L\t12\t+\t14\t+\t0M", "L\t13\t+\t14\t+\t0M", "L\t14\t+\t15\t+\t0M", "L\t14\t+\t16\t+\t0M", // Append reference path manually. }; std::vector reference_paths { "W\t" + this->reference_samples[0] + "\t0\tA\t1\t4\t>12>14>15", "W\t" + this->reference_samples[1] + "\t0\tA\t1\t4\t>13>14>15", }; for(size_t i = 0; i < this->graphs.size(); i++) { GBZ gbz = build_gbz(this->graphs[i]); gbwt::FullPathName path_name { this->reference_samples[i], contig_name, this->reference_haplotypes[i], 0 }; SubgraphQuery query = SubgraphQuery::path_offset(path_name, offset, context, SubgraphQuery::reference_only); Subgraph subgraph = this->find_subgraph(gbz, query); this->check_subgraph(i, subgraph, query, nodes, path_count); std::vector true_gfa = gfa; true_gfa[0] += this->reference_samples[i]; true_gfa.push_back(reference_paths[i]); this->check_gfa(gbz, i, subgraph, query, true_gfa); } } TEST_F(SubgraphQueryTest, ContigB) { // Distinct haplotypes within 1 bp of contig B, offset 2. std::string contig_name = "B"; size_t offset = 2, context = 1; std::set nodes { 22, 23, 24, 25 }; std::vector path_counts { 2, 3 }; std::vector gfa { "H\tVN:Z:1.1\tRS:Z:", // Append reference sample to line 0. "S\t22\tA", "S\t23\tT", "S\t24\tT", "S\t25\tA", "L\t22\t+\t24\t+\t0M", "L\t23\t+\t24\t-\t0M", "L\t24\t+\t25\t+\t0M", // Append paths manually. }; std::vector> gfa_paths { { "W\t" + this->reference_samples[0] + "\t0\tB\t1\t4\t>22>24>25\tWT:i:2", "W\tunknown\t1\tB\t0\t3\t>22>24<23\tWT:i:1\tCG:Z:3M", }, { "W\t" + this->reference_samples[1] + "\t0\tB\t1\t4\t>23<24<22\tWT:i:1", "W\tunknown\t1\tB\t0\t3\t>22>24<23\tWT:i:2\tCG:Z:3M", "W\tunknown\t2\tB\t0\t3\t>22>24>25\tWT:i:1\tCG:Z:3M", } }; for(size_t i = 0; i < this->graphs.size(); i++) { GBZ gbz = build_gbz(this->graphs[i]); gbwt::FullPathName path_name { this->reference_samples[i], contig_name, this->reference_haplotypes[i], 0 }; SubgraphQuery query = SubgraphQuery::path_offset(path_name, offset, context, SubgraphQuery::distinct_haplotypes); Subgraph subgraph = this->find_subgraph(gbz, query); this->check_subgraph(i, subgraph, query, nodes, path_counts[i]); std::vector true_gfa = gfa; true_gfa[0] += this->reference_samples[i]; true_gfa.insert(true_gfa.end(), gfa_paths[i].begin(), gfa_paths[i].end()); this->check_gfa(gbz, i, subgraph, query, true_gfa); } } TEST_F(SubgraphQueryTest, Fragmented) { GBZ gbz = build_gbz("gfas/fragmented.gfa"); gbwt::FullPathName path_name { "ref", "contig", 0, 0 }; size_t context = 3; std::vector offsets { 4, 11 }; std::vector> nodes { { 1, 2, 3, 4, 5 }, { 6, 7, 8, 9 } }; std::vector path_counts { 3, 2 }; std::vector> gfas { { "H\tVN:Z:1.1\tRS:Z:ref", "S\t1\tGAT", "S\t2\tTA", "S\t3\tCA", "S\t4\tC", "S\t5\tG", "L\t1\t+\t2\t+\t0M", "L\t2\t+\t3\t+\t0M", "L\t3\t+\t4\t+\t0M", "L\t3\t+\t5\t+\t0M", "W\tref\t0\tcontig\t0\t7\t>1>2>3\tWT:i:1", "W\tunknown\t1\tcontig\t0\t8\t>1>2>3>4\tWT:i:1\tCG:Z:7M1I", "W\tunknown\t2\tcontig\t0\t8\t>1>2>3>5\tWT:i:1\tCG:Z:7M1I" }, { "H\tVN:Z:1.1\tRS:Z:ref", "S\t6\tATTA", "S\t7\tC", "S\t8\tG", "S\t9\tAT", "L\t6\t+\t7\t+\t0M", "L\t6\t+\t8\t+\t0M", "L\t7\t+\t9\t+\t0M", "L\t8\t+\t9\t+\t0M", "W\tref\t0\tcontig\t8\t15\t>6>7>9\tWT:i:2", "W\tunknown\t1\tcontig\t0\t7\t>6>8>9\tWT:i:1\tCG:Z:7M" } }; for(size_t i = 0; i < offsets.size(); i++) { SubgraphQuery query = SubgraphQuery::path_offset(path_name, offsets[i], context, SubgraphQuery::distinct_haplotypes); Subgraph subgraph = this->find_subgraph(gbz, query); this->check_subgraph(0, subgraph, query, nodes[i], path_counts[i]); this->check_gfa(gbz, 0, subgraph, query, gfas[i]); } } //------------------------------------------------------------------------------ } // namespace jltsiren-gbwtgraph-e27bc43/tests/test_utils.cpp000066400000000000000000000621201524136466400220230ustar00rootroot00000000000000#include #include #include #include "shared.h" using namespace gbwtgraph; namespace { //------------------------------------------------------------------------------ class DigestTest : public ::testing::Test { public: std::vector> test_vectors { { "", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "abc", "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" }, { "message digest", "f7846f55cf23e14eebeab5b4e1550cad5b509e3348fbc4efa3a1413d393cb650" }, { "abcdefghijklmnopqrstuvwxyz", "71c480df93d6ae2f1efad1447c66c9525e316218cf51fc8d9ed832f2daf18b73" }, { "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", "db4bfcbd4da0cd85a60c3c37d3fbd8805c77f15fc6b1fdfe614ee0a7c8fdb4c0" }, { "12345678901234567890123456789012345678901234567890123456789012345678901234567890", "f371bc4a311f2b009eef952dd83ca80e2b60026c8e935592d0f9c308453c813e" } }; }; TEST_F(DigestTest, ByCharacter) { for(const auto& vector : this->test_vectors) { DigestStream digest_stream; std::string input = vector.first; for(char c : input) { digest_stream.put(c); } std::string digest = digest_stream.finish(); EXPECT_EQ(digest, vector.second) << "Wrong digest for input \"" << input << "\""; } } TEST_F(DigestTest, ByBlock) { for(const auto& vector : this->test_vectors) { DigestStream digest_stream; std::string input = vector.first; digest_stream.write(input.data(), input.length()); std::string digest = digest_stream.finish(); EXPECT_EQ(digest, vector.second) << "Wrong digest for input \"" << input << "\""; } } //------------------------------------------------------------------------------ class GraphNameTest : public ::testing::Test { public: // NOTE: Set to true to print descriptions of graph relationships during tests. bool print_descriptions = false; const std::string GRAPH = "A"; const std::vector> SUBGRAPH { { "A", "B" }, { "C", "D" }, { "D", "E" } }; const std::vector> TRANSLATION { { "B", "C" }, { "C", "F" } }; GraphName build_manual() const { GraphName gn(this->GRAPH); for(const auto& pair : this->SUBGRAPH) { gn.add_subgraph(pair.first, pair.second); } for(const auto& pair : this->TRANSLATION) { gn.add_translation(pair.first, pair.second); } return gn; } // The first object contains all tags and the second only those relevant to graph names. std::pair build_tags() const { gbwt::Tags all_tags, graph_name_tags; all_tags.set(Version::SOURCE_KEY, Version::SOURCE_VALUE); all_tags.set(GraphName::GBZ_NAME_TAG, this->GRAPH); graph_name_tags.set(GraphName::GBZ_NAME_TAG, this->GRAPH); std::string subgraph_value; for(size_t i = 0; i < this->SUBGRAPH.size(); i++) { if(i > 0) { subgraph_value += ";"; } subgraph_value += this->SUBGRAPH[i].first + "," + this->SUBGRAPH[i].second; } all_tags.set(GraphName::GBZ_SUBGRAPH_TAG, subgraph_value); graph_name_tags.set(GraphName::GBZ_SUBGRAPH_TAG, subgraph_value); std::string translation_value; for(size_t i = 0; i < this->TRANSLATION.size(); i++) { if(i > 0) { translation_value += ";"; } translation_value += this->TRANSLATION[i].first + "," + this->TRANSLATION[i].second; } all_tags.set(GraphName::GBZ_TRANSLATION_TAG, translation_value); graph_name_tags.set(GraphName::GBZ_TRANSLATION_TAG, translation_value); return { all_tags, graph_name_tags }; } // The first vector contains all header lines and the second only those relevant to graph names. std::pair, std::vector> build_gfa_headers() const { std::vector all_headers; all_headers.push_back("H\tVN:Z:1.1"); all_headers.push_back("H\tNM:Z:" + this->GRAPH); for(size_t i = 0; i < this->SUBGRAPH.size(); i++) { std::string subgraph_value = "H\tSG:Z:" + this->SUBGRAPH[i].first + "," + this->SUBGRAPH[i].second;; all_headers.push_back(subgraph_value); } for(size_t i = 0; i < this->TRANSLATION.size(); i++) { std::string translation_value = "H\tTL:Z:" + this->TRANSLATION[i].first + "," + this->TRANSLATION[i].second;; all_headers.push_back(translation_value); } std::vector graph_name_headers(all_headers.begin() + 1, all_headers.end()); return { all_headers, graph_name_headers }; } // The first vector contains all header lines and the second only those relevant to graph names. std::pair, std::vector> build_gaf_headers() const { std::vector all_headers; all_headers.push_back("@HD\tVN:Z:1.0"); all_headers.push_back("@RN\t" + this->GRAPH); for(size_t i = 0; i < this->SUBGRAPH.size(); i++) { std::string subgraph_value = "@SG\t" + this->SUBGRAPH[i].first + "\t" + this->SUBGRAPH[i].second;; all_headers.push_back(subgraph_value); } for(size_t i = 0; i < this->TRANSLATION.size(); i++) { std::string translation_value = "@TL\t" + this->TRANSLATION[i].first + "\t" + this->TRANSLATION[i].second;; all_headers.push_back(translation_value); } std::vector graph_name_headers(all_headers.begin() + 1, all_headers.end()); return { all_headers, graph_name_headers }; } size_t expected_lines(size_t steps, bool no_path) const { size_t result = 2; // from_desc + to_desc result += steps; // one line per relationship result++; // "With:" line result += steps + 1; // one line per graph if(no_path) { result++; } // final graph return result; } size_t count_lines(const std::string& description) const { return std::count(description.begin(), description.end(), '\n'); } void describe_relationships ( const GraphName& from, const GraphName& to, const std::string& from_desc, const std::string& to_desc, size_t steps, bool no_path ) { std::string desc = from.describe_relationship(to, from_desc, to_desc); size_t expected_lines = this->expected_lines(steps, no_path); size_t actual_lines = this->count_lines(desc); EXPECT_EQ(actual_lines, expected_lines) << "Wrong number of lines in description of relationships between " << from.name() << " and " << to.name(); if(this->print_descriptions) { std::cout << desc << std::endl; } } }; TEST_F(GraphNameTest, Empty) { gbwt::Tags empty_tags; std::vector empty_headers; GraphName manual; EXPECT_EQ(manual.name(), "") << "Default constructor sets non-empty name"; EXPECT_FALSE(manual.has_name()) << "Default constructor sets a name"; GraphName from_tags(empty_tags); EXPECT_EQ(from_tags.name(), "") << "Non-empty name from empty tags"; EXPECT_FALSE(from_tags.has_name()) << "Importing empty tags sets a name"; EXPECT_FALSE(from_tags.same(manual)) << "Empty names refer to the same graph (from tags)"; gbwt::Tags to_tags; manual.set_tags(to_tags); EXPECT_EQ(to_tags, empty_tags) << "Tags set from default GraphName are not empty"; this->describe_relationships(from_tags, manual, "tags", "manually built", 0, true); GraphName from_headers(empty_headers); EXPECT_EQ(from_headers.name(), "") << "Non-empty name from empty headers"; EXPECT_FALSE(from_headers.has_name()) << "Importing empty headers sets a name"; EXPECT_FALSE(from_headers.same(manual)) << "Empty names refer to the same graph (from headers)"; std::vector to_headers = manual.gfa_header_lines(); EXPECT_EQ(to_headers, empty_headers) << "GFA headers from default GraphName are not empty"; to_headers = manual.gaf_header_lines(); EXPECT_EQ(to_headers, empty_headers) << "GAF headers from default GraphName are not empty"; }; TEST_F(GraphNameTest, Tags) { gbwt::Tags all_tags; gbwt::Tags graph_name_tags; std::tie(all_tags, graph_name_tags) = this->build_tags(); GraphName from_tags(all_tags); EXPECT_EQ(from_tags.name(), this->GRAPH) << "Wrong graph name from tags"; EXPECT_TRUE(from_tags.has_name()) << "Importing non-empty tags does not set a name"; GraphName manual = this->build_manual(); EXPECT_TRUE(from_tags.same(manual)) << "GraphName from tags is not same as manually built"; EXPECT_EQ(from_tags, manual) << "GraphName from tags is not equal to manually built"; gbwt::Tags to_tags; manual.set_tags(to_tags); EXPECT_EQ(to_tags, graph_name_tags) << "Tags set from manually built GraphName do not match original tags"; GraphName empty; empty.set_tags(to_tags); EXPECT_FALSE(to_tags.contains(GraphName::GBZ_NAME_TAG)) << "Name tag not removed from tags set from empty GraphName"; EXPECT_FALSE(to_tags.contains(GraphName::GBZ_SUBGRAPH_TAG)) << "Subgraph tag not removed from tags set from empty GraphName"; EXPECT_FALSE(to_tags.contains(GraphName::GBZ_TRANSLATION_TAG)) << "Translation tag not removed from tags set from empty GraphName"; } TEST_F(GraphNameTest, GFAHeaders) { std::vector all_headers; std::vector graph_name_headers; std::tie(all_headers, graph_name_headers) = this->build_gfa_headers(); GraphName from_headers(all_headers); EXPECT_EQ(from_headers.name(), this->GRAPH) << "Wrong graph name from GFA headers"; EXPECT_TRUE(from_headers.has_name()) << "Importing non-empty GFA headers does not set a name"; GraphName manual = this->build_manual(); EXPECT_TRUE(from_headers.same(manual)) << "GraphName from GFA headers is not same as manually built"; EXPECT_EQ(from_headers, manual) << "GraphName from GFA headers is not equal to manually built"; std::vector to_headers = manual.gfa_header_lines(); EXPECT_EQ(to_headers, graph_name_headers) << "GFA headers from manually built GraphName do not match original headers"; std::vector weird_headers { "H\tVN:Z:1.1", "H\tNM:Z:A:B", // We allow colons in string values. "H\txy:Z:C:D", // Including unknown typed fields. }; GraphName from_weird(weird_headers); EXPECT_EQ(from_weird.name(), "A:B") << "Wrong graph name from weird GFA headers"; } TEST_F(GraphNameTest, GAFHeaders) { std::vector all_headers; std::vector graph_name_headers; std::tie(all_headers, graph_name_headers) = this->build_gaf_headers(); GraphName from_headers(all_headers); EXPECT_EQ(from_headers.name(), this->GRAPH) << "Wrong graph name from GAF headers"; EXPECT_TRUE(from_headers.has_name()) << "Importing non-empty GAF headers does not set a name"; GraphName manual = this->build_manual(); EXPECT_TRUE(from_headers.same(manual)) << "GraphName from GAF headers is not same as manually built"; EXPECT_EQ(from_headers, manual) << "GraphName from GAF headers is not equal to manually built"; std::vector to_headers = manual.gaf_header_lines(); EXPECT_EQ(to_headers, graph_name_headers) << "GAF headers from manually built GraphName do not match original headers"; } TEST_F(GraphNameTest, SubgraphPath) { GraphName a = this->build_manual(); GraphName a_empty(this->GRAPH); // Same graph. EXPECT_TRUE(a.subgraph_of(a_empty)) << "Graph is not subgraph of itself (relationships in subgraph)"; EXPECT_TRUE(a_empty.subgraph_of(a)) << "Graph is not subgraph of itself (relationships in supergraph)"; EXPECT_TRUE(a_empty.subgraph_of(a_empty)) << "Graph is not subgraph of itself (no relationships)"; this->describe_relationships(a, a_empty, "the original graph", "the same graph", 0, false); // Single step. { GraphName b_empty("B"); GraphName b = b_empty; b.add_relationships(a); EXPECT_TRUE(a.subgraph_of(b_empty)) << "A is not subgraph of B (relationships in subgraph)"; EXPECT_TRUE(a_empty.subgraph_of(b)) << "A is not subgraph of B (relationships in supergraph)"; EXPECT_FALSE(b.subgraph_of(a)) << "B is subgraph of A"; this->describe_relationships(a, b, "subgraph (first)", "supergraph (second)", 1, false); this->describe_relationships(b, a, "supergraph (first)", "subgraph (second)", 1, false); } // No path. GraphName c_empty("C"); GraphName c = c_empty; c.add_relationships(a); EXPECT_FALSE(a.subgraph_of(c)) << "A is subgraph of C"; EXPECT_FALSE(c.subgraph_of(a)) << "C is subgraph of A"; // Skip description, as there is a path with a translation. // Multiple steps. { GraphName e_empty("E"); GraphName e = e_empty; e.add_relationships(a); EXPECT_TRUE(c.subgraph_of(e_empty)) << "C is not subgraph of E (relationships in subgraph)"; EXPECT_TRUE(c_empty.subgraph_of(e)) << "C is not subgraph of E (relationships in supergraph)"; EXPECT_FALSE(e.subgraph_of(c)) << "E is subgraph of C"; this->describe_relationships(c, e, "subgraph", "supergraph", 2, false); } // Relationships split between graphs. { GraphName from("from"); from.add_subgraph("from", "middle"); GraphName to("to"); to.add_subgraph("middle", "to"); EXPECT_TRUE(from.subgraph_of(to)) << "from is not subgraph of to (relationships split)"; EXPECT_FALSE(to.subgraph_of(from)) << "to is subgraph of from"; this->describe_relationships(from, to, "subgraph", "supergraph", 2, false); } } TEST_F(GraphNameTest, TranslationPath) { GraphName a = this->build_manual(); GraphName a_empty(this->GRAPH); // Same graph. EXPECT_TRUE(a.translates_to(a_empty)) << "Graph does not translate to itself (relationships in translation)"; EXPECT_TRUE(a_empty.translates_to(a)) << "Graph does not translate to itself (no relationships)"; EXPECT_TRUE(a_empty.translates_to(a_empty)) << "Graph does not translate to itself (no relationships)"; this->describe_relationships(a, a_empty, "the original graph", "the same graph", 0, false); // Single subgraph step. GraphName b_empty("B"); GraphName b = b_empty; b.add_relationships(a); EXPECT_TRUE(a.subgraph_of(b_empty)) << "A is not subgraph of B (relationships in subgraph)"; EXPECT_TRUE(a_empty.subgraph_of(b)) << "A is not subgraph of B (relationships in supergraph)"; EXPECT_FALSE(b.subgraph_of(a)) << "B is subgraph of A"; this->describe_relationships(a, b, "subgraph", "supergraph", 1, false); // Single translation step. GraphName c_empty("C"); GraphName c = c_empty; c.add_relationships(a); EXPECT_TRUE(b.translates_to(c_empty)) << "B does not translate to C (relationships in source graph)"; EXPECT_TRUE(b_empty.translates_to(c)) << "B does not translate to C (relationships in target graph)"; EXPECT_FALSE(c.translates_to(b)) << "C translates to B"; this->describe_relationships(b, c, "source graph", "target graph", 1, false); // Mixed steps. EXPECT_TRUE(a.translates_to(c_empty)) << "A does not translate to C (relationships in source graph)"; EXPECT_TRUE(a_empty.translates_to(c)) << "A does not translate to C (relationships in target graph)"; EXPECT_FALSE(c.translates_to(a)) << "C translates to A"; this->describe_relationships(a, c, "source graph", "target graph", 2, false); // Multiple subgraph steps. { GraphName e_empty("E"); GraphName e = e_empty; e.add_relationships(a); EXPECT_TRUE(c.translates_to(e_empty)) << "C does not translate to E (relationships in source graph)"; EXPECT_TRUE(c_empty.translates_to(e)) << "C does not translate to E (relationships in target graph)"; EXPECT_FALSE(e.translates_to(c)) << "E translates to C"; this->describe_relationships(c, e, "source graph", "target graph", 2, false); } // Multiple translation steps. { GraphName f_empty("F"); GraphName f = f_empty; f.add_relationships(a); EXPECT_TRUE(b.translates_to(f_empty)) << "B does not translate to F (relationships in source graph)"; EXPECT_TRUE(b_empty.translates_to(f)) << "B does not translate to F (relationships in target graph)"; EXPECT_FALSE(f.translates_to(b)) << "F translates to B"; this->describe_relationships(b, f, "source graph", "target graph", 2, false); } // Relationships split between graphs. { GraphName from("from"); from.add_translation("from", "middle"); GraphName to("to"); to.add_translation("middle", "to"); EXPECT_TRUE(from.translates_to(to)) << "from does not translate to to (relationships split)"; EXPECT_FALSE(to.translates_to(from)) << "to translates to from"; this->describe_relationships(from, to, "source graph", "target graph", 2, false); } } //------------------------------------------------------------------------------ class MetadataBuilderTest : public ::testing::Test { public: void check_empty(const MetadataBuilder& builder) const { gbwt::Metadata metadata = builder.get_metadata(); ASSERT_EQ(metadata.samples(), gbwt::size_type(0)) << "Invalid number of samples"; ASSERT_EQ(metadata.contigs(), gbwt::size_type(0)) << "Invalid number of contigs"; ASSERT_EQ(metadata.paths(), gbwt::size_type(0)) << "Invalid number of paths"; } void create_example( std::vector& samples, std::vector& contigs, std::vector& paths, bool generic_reference) const { std::string reference_sample = (generic_reference ? GENERIC_PATH_SAMPLE_NAME : "GRCh38"); size_t reference_haplotype = (generic_reference ? GBWTGraph::NO_PHASE : 0); samples.push_back(reference_sample); samples.push_back("HG002"); samples.push_back("HG003"); contigs.push_back("chr1"); contigs.push_back("chr2"); paths.push_back({ reference_sample, "chr1", reference_haplotype, 0 }); paths.push_back({ reference_sample, "chr2", reference_haplotype, 0 }); paths.push_back({ "HG002", "chr1", 1, 0 }); paths.push_back({ "HG002", "chr1", 2, 0 }); paths.push_back({ "HG002", "chr2", 1, 0 }); paths.push_back({ "HG002", "chr2", 2, 0 }); paths.push_back({ "HG003", "chr1", 1, 0 }); paths.push_back({ "HG003", "chr1", 2, 0 }); paths.push_back({ "HG003", "chr2", 1, 0 }); paths.push_back({ "HG003", "chr2", 2, 0 }); } void add_hg004( std::vector& samples, std::vector& paths) const { samples.push_back("HG004"); paths.push_back({ "HG004", "chr1", 1, 0 }); paths.push_back({ "HG004", "chr1", 2, 0 }); paths.push_back({ "HG004", "chr2", 1, 0 }); paths.push_back({ "HG004", "chr2", 2, 0 }); } size_t get_job(const gbwt::FullPathName& path) const { if(path.contig_name == "chr1") { return 0; } if(path.contig_name == "chr2") { return 1; } return 0; } void add_haplotypes(MetadataBuilder& builder, const std::vector& paths, size_t from, bool assign_job) { for(size_t i = from; i < paths.size(); i++) { const gbwt::FullPathName& path = paths[i]; size_t job = (assign_job ? get_job(path) : 0); if(path.sample_name == GENERIC_PATH_SAMPLE_NAME) { builder.add_generic_path(path.contig_name, job); } else { builder.add_haplotype(path.sample_name, path.contig_name, path.haplotype, path.offset, job); } } } void add_walks(MetadataBuilder& builder, const std::vector& paths) { for(const gbwt::FullPathName& path : paths) { if(path.sample_name == GENERIC_PATH_SAMPLE_NAME) { builder.add_generic_path(path.contig_name); } else { std::string haplotype = std::to_string(path.haplotype); std::string start = std::to_string(path.offset); builder.add_walk(path.sample_name, haplotype, path.contig_name, start); } } } void add_walks_no_interval(MetadataBuilder& builder, const std::vector& paths) { std::string no_interval = "*"; for(const gbwt::FullPathName& path : paths) { if(path.sample_name == GENERIC_PATH_SAMPLE_NAME) { builder.add_generic_path(path.contig_name); } else { std::string haplotype = std::to_string(path.haplotype); builder.add_walk(path.sample_name, haplotype, path.contig_name, no_interval); } } } void add_named_paths(MetadataBuilder& builder, const std::vector& paths) { for(const gbwt::FullPathName& path : paths) { std::string name; if(path.sample_name == GENERIC_PATH_SAMPLE_NAME) { name = path.contig_name; } else { name = path.sample_name + "#" + std::to_string(path.haplotype) + "#" + path.contig_name; } builder.add_path(name); } } void check_metadata( const gbwt::Metadata& metadata, const std::vector& samples, const std::vector& contigs, const std::vector& paths) const { ASSERT_EQ(metadata.samples(), samples.size()) << "Invalid number of samples"; for(size_t i = 0; i < samples.size(); i++) { EXPECT_EQ(metadata.sample(i), samples[i]) << "Invalid sample name at index " << i; } ASSERT_EQ(metadata.contigs(), contigs.size()) << "Invalid number of contigs"; for(size_t i = 0; i < contigs.size(); i++) { EXPECT_EQ(metadata.contig(i), contigs[i]) << "Invalid contig name at index " << i; } ASSERT_EQ(metadata.paths(), paths.size()) << "Invalid number of paths"; for(size_t i = 0; i < paths.size(); i++) { gbwt::PathName path = metadata.path(i); EXPECT_EQ(metadata.sample(path.sample), paths[i].sample_name) << "Invalid sample name for path " << i; EXPECT_EQ(metadata.contig(path.contig), paths[i].contig_name) << "Invalid contig name for path " << i; EXPECT_EQ(path.phase, paths[i].haplotype) << "Invalid haplotype for path " << i; EXPECT_EQ(path.count, paths[i].offset) << "Invalid offset for path " << i; } } }; TEST_F(MetadataBuilderTest, Empty) { this->check_empty(MetadataBuilder()); } TEST_F(MetadataBuilderTest, GenericPathsAndHaplotypes) { std::vector samples, contigs; std::vector paths; this->create_example(samples, contigs, paths, true); MetadataBuilder builder; this->add_haplotypes(builder, paths, 0, false); this->check_metadata(builder.get_metadata(), samples, contigs, paths); } TEST_F(MetadataBuilderTest, GFAPathsAndWalks) { std::vector samples, contigs; std::vector paths; this->create_example(samples, contigs, paths, true); MetadataBuilder builder; this->add_walks(builder, paths); this->check_metadata(builder.get_metadata(), samples, contigs, paths); } TEST_F(MetadataBuilderTest, GFAWalksNoInterval) { std::vector samples, contigs; std::vector paths; this->create_example(samples, contigs, paths, true); MetadataBuilder builder; this->add_walks_no_interval(builder, paths); this->check_metadata(builder.get_metadata(), samples, contigs, paths); } TEST_F(MetadataBuilderTest, PanSN) { std::vector samples, contigs; std::vector paths; this->create_example(samples, contigs, paths, false); MetadataBuilder builder( GFAParsingParameters::PAN_SN_REGEX, GFAParsingParameters::PAN_SN_FIELDS, GFAParsingParameters::PAN_SN_SENSE ); this->add_haplotypes(builder, paths, 0, false); this->check_metadata(builder.get_metadata(), samples, contigs, paths); } TEST_F(MetadataBuilderTest, Clear) { std::vector samples, contigs; std::vector paths; this->create_example(samples, contigs, paths, true); MetadataBuilder builder; this->add_haplotypes(builder, paths, 0, false); builder.clear(); this->check_empty(builder); } TEST_F(MetadataBuilderTest, MultipleFormats) { std::vector samples, contigs; std::vector paths; this->create_example(samples, contigs, paths, true); MetadataBuilder builder; builder.add_path_name_format( GFAParsingParameters::PAN_SN_REGEX, GFAParsingParameters::PAN_SN_FIELDS, GFAParsingParameters::PAN_SN_SENSE ); builder.add_path_name_format(".*", "C", PathSense::GENERIC); this->add_named_paths(builder, paths); this->check_metadata(builder.get_metadata(), samples, contigs, paths); } TEST_F(MetadataBuilderTest, FromMetadata) { std::vector samples, contigs; std::vector paths; this->create_example(samples, contigs, paths, true); size_t old_paths = paths.size(); MetadataBuilder builder; this->add_haplotypes(builder, paths, 0, false); gbwt::Metadata metadata = builder.get_metadata(); MetadataBuilder new_builder(metadata); this->add_hg004(samples, paths); this->add_haplotypes(new_builder, paths, old_paths, false); this->check_metadata(new_builder.get_metadata(), samples, contigs, paths); } TEST_F(MetadataBuilderTest, MultipleJobs) { std::vector samples, contigs; std::vector paths; this->create_example(samples, contigs, paths, true); MetadataBuilder builder; this->add_haplotypes(builder, paths, 0, true); std::vector reordered_paths; for(size_t job = 0; job < contigs.size(); job++) { for(size_t i = 0; i < paths.size(); i++) { if(this->get_job(paths[i]) == job) { reordered_paths.push_back(paths[i]); } } } this->check_metadata(builder.get_metadata(), samples, contigs, reordered_paths); } //------------------------------------------------------------------------------ } // namespace