odc-1.6.3/0000775000175000017500000000000015146027420012453 5ustar alastairalastairodc-1.6.3/.github/0000775000175000017500000000000015146027420014013 5ustar alastairalastairodc-1.6.3/.github/.cmake-options0000664000175000017500000000002315146027420016560 0ustar alastairalastair-DENABLE_FORTRAN=ONodc-1.6.3/.github/ci-hpc-config.yml0000664000175000017500000000016315146027420017144 0ustar alastairalastairbuild: modules: - ninja dependencies: - ecmwf/ecbuild@develop - ecmwf/eckit@develop parallel: 64 odc-1.6.3/.github/ci-config.yml0000664000175000017500000000021215146027420016367 0ustar alastairalastairdependencies: | ecmwf/ecbuild MathisRosenhauer/libaec@refs/tags/v1.1.3 ecmwf/eckit dependency_branch: develop parallelism_factor: 8 odc-1.6.3/.github/workflows/0000775000175000017500000000000015146027420016050 5ustar alastairalastairodc-1.6.3/.github/workflows/notify-new-pr.yml0000664000175000017500000000042215146027420021307 0ustar alastairalastairname: Notify new PR on: pull_request: types: - "opened" jobs: notify: runs-on: ubuntu-latest steps: - name: Notify new PR uses: ecmwf/notify-teams-pr@v1 with: incoming_webhook: ${{ secrets.MS_TEAMS_INCOMING_WEBHOOK }} odc-1.6.3/.github/workflows/notify-new-issue.yml0000664000175000017500000000042515146027420022021 0ustar alastairalastairname: Notify new issue on: issues: types: - "opened" jobs: notify: runs-on: ubuntu-latest steps: - name: Notify new issue uses: ecmwf/notify-teams-issue@v1 with: incoming_webhook: ${{ secrets.MS_TEAMS_INCOMING_WEBHOOK }} odc-1.6.3/.github/workflows/label-public-pr.yml0000664000175000017500000000034315146027420021545 0ustar alastairalastair# Manage labels of pull requests that originate from forks name: label-public-pr on: pull_request_target: types: [opened, synchronize] jobs: label: uses: ecmwf/reusable-workflows/.github/workflows/label-pr.yml@v2 odc-1.6.3/.github/workflows/cd.yml0000664000175000017500000000036215146027420017162 0ustar alastairalastairname: cd on: push: tags: - '**' jobs: deploy: uses: ecmwf/reusable-workflows/.github/workflows/create-package.yml@v2 secrets: inherit wheel: uses: ./.github/workflows/build-wheel-wrapper.yml secrets: inherit odc-1.6.3/.github/workflows/docs.yml0000664000175000017500000000502615146027420017526 0ustar alastairalastairname: Publish Documentation to sites.ecmwf.int on: pull_request: types: [opened, synchronize, reopened, closed] push: branches: - 'develop' - 'master' tags: - '**' jobs: build: if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed' || github.event_name == 'push'}} runs-on: ubuntu-latest outputs: artifact-id: ${{ steps.upload-doc.outputs.artifact-id }} steps: - name: Checkout uses: actions/checkout@v4 - name: Setup Python uses: actions/setup-python@v5 with: python-version: '3.13' - name: Install python dependencies run: | python -m pip install -r docs/requirements.txt - name: Prepare Doxygen uses: ecmwf/reusable-workflows/setup-doxygen@main with: version: 1.14.0 - name: Build doc run: | cd docs make html -j 32 - name: Archive documentation id: upload-doc uses: actions/upload-artifact@v4 with: name: documentation path: docs/_build/html preview-publish: if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed' }} needs: build uses: ecmwf/reusable-workflows/.github/workflows/pr-preview-publish.yml@main with: artifact-id: ${{ needs.build.outputs.artifact-id }} space: docs name: dev-section path: odc/pull-requests link-text: 📖 Documentation 📖 secrets: sites-token: ${{ secrets.ECMWF_SITES_DOCS_DEV_SECTION_TOKEN }} preview-unpublish: if: ${{ github.event_name == 'pull_request' && github.event.action == 'closed' }} uses: ecmwf/reusable-workflows/.github/workflows/pr-preview-unpublish.yml@main with: space: docs name: dev-section path: odc/pull-requests secrets: sites-token: ${{ secrets.ECMWF_SITES_DOCS_DEV_SECTION_TOKEN }} publish: if: >- ${{ github.event_name == 'push' && ( github.ref_name == 'develop' || github.ref_name == 'master' || github.ref_type == 'tag' ) }} needs: build uses: ecmwf/reusable-workflows/.github/workflows/docs-publish.yml@main with: artifact-id: ${{ needs.build.outputs.artifact-id }} space: docs name: dev-section path: odc id: ${{ github.ref_name }} softlink: ${{ github.ref_name == 'master' && 'stable' || github.ref_name == 'develop' && 'latest' }} secrets: sites-token: ${{ secrets.ECMWF_SITES_DOCS_DEV_SECTION_TOKEN }} odc-1.6.3/.github/workflows/sync.yml0000664000175000017500000000114415146027420017547 0ustar alastairalastairname: sync # Controls when the workflow will run on: # Trigger the workflow on all pushes push: branches: - "**" tags: - "**" # Trigger the workflow when a branch or tag is deleted delete: ~ jobs: # Calls a reusable CI workflow to sync the current with a remote repository. # It will correctly handle addition of any new and removal of existing Git objects. sync: name: sync uses: ecmwf/reusable-workflows/.github/workflows/sync.yml@v2 secrets: target_repository: odb/odc target_username: ClonedDuck target_token: ${{ secrets.BITBUCKET_PAT }} odc-1.6.3/.github/workflows/ci.yml0000664000175000017500000000237615146027420017176 0ustar alastairalastairname: ci on: # Trigger the workflow on push to master or develop, except tag creation push: branches: - 'master' - 'develop' tags-ignore: - '**' # Trigger the workflow on pull request pull_request: ~ # Trigger the workflow manually workflow_dispatch: ~ # Trigger after public PR approved for CI pull_request_target: types: [labeled] jobs: # Run CI including downstream packages on self-hosted runners downstream-ci: name: downstream-ci if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} uses: ecmwf/downstream-ci/.github/workflows/downstream-ci.yml@main with: odc: ecmwf/odc@${{ github.event.pull_request.head.sha || github.sha }} codecov_upload: true clang_format: true secrets: inherit # Build downstream packages on HPC downstream-ci-hpc: name: downstream-ci-hpc if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} uses: ecmwf/downstream-ci/.github/workflows/downstream-ci-hpc.yml@main with: odc: ecmwf/odc@${{ github.event.pull_request.head.sha || github.sha }} secrets: inherit odc-1.6.3/.github/workflows/build-wheel-wrapper.yml0000664000175000017500000000132115146027420022447 0ustar alastairalastair# (C) Copyright 2024- ECMWF. # # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. # In applying this licence, ECMWF does not waive the privileges and immunities # granted to it by virtue of its status as an intergovernmental organisation # nor does it submit to any jurisdiction. name: Build Python Wrapper Wheel on: # Trigger the workflow manually workflow_dispatch: ~ # Allow to be called from another workflow -- eg `cd.yml` workflow_call: ~ jobs: python-wrapper-wheel: name: Python Wrapper Wheel uses: ecmwf/reusable-workflows/.github/workflows/python-wrapper-wheel.yml@main secrets: inherit odc-1.6.3/python_wrapper/0000775000175000017500000000000015146027420015534 5ustar alastairalastairodc-1.6.3/python_wrapper/setup.py0000664000175000017500000000006215146027420017244 0ustar alastairalastairfrom setup_utils import plain_setup plain_setup() odc-1.6.3/python_wrapper/buildconfig0000664000175000017500000000145215146027420017746 0ustar alastairalastair# (C) Copyright 2024- ECMWF. # # This software is licensed under the terms of the Apache Licence Version 2.0 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. # In applying this licence, ECMWF does not waive the privileges and immunities # granted to it by virtue of its status as an intergovernmental organisation # nor does it submit to any jurisdiction. # to be source'd by wheelmaker's compile.sh *and* wheel-linux.sh # NOTE replace the whole thing with pyproject.toml? Less powerful, and quaint to use for sourcing ecbuild invocation # TODO we duplicate information -- pyproject.toml's `name` and `packages` are derivable from $NAME and must stay consistent NAME="odc" CMAKE_PARAMS="-Deckit_ROOT=/tmp/odc/prereqs/eckitlib" PYPROJECT_DIR="python_wrapper" DEPENDENCIES='["eckitlib"]' odc-1.6.3/python_wrapper/setup.cfg0000664000175000017500000000021215146027420017350 0ustar alastairalastair[metadata] description = "odclib" long_description = file: README.md long_description_content_type = text/markdown author = file: AUTHORS odc-1.6.3/README.md0000664000175000017500000000476715146027420013750 0ustar alastairalastair# odc [![Build Status](https://img.shields.io/github/workflow/status/ecmwf/odc/Continuous%20Integration/develop)](https://github.com/ecmwf/odc/actions/workflows/ci.yml) [![Documentation Status](https://readthedocs.org/projects/odc/badge/?version=latest)](https://odc.readthedocs.io/en/latest/?badge=latest) [![Licence](https://img.shields.io/github/license/ecmwf/odc)](https://github.com/ecmwf/odc/blob/develop/LICENSE) [![PyPI](https://img.shields.io/pypi/v/odclib)](https://pypi.org/project/odclib/) C, C++ and Fortran libraries providing encoders and decoders and interrogation of meteorological data encoded in ODB-2 format as well as command line tools for investigation and manipulation of ODB-2 data. [Documentation] ## Dependencies ### Required * C or C++ compiler * [CMake] * [ecbuild] * [eckit] ### Optional * Fortran compiler * [Doxygen] ## Installation ```sh git clone https://github.com/ecmwf/odc cd odc # Setup environment variables (edit as needed) SRC_DIR=$(pwd) BUILD_DIR=build INSTALL_DIR=$HOME/local export eckit_DIR=$INSTALL_DIR # set to eckit install prefix # Create the the build directory mkdir $BUILD_DIR cd $BUILD_DIR # Run ecbuild (CMake) ecbuild --prefix=$INSTALL_DIR -- $SRC_DIR # Build and install make -j10 make test # optional make install # Check installation $INSTALL_DIR/bin/odc --version ``` ## Usage Include the `odc` headers like so: ```c // odc_test.c #include "odc/api/odc.h" int main() { odc_initialise_api(); return 0; } ``` Make sure to reference the linked library when compiling: ```sh gcc -lodccore odc_test.c ``` ## Build Documentation The documentation is generated using Sphinx. First, make sure that `Doxygen` module is available, and then install Python dependencies in your environment: ```sh cd docs pip install -r requirements.txt ``` You can then build the documentation by using **make**: ```sh cd docs make html ``` The built HTML documentation will be available under the `docs/_build/html/index.html` path. ## Licence This software is licensed under the terms of the Apache Licence Version 2.0 which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. In applying this licence, ECMWF does not waive the privileges and immunities granted to it by virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction. [Documentation]: https://odc.readthedocs.io/en/latest/ [CMake]: https://cmake.org [ecbuild]: https://github.com/ecmwf/ecbuild [eckit]: https://github.com/ecmwf/eckit [Doxygen]: https://www.doxygen.nl odc-1.6.3/odc-import.cmake.in0000664000175000017500000000016315146027420016137 0ustar alastairalastairinclude( CMakeFindDependencyMacro ) find_dependency( eckit HINTS ${CMAKE_CURRENT_LIST_DIR}/../eckit @eckit_DIR@ ) odc-1.6.3/tests/0000775000175000017500000000000015146027420013615 5ustar alastairalastairodc-1.6.3/tests/sql/0000775000175000017500000000000015146027420014414 5ustar alastairalastairodc-1.6.3/tests/sql/test_text_reader_select.cc0000664000175000017500000001315515146027420021634 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/testing/Test.h" #include "eckit/types/FloatCompare.h" #include "odc/Select.h" #include "odc/sql/TODATable.h" using namespace eckit::testing; // ------------------------------------------------------------------------------------------------------ CASE("Simple select on CSV (one column) data") { // Construct some data to use std::stringstream DATA; DATA << "a:REAL\n"; for (size_t i = 1; i <= 10; i++) { DATA << i << "\n"; } SECTION("Select with where condition") { odc::Select select("select * where a > 4"); select.database().addImplicitTable(new odc::sql::ODBCSVTable(select.database(), DATA, "input", ",")); size_t count = 0; for (odc::Select::iterator it = select.begin(); it != select.end(); ++it) { count++; EXPECT((*it)[0] == count + 4); } EXPECT(count == 6); } SECTION("Sum one column") { odc::Select select("select sum(a), count(a), count(*);"); select.database().addImplicitTable(new odc::sql::ODBCSVTable(select.database(), DATA, "input", ",")); odc::Select::iterator it = select.begin(); EXPECT(it != select.end()); EXPECT((*it)[0] == 55); EXPECT((*it)[1] == 10); EXPECT((*it)[2] == 10); ++it; EXPECT(it == select.end()); } SECTION("Mixed output and aggregate") { odc::Select select("select a, sum(a), count(a)"); select.database().addImplicitTable(new odc::sql::ODBCSVTable(select.database(), DATA, "input", ",")); odc::Select::iterator it = select.begin(); EXPECT(it != select.end()); // n.b. as all values of a are different, we get counts of 1 size_t count = 0; for (odc::Select::iterator it = select.begin(); it != select.end(); ++it) { count++; EXPECT(it->data(0) == count); EXPECT(it->data(1) == count); EXPECT(it->data(2) == 1); } EXPECT(count = 10); } } CASE("Simple select on CSV (two columns) data") { // Construct some data to use std::stringstream DATA; DATA << "a:REAL,b:STRING,c:INTEGER,d:INTEGER\n"; for (size_t i = 1; i <= 10; i++) { DATA << i << ","; // length=16/17 --> tests overflowing 16 char and resizing string on last row DATA << "a-long-" << i << "-str****,"; DATA << 11 - i << ","; DATA << int(i / 3) << "\n"; } SECTION("Select with where condition") { odc::Select select("select * where a > 4"); select.database().addImplicitTable(new odc::sql::ODBCSVTable(select.database(), DATA, "input", ",")); size_t count = 0; for (odc::Select::iterator it = select.begin(); it != select.end(); ++it) { count++; std::stringstream colb; colb << "a-long-" << (count + 4) << "-str****"; EXPECT(it->data(0) == count + 4); EXPECT(it->dataSizeDoubles(1) == (count == 6 ? 3 : 2)); EXPECT(::strncmp(colb.str().c_str(), (char*)&it->data(1), it->dataSizeDoubles(1) * sizeof(double)) == 0); EXPECT(it->data(2) == 11 - (count + 4)); } EXPECT(count == 6); } SECTION("Sum and count") { odc::Select select("select sum(a), count(a), count(*), sum(c), count(c), count(b);"); select.database().addImplicitTable(new odc::sql::ODBCSVTable(select.database(), DATA, "input", ",")); odc::Select::iterator it = select.begin(); EXPECT(it != select.end()); EXPECT(it->data(0) == 55); EXPECT(it->data(1) == 10); EXPECT(it->data(2) == 10); EXPECT(it->data(3) == 55); EXPECT(it->data(4) == 10); EXPECT(it->data(5) == 10); ++it; EXPECT(it == select.end()); } SECTION("Mixed output and aggregate") { odc::Select select("select d, sum(a), count(c)"); select.database().addImplicitTable(new odc::sql::ODBCSVTable(select.database(), DATA, "input", ",")); odc::Select::iterator it = select.begin(); EXPECT(it != select.end()); bool found[4] = {false, false, false, false}; for (odc::Select::iterator it = select.begin(); it != select.end(); ++it) { EXPECT(0 <= it->data(0) && 3 >= it->data(0)); found[int(it->data(0))] = true; switch (int(it->data(0))) { case 0: EXPECT(it->data(1) == 3); EXPECT(it->data(2) == 2); break; case 1: EXPECT(it->data(1) == 12); EXPECT(it->data(2) == 3); break; case 2: EXPECT(it->data(1) == 21); EXPECT(it->data(2) == 3); break; case 3: EXPECT(it->data(1) == 19); EXPECT(it->data(2) == 2); break; default: ASSERT(false); } } EXPECT(std::all_of(std::begin(found), std::end(found), [](bool x) { return x; })); } } // ------------------------------------------------------------------------------------------------------ int main(int argc, char* argv[]) { return run_tests(argc, argv); } odc-1.6.3/tests/sql/CMakeLists.txt0000664000175000017500000000072715146027420017162 0ustar alastairalastair list( APPEND _sql_odc_tests test_functions test_text_reader_select test_oda_output ) foreach( _test ${_sql_odc_tests} ) ecbuild_add_test( TARGET odc_${_test} SOURCES ${_test}.cc ../TemporaryFiles.h ENVIRONMENT ${test_environment} TEST_DEPENDS odc_get_test_data LIBS eckit odccore ) endforeach() # TODO: Pure tests for the pure-sql stuff. The functions clearly invoke odc-specific behaviour odc-1.6.3/tests/sql/test_oda_output.cc0000664000175000017500000003734715146027420020163 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/testing/Test.h" #include "eckit/io/FileHandle.h" #include "eckit/io/MemoryHandle.h" #include "eckit/log/Log.h" #include "eckit/types/FixedString.h" #include "eckit/sql/SQLOutputConfig.h" #include "eckit/sql/SQLParser.h" #include "eckit/sql/SQLSession.h" #include "eckit/sql/SQLStatement.h" #include "odc/Writer.h" #include "odc/api/Odb.h" #include "odc/sql/ODAOutput.h" #include "odc/sql/TODATable.h" #include #include #include #include using namespace eckit::testing; using eckit::Log; // ------------------------------------------------------------------------------------------------------ template odc::api::ColumnType typeToName() { ASSERT(false); return odc::api::IGNORE; } template <> odc::api::ColumnType typeToName() { return odc::api::DOUBLE; } template <> odc::api::ColumnType typeToName() { return odc::api::INTEGER; } template <> odc::api::ColumnType typeToName() { return odc::api::BITFIELD; } template <> odc::api::ColumnType typeToName>() { return odc::api::STRING; } template <> odc::api::ColumnType typeToName>() { return odc::api::STRING; } template <> odc::api::ColumnType typeToName>() { return odc::api::STRING; } void quick_encode_internal(eckit::DataHandle& out, std::vector& columnInfo, std::vector& strides) { odc::api::encode(out, columnInfo, strides); } template typename std::enable_if::value, void>::type quick_encode_internal( eckit::DataHandle& out, std::vector& columnInfo, std::vector& strides, const std::string& colname, const std::vector& vals, Ts&&... args) { columnInfo.emplace_back(odc::api::ColumnInfo{colname, typeToName(), sizeof(vals[0])}); strides.emplace_back(odc::api::ConstStridedData{&vals[0], vals.size(), sizeof(vals[0]), sizeof(vals[0])}); quick_encode_internal(out, columnInfo, strides, std::forward(args)...); } // Bitfields need some extra info template void quick_encode_internal(eckit::DataHandle& out, std::vector& columnInfo, std::vector& strides, const std::string& colname, const std::vector& vals, const std::vector& bits, Ts&&... args) { columnInfo.emplace_back(odc::api::ColumnInfo{colname, odc::api::BITFIELD, sizeof(vals[0]), bits}); strides.emplace_back(odc::api::ConstStridedData{&vals[0], vals.size(), sizeof(vals[0]), sizeof(vals[0])}); quick_encode_internal(out, columnInfo, strides, std::forward(args)...); } template void quick_encode(eckit::DataHandle& out, Ts&&... args) { std::vector columnInfo; std::vector strides; quick_encode_internal(out, columnInfo, strides, std::forward(args)...); } // ------------------------------------------------------------------------------------------------------ // We need to specialise the comparison for FixedString, so that we can have different sized fixed // buffers containing the same string template bool vectorEquals(const std::vector& lhs, const std::vector& rhs) { return lhs == rhs; } template bool vectorEquals(const std::vector>& lhs, const std::vector>& rhs) { if (lhs.size() != rhs.size()) return false; for (size_t i = 0; i < lhs.size(); ++i) { if (lhs[i].asString() != rhs[i].asString()) return false; } return true; } template void decode_and_test_internal(eckit::DataHandle& in, std::vector& columnNames, std::vector& strides) { odc::api::Decoder decoder(columnNames, strides); in.openForRead(); eckit::AutoClose closer(in); // n.b. we don't use an aggregated reader, as we may well be able to extract the specified // columns, even if the overall structure cannot be aggregated. odc::api::Reader reader(in); size_t nrows = 0; while (nrows < strides[0].nelem()) { odc::api::Frame frame = reader.next(); EXPECT(frame); size_t frameRows = frame.rowCount(); odc::api::Decoder&& subDecoder = decoder.slice(nrows, frameRows); subDecoder.decode(frame); nrows += frameRows; } // We have consumed all the data EXPECT(!reader.next()); } template void decode_and_test_internal(eckit::DataHandle& in, std::vector& columnNames, std::vector& strides, const std::string& colname, const std::vector& correctVals, Ts&&... args) { std::vector decodeBuffer(correctVals.size()); ASSERT(decodeBuffer.size() == correctVals.size()); columnNames.push_back(colname); strides.emplace_back(&decodeBuffer[0], decodeBuffer.size(), sizeof(decodeBuffer[0]), sizeof(decodeBuffer[0])); decode_and_test_internal(in, columnNames, strides, std::forward(args)...); if (!vectorEquals(decodeBuffer, correctVals)) { Log::warning() << "Buffers mismatched for column " << colname << std::endl; ASSERT(decodeBuffer.size() == correctVals.size()); for (size_t i = 0; i < decodeBuffer.size(); ++i) { bool match = (decodeBuffer[i] == correctVals[i]); Log::warning() << (match ? " " : " *** ") << std::setw(15) << std::right << correctVals[i] << (match ? " == " : " != ") << std::setw(15) << std::left << decodeBuffer[i] << (match ? " " : " *** ") << std::endl; } } EXPECT(vectorEquals(decodeBuffer, correctVals)); } template void decode_and_test(eckit::DataHandle& in, Ts&&... args) { odc::api::Settings::treatIntegersAsDoubles(false); std::vector columnNames; std::vector strides; decode_and_test_internal(in, columnNames, strides, std::forward(args)...); } // ------------------------------------------------------------------------------------------------------ class DHOutputConfig : public eckit::sql::SQLOutputConfig { eckit::DataHandle& dh_; public: DHOutputConfig(eckit::DataHandle& dh) : eckit::sql::SQLOutputConfig(), dh_(dh) {} eckit::sql::SQLOutput* buildOutput(const eckit::PathName& path) const override { return new odc::sql::ODAOutput>(new odc::Writer<>(dh_)); } }; void oda_select_filter(const std::string& sql, eckit::DataHandle& dh_in, eckit::DataHandle& dh_out) { std::unique_ptr output_config(new DHOutputConfig(dh_out)); eckit::sql::SQLSession session{std::move(output_config)}; dh_in.openForRead(); eckit::AutoClose closer(dh_in); eckit::sql::SQLDatabase& db(session.currentDatabase()); db.addImplicitTable(new odc::sql::ODATable(db, dh_in)); ; eckit::sql::SQLParser parser; parser.parseString(session, sql); session.statement().execute(); } // ------------------------------------------------------------------------------------------------------ CASE("short and then longer strings") { double dmiss = odc::api::Settings::doubleMissingValue(); int64_t imiss = odc::api::Settings::integerMissingValue(); std::vector doubleVals{111.1, 222.2, dmiss, 333.3}; std::vector intVals{1, imiss, 2, 3}; std::vector> stringVals1{"a-string", "bbbb", "", "zzzzzzzz"}; std::vector> stringVals2{"a-stringa-string", "bbbbaaacccppp", "", "zzzzzzzz"}; // Construct the source odb eckit::MemoryHandle source_odb; { source_odb.openForWrite(0); eckit::AutoClose closer(source_odb); odc::api::Settings::treatIntegersAsDoubles(false); quick_encode(source_odb, "doublecol", doubleVals, "stringcol", stringVals1, "intcolumn", intVals); quick_encode(source_odb, "doublecol", doubleVals, "stringcol", stringVals2, "intcolumn", intVals); quick_encode(source_odb, "doublecol", doubleVals, "stringcol", stringVals1, "intcolumn", intVals); EXPECT(source_odb.position() == eckit::Offset(1289)); } EXPECT(source_odb.size() == eckit::Length(1289)); eckit::MemoryHandle output_odb; std::string select_string = "select *"; bool filtered = false; { SECTION("Select Class (integer internals)") { odc::api::Settings::treatIntegersAsDoubles(false); ::odc::api::filter(select_string, source_odb, output_odb); filtered = true; } SECTION("Select Class (double internals)") { odc::api::Settings::treatIntegersAsDoubles(true); ::odc::api::filter(select_string, source_odb, output_odb); filtered = true; } SECTION("ODAOutput (integer internals)") { odc::api::Settings::treatIntegersAsDoubles(false); oda_select_filter(select_string, source_odb, output_odb); filtered = true; } SECTION("ODAOutput (double internals)") { odc::api::Settings::treatIntegersAsDoubles(true); oda_select_filter(select_string, source_odb, output_odb); filtered = true; } } // Read back from the source ODB, and check that the contents are sane if (filtered) { std::vector doubleCorrect; std::vector intCorrect; for (int i = 0; i < 3; i++) { doubleCorrect.insert(doubleCorrect.end(), doubleVals.begin(), doubleVals.end()); intCorrect.insert(intCorrect.end(), intVals.begin(), intVals.end()); } std::vector> stringCorrect; stringCorrect.insert(stringCorrect.end(), stringVals1.begin(), stringVals1.end()); stringCorrect.insert(stringCorrect.end(), stringVals2.begin(), stringVals2.end()); stringCorrect.insert(stringCorrect.end(), stringVals1.begin(), stringVals1.end()); decode_and_test(output_odb, "doublecol", doubleCorrect, "stringcol", stringCorrect, "intcolumn", intCorrect); } } CASE("Extraction of bitfield values") { int64_t imiss = odc::api::Settings::integerMissingValue(); uint64_t uimiss = static_cast(odc::api::Settings::integerMissingValue()); std::vector bits1{{"bit1", 1, 0}, {"bit2", 2, 1}, {"bit3", 1, 3}}; std::vector bits2{{"bit4", 4, 0}, {"bit5", 2, 4}, {"bit6", 1, 6}}; std::vector bf1Vals{0, 5, uimiss, 15}; std::vector bf2Vals{127, uimiss, 42, 0}; std::vector bit1Vals{0, 1, imiss, 1}; std::vector bit2Vals{0, 2, imiss, 3}; std::vector bit3Vals{0, 0, imiss, 1}; std::vector bit4Vals{15, imiss, 10, 0}; std::vector bit5Vals{3, imiss, 2, 0}; std::vector bit6Vals{1, imiss, 0, 0}; // Construct the source odb static int cnt = 0; eckit::MemoryHandle source_odb; { source_odb.openForWrite(0); eckit::AutoClose closer(source_odb); odc::api::Settings::treatIntegersAsDoubles(false); quick_encode(source_odb, "bitfield1", bf1Vals, bits1, "bitfield2@table", bf2Vals, bits2); } // Select a selection of bits, both with and without the entire column /// @note - it would be really nice to test the SQL layer using treatIntegersAsDoubles(false). Unfortunately, it /// doesn't work, and can't cleanly work. eckit has to be agnostic to the odc interface, and it just /// passes values through (whether doubles, or integers punned into doubles). But to do bitfield /// extraction/manipulation, it really needs to know what form they are. This is hard coded to involve /// casting around doubles, which is really yucky. Bleurgh. /// TODO: Make the eckit layer use only proper integers internally... eckit::MemoryHandle output_odb; std::string select_string = "select bitfield1.bit1, bitfield1.bit3, bitfield1.bit2, bitfield1, bitfield2.bit5, bitfield2.bit4, " "bitfield2.bit6"; bool filtered = false; { #if 0 SECTION("Select Class (integer internals)") { odc::api::Settings::treatIntegersAsDoubles(false); EXPECT_THROWS_AS(::odc::api::filter(select_string, source_odb, output_odb), eckit::SeriousBug); } #endif SECTION("Select Class (double internals)") { odc::api::Settings::treatIntegersAsDoubles(true); ::odc::api::filter(select_string, source_odb, output_odb); filtered = true; } #if 0 SECTION( "ODAOutput (integer internals)") { odc::api::Settings::treatIntegersAsDoubles(false); EXPECT_THROWS_AS(oda_select_filter(select_string, source_odb, output_odb), eckit::SeriousBug); } #endif SECTION("ODAOutput (double internals)") { odc::api::Settings::treatIntegersAsDoubles(true); oda_select_filter(select_string, source_odb, output_odb); filtered = true; } } // Read back from the source ODB, and check that the contents are sane if (filtered) { decode_and_test(output_odb, "bitfield1.bit1", bit1Vals, "bitfield1.bit3", bit3Vals, "bitfield1.bit2", bit2Vals, "bitfield1", bf1Vals, "bitfield2.bit5", bit5Vals, "bitfield2.bit4", bit4Vals, "bitfield2.bit6", bit6Vals); } // Check if expansion of elements works, both with and without the table name as part of column identifier select_string = "select bitfield1.*, bitfield2.*@table"; filtered = false; { #if 0 SECTION("Select Class (integer internals)") { odc::api::Settings::treatIntegersAsDoubles(false); EXPECT_THROWS_AS(::odc::api::filter(select_string, source_odb, output_odb), eckit::SeriousBug); } #endif SECTION("Select Class (double internals)") { odc::api::Settings::treatIntegersAsDoubles(true); ::odc::api::filter(select_string, source_odb, output_odb); filtered = true; } #if 0 SECTION( "ODAOutput (integer internals)") { odc::api::Settings::treatIntegersAsDoubles(false); EXPECT_THROWS_AS(oda_select_filter(select_string, source_odb, output_odb), eckit::SeriousBug); } #endif SECTION("ODAOutput (double internals)") { odc::api::Settings::treatIntegersAsDoubles(true); oda_select_filter(select_string, source_odb, output_odb); filtered = true; } } // Read back from the source ODB, and check that the contents are sane if (filtered) { decode_and_test(output_odb, "bitfield1.bit1", bit1Vals, "bitfield1.bit3", bit3Vals, "bitfield1.bit2", bit2Vals, "bitfield2.bit5", bit5Vals, "bitfield2.bit4", bit4Vals, "bitfield2.bit6", bit6Vals); } } // ------------------------------------------------------------------------------------------------------ int main(int argc, char* argv[]) { return run_tests(argc, argv); } odc-1.6.3/tests/sql/test_functions.cc0000664000175000017500000002327715146027420020005 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/testing/Test.h" #include "eckit/types/FloatCompare.h" #include "odc/Select.h" #include "odc/Writer.h" #include "TemporaryFiles.h" using namespace eckit::testing; using eckit::types::is_approximately_equal; // ------------------------------------------------------------------------------------------------------ CASE("Various distance measuring functions return sensible things") { TemporaryFile f; // Write some data with a latitude and longitude. { odc::Writer<> oda(f.path()); odc::Writer<>::iterator row = oda.begin(); row->setNumberOfColumns(2); row->setColumn(0, "lat", odc::api::REAL); row->setColumn(1, "lon", odc::api::REAL); row->writeHeader(); // Include some extreme values (*row)[0] = 45.0; (*row)[1] = 0.0; ++row; (*row)[0] = 90.0; (*row)[1] = 180.0; ++row; (*row)[0] = -90.0; (*row)[1] = 360.0; ++row; (*row)[0] = -45.0; (*row)[1] = -90.0; ++row; } // And test that the SQL functions get the right data out!!! const std::string sql = std::string("select rad(45.0,0.0,1.0,lat,lon), ") + "rad(10.0,0.0,0.0,lat,lon), " + "distance(46.0,0.0,lat,lon), " + "km(46.0,0.0,lat,lon), " + "dist(100.,46.0,1.0,lat,lon), " + "dist(40.0,5.0,1000.0,lat,lon) " + "from \"" + f.path() + "\";"; const double eps = 7.e-6; { odc::Select oda(sql); odc::Select::iterator it = oda.begin(); EXPECT((*it)[0] == 1); // Inside relevant great-circle EXPECT((*it)[1] == 0); EXPECT(is_approximately_equal((*it)[2], 111120., eps)); // Surface distance to specified point EXPECT(is_approximately_equal((*it)[3], 111.12, eps)); // In m EXPECT((*it)[4] == 0); // Within specified distance of given point EXPECT((*it)[5] == 1); ++it; EXPECT((*it)[0] == 0); // Inside relevant great-circle EXPECT((*it)[1] == 0); // pi/4 * R EXPECT(is_approximately_equal((*it)[2], 4889280., eps)); // Surface distance to specified point EXPECT(is_approximately_equal((*it)[3], 4889.28, eps)); // In m EXPECT((*it)[4] == 0); // Within specified distance of given point EXPECT((*it)[5] == 0); ++it; EXPECT((*it)[0] == 0); // Inside relevant great-circle EXPECT((*it)[1] == 0); // 3*pi/4 * R EXPECT(is_approximately_equal((*it)[2], 15112320., eps)); // Surface distance to specified point EXPECT(is_approximately_equal((*it)[3], 15112.32, eps)); // In m EXPECT((*it)[4] == 0); // Within specified distance of given point EXPECT((*it)[5] == 0); ++it; EXPECT((*it)[0] == 0); // Inside relevant great-circle EXPECT((*it)[1] == 0); EXPECT(is_approximately_equal((*it)[2], 13398177.5541776344, eps)); // Surface distance to specified point EXPECT(is_approximately_equal((*it)[3], 13398.1775541776344, eps)); // In m EXPECT((*it)[4] == 0); // Within specified distance of given point EXPECT((*it)[5] == 0); ++it; } // Test against things measured from the north and south poles const std::string sql2 = std::string( "select " "distance(90.0,0.0,lat,lon), " "km(90.0,0.0,lat,lon), " "distance(-90.0,0.0,lat,lon), " "km(-90.0,0.0,lat,lon), " "distance(0.0,180.0,lat,lon), " "km(0.0,180.0,lat,lon), " "from \"") + f.path() + "\";"; { odc::Select oda(sql2); odc::Select::iterator it = oda.begin(); // pi/4 * R_e EXPECT(is_approximately_equal((*it)[0], 5000400., eps)); EXPECT(is_approximately_equal((*it)[1], 5000.4, eps)); // 3*pi/4 * R_e EXPECT(is_approximately_equal((*it)[2], 15001200., eps)); EXPECT(is_approximately_equal((*it)[3], 15001.2, eps)); // 3*pi/4 * R_e EXPECT(is_approximately_equal((*it)[4], 15001200., eps)); EXPECT(is_approximately_equal((*it)[5], 15001.2, eps)); ++it; EXPECT(is_approximately_equal((*it)[0], 0., eps)); EXPECT(is_approximately_equal((*it)[1], 0., eps)); // pi * R_e EXPECT(is_approximately_equal((*it)[2], 20001600., eps)); EXPECT(is_approximately_equal((*it)[3], 20001.6, eps)); // pi / 2 * R_e EXPECT(is_approximately_equal((*it)[4], 10000800., eps)); EXPECT(is_approximately_equal((*it)[5], 10000.8, eps)); ++it; EXPECT(is_approximately_equal((*it)[0], 20001600., eps)); EXPECT(is_approximately_equal((*it)[1], 20001.6, eps)); // pi * R_e EXPECT(is_approximately_equal((*it)[2], 0., eps)); EXPECT(is_approximately_equal((*it)[3], 0., eps)); // pi / 2 * R_e EXPECT(is_approximately_equal((*it)[4], 10000800., eps)); EXPECT(is_approximately_equal((*it)[5], 10000.8, eps)); ++it; // 3*pi/4 * R_e EXPECT(is_approximately_equal((*it)[0], 15001200., eps)); EXPECT(is_approximately_equal((*it)[1], 15001.2, eps)); // pi/4 * R_e EXPECT(is_approximately_equal((*it)[2], 5000400., eps)); EXPECT(is_approximately_equal((*it)[3], 5000.4, eps)); // pi / 2 * R_e EXPECT(is_approximately_equal((*it)[4], 10000800., eps)); EXPECT(is_approximately_equal((*it)[5], 10000.8, eps)); ++it; } } CASE("Inside or outside detection works for circles") { TemporaryFile f; { odc::Writer<> oda(f.path()); odc::Writer<>::iterator row = oda.begin(); row->setNumberOfColumns(2); row->setColumn(0, "x", odc::api::REAL); row->setColumn(1, "y", odc::api::REAL); row->writeHeader(); (*row)[0] = 45.0; (*row)[1] = 10.0; ++row; (*row)[0] = 0.0; (*row)[1] = 0.0; ++row; (*row)[0] = -45.5; (*row)[1] = 37.4; ++row; (*row)[0] = 45.5; (*row)[1] = -37.4; ++row; } // And test that the SQL functions get the right data out!!! const std::string sql = std::string("select ") + "circle(x, 46.0, y, 11.0, 1.0), " "circle(x, 46.0, y, 11.0, 1.5), " "circle(x, 0.0, y, 0.0, 1.0), " "circle(x, 0.0, y, 0.0, -1.0), " "circle(x, 0.0, y, 0.0, 0.0), " // If we are on a point, a radius of zero is OK! "circle(x, 45.5, y, -37.4, 117.7966), " "circle(x, 45.5, y, -37.4, 117.7967) " "from \"" + f.path() + "\";"; { odc::Select oda(sql); odc::Select::iterator it = oda.begin(); EXPECT((*it)[0] == 0); EXPECT((*it)[1] == 1); EXPECT((*it)[2] == 0); EXPECT((*it)[3] == 0); EXPECT((*it)[4] == 0); EXPECT((*it)[5] == 1); EXPECT((*it)[6] == 1); ++it; EXPECT((*it)[0] == 0); EXPECT((*it)[1] == 0); EXPECT((*it)[2] == 1); EXPECT((*it)[3] == 1); EXPECT((*it)[4] == 1); EXPECT((*it)[5] == 1); EXPECT((*it)[6] == 1); ++it; EXPECT((*it)[0] == 0); EXPECT((*it)[1] == 0); EXPECT((*it)[2] == 0); EXPECT((*it)[3] == 0); EXPECT((*it)[4] == 0); EXPECT((*it)[5] == 0); EXPECT((*it)[6] == 1); ++it; EXPECT((*it)[0] == 0); EXPECT((*it)[1] == 0); EXPECT((*it)[2] == 0); EXPECT((*it)[3] == 0); EXPECT((*it)[4] == 0); EXPECT((*it)[5] == 1); EXPECT((*it)[6] == 1); } } CASE("Norms are correctly calculated") { TemporaryFile f; { odc::Writer<> oda(f.path()); odc::Writer<>::iterator row = oda.begin(); row->setNumberOfColumns(2); row->setColumn(0, "x", odc::api::REAL); row->setColumn(1, "y", odc::api::REAL); row->writeHeader(); (*row)[0] = 3.0; (*row)[1] = 16.0; ++row; (*row)[0] = 4.0; (*row)[1] = 12.0; ++row; (*row)[0] = 2.0; (*row)[1] = 24.0; ++row; } // And test that the SQL functions get the right data out!!! // See ODB-382 for buggy behaviour. const std::string sql = std::string("select ") + "norm(x, y) " "from \"" + f.path() + "\";"; { odc::Select oda(sql); odc::Select::iterator it = oda.begin(); // Norm is an aggregate function that calculates sqrt(x . y) for the // entire columns of data EXPECT(is_approximately_equal((*it)[0], 12.)); ++it; } } // ------------------------------------------------------------------------------------------------------ int main(int argc, char* argv[]) { return run_tests(argc, argv); } odc-1.6.3/tests/tools/0000775000175000017500000000000015146027420014755 5ustar alastairalastairodc-1.6.3/tests/tools/test_odb_header.sh0000775000175000017500000000231415146027420020427 0ustar alastairalastair#!/bin/bash set -uex # A unique working directory wd=$(pwd) test_wd=$(pwd)/test_odb_header mkdir -p ${test_wd} cd ${test_wd} # In case we are resuming from a previous failed run, which has left output in the directory rm *.odb || true # Create some test data cat > data.csv < header.txt cat > header-expected.txt < data.csv < expect.csv < data.csv < expect.csv < ascii_out1.txt odc sql 'select *' -i data.odb -f default > ascii_out2.txt odc sql 'select *' -i data.odb -o odb_out1.odb odc sql 'select *' -i data.odb -o odb_out2.odb -f default cat > ascii_reference.txt < wide_out1.txt odc sql 'select *' -i data.odb -o wide_out2.txt -f wide cat > wide_reference.txt < schema.hh < data.csv < expect.csv < expect.csv < expect.csv < expect.csv < expect.csv < expect.csv < expect.csv < data.csv < odb_ls_reference < odb_ls cmp odb_ls_reference odb_ls # Clean up cd ${wd} rm -rf ${test_wd} odc-1.6.3/tests/tools/test_odb_sql_bitfields.sh0000775000175000017500000000516515146027420022032 0ustar alastairalastair#!/bin/bash set -uex # A unique working directory wd=$(pwd) test_wd=$(pwd)/test_odb_sql_bitfields mkdir -p ${test_wd} cd ${test_wd} # In case we are resuming from a previous failed run, which has left output in the directory rm *.odb || true # Create some test data cat > data.csv < odb_ls_reference < odb_ls cmp odb_ls_reference odb_ls # Check if selecting whole bitfield column works, both with and without table identifier cat > odb_select_reference < odb_select_reference < odb_select_reference < data-1-2.csv < data-3.csv < ARGS ${_test} ENVIRONMENT ${test_environment} LABELS odc TEST_DEPENDS ${_dependencies}) set( _prev_test ${_test} ) endforeach() ecbuild_add_executable( TARGET test_client_lib_fortran CONDITION HAVE_FORTRAN AND HAVE_ODB SOURCES test_client_lib_fortran.f90 LIBS odc_fortran LINKER_LANGUAGE Fortran) ecbuild_add_executable( TARGET test_client_lib_cpp CONDITION HAVE_ODB SOURCES test_client_lib_cpp.cc LIBS odccore odctools ) include_directories( ${CMAKE_CURRENT_SOURCE_DIR} ) add_subdirectory( core ) add_subdirectory( sql ) add_subdirectory( tools ) add_subdirectory( api ) add_subdirectory( c_api ) add_subdirectory( f_api ) odc-1.6.3/tests/core/0000775000175000017500000000000015146027420014545 5ustar alastairalastairodc-1.6.3/tests/core/test_reencode_string_table.cc0000664000175000017500000001353515146027420022443 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include #include #include "eckit/eckit_ecbuild_config.h" #include "eckit/system/SystemInfo.h" #include "eckit/testing/Test.h" #include "odc/api/ColumnType.h" #include "odc/codec/String.h" #include "odc/core/Codec.h" using namespace eckit::testing; using namespace odc::core; using namespace odc::codec; // ------------------------------------------------------------------------------------------------------ // TODO with codecs: // // i) Make them templated on the stream/datahandle directly // ii) Construct them with a specific data handle/stream // iii) Why are we casting data handles via a void* ??? // Given the codec-initialising data, add the header on that is used to construct the // codec. size_t prepend_codec_selection_header(std::vector& data, const std::string& codec_name, bool bigEndian = false) { data.insert(data.begin(), 4, 0); data[bigEndian ? 3 : 0] = static_cast(codec_name.size()); data.insert(data.begin() + 4, codec_name.begin(), codec_name.end()); return 4 + codec_name.length(); } CASE("Character strings can be stored in a flat list, and indexed") { // n.b. no missing values const char* source_data[] = { // Codec header "\x00\x00\x00\x00", // 0 = hasMissing "\x00\x00\x00\x00\x00\x00\x00\x00", // min unspecified "\x00\x00\x00\x00\x00\x00\x00\x00", // max unspecified "\x00\x00\x00\x00\x00\x00\x00\x00", // missingValue unspecified // How many strings are there in the table? "\x06\x00\x00\x00", // String data (prepended with lengths) // length, data, "cnt (discarded)", index "\x08\x00\x00\x00", "ghijklmn", "\x00\x00\x00\x00", "\x00\x00\x00\x00", "\x0c\x00\x00\x00", "uvwxyzabcdef", "\x00\x00\x00\x00", "\x01\x00\x00\x00", // too long "\x08\x00\x00\x00", "opqrstuv", "\x00\x00\x00\x00", "\x02\x00\x00\x00", "\x02\x00\x00\x00", "ab", "\x00\x00\x00\x00", "\x03\x00\x00\x00", // This string is too short "\x06\x00\x00\x00", "ghijkl", "\x00\x00\x00\x00", "\x04\x00\x00\x00", "\x08\x00\x00\x00", "mnopqrst", "\x00\x00\x00\x00", "\x05\x00\x00\x00", // 8-byte length }; // Loop throumgh endiannesses for the source data for (int i = 0; i < 4; i++) { bool bigEndianSource = (i % 2 == 0); bool bits16 = (i > 1); std::vector data; for (size_t j = 0; j < sizeof(source_data) / sizeof(const char*); j++) { size_t len = (j < 5) ? ((j == 0 || j == 4) ? 4 : 8) : ((j + 2) % 4 == 0 ? ::strlen(source_data[j]) : 4); data.insert(data.end(), source_data[j], source_data[j] + len); // n.b. Don't reverse the endianness of the string data. if (bigEndianSource && !((j > 5) && ((j + 2) % 4 == 0))) std::reverse(data.end() - len, data.end()); } // Which strings do we wish to decode (look at them in reverse. nb refers to index column) for (int n = 5; n >= 0; n--) { if (bits16 && bigEndianSource) data.push_back(0); data.push_back(static_cast(n)); if (bits16 && !bigEndianSource) data.push_back(0); } // Construct codec directly, and decode the header // Skip name of codec GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size()); std::unique_ptr c; if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) { if (bits16) { c.reset(new CodecInt16String(odc::api::STRING)); } else { c.reset(new CodecInt8String(odc::api::STRING)); } } else { if (bits16) { c.reset(new CodecInt16String(odc::api::STRING)); } else { c.reset(new CodecInt8String(odc::api::STRING)); } } c->load(ds); EXPECT(ds.position() == eckit::Offset(148)); // Now re-encode the codec header, and check that we get what we started with! eckit::Buffer writeBuffer(4096); ::memset(writeBuffer, 0, writeBuffer.size()); GeneralDataStream ds2(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), writeBuffer); c->save(ds2); // Check that the data is the same both times! EXPECT(ds2.position() == eckit::Offset(148)); // eckit::Log::info() << "DATA: " << std::endl; // for (size_t n = 0; n < data.size(); n++) { // eckit::Log::info() << std::hex << int(data[n]) << " " << int(dh_write.getBuffer()[n]) << // std::endl; // if (int(data[n]) != int(dh_write.getBuffer()[n])) // eckit::Log::info() << "******************************" << std::endl; // } // The header should be correctly re-encoded. EXPECT(::memcmp(writeBuffer, &data[0], 148) == 0); // We haven't encoded the data itself for (size_t i = 148; i < 154; i++) { EXPECT(writeBuffer[i] == 0); } } } // ------------------------------------------------------------------------------------------------------ int main(int argc, char* argv[]) { return run_tests(argc, argv); } odc-1.6.3/tests/core/test_concatenated_odbs.cc0000664000175000017500000002621715146027420021562 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/io/FileHandle.h" #include "eckit/io/MemoryHandle.h" #include "eckit/io/MultiHandle.h" #include "eckit/testing/Test.h" #include "TemporaryFiles.h" #include "odc/Comparator.h" #include "odc/Reader.h" #include "odc/Writer.h" #include "odc/core/Exceptions.h" #include using namespace eckit::testing; // ------------------------------------------------------------------------------------------------------ CASE("ODBs concatenated in a file are valid (columns change)") { SETUP("Write multiple ODBs, then concatenate them together") { // Construct four ODBs, with differing characteristics class TemporaryODB1 : public TemporaryFile { public: TemporaryODB1() { odc::Writer<> oda(path()); odc::Writer<>::iterator writer = oda.begin(); writer->setNumberOfColumns(3); writer->setColumn(0, "x", odc::api::REAL); writer->setColumn(1, "y", odc::api::REAL); writer->setColumn(2, "z", odc::api::INTEGER); writer->writeHeader(); for (size_t i = 1; i <= 2; i++) { (*writer)[0] = i; // col 0 (*writer)[1] = i; // col 1 (*writer)[2] = i; // col 2 ++writer; } } }; // Change column names class TemporaryODB2 : public TemporaryFile { public: TemporaryODB2() { odc::Writer<> oda(path()); odc::Writer<>::iterator writer = oda.begin(); writer->setNumberOfColumns(3); writer->setColumn(0, "x", odc::api::REAL); writer->setColumn(1, "y", odc::api::INTEGER); writer->setColumn(2, "v", odc::api::REAL); writer->writeHeader(); for (size_t i = 1; i <= 2; i++) { (*writer)[0] = i * 10; // col 0 (*writer)[1] = i * 100; // col 1 (*writer)[2] = i * 1000; // col 2 ++writer; } } }; // Increase number of columns class TemporaryODB3 : public TemporaryFile { public: TemporaryODB3() { odc::Writer<> oda(path()); odc::Writer<>::iterator writer = oda.begin(); writer->setNumberOfColumns(4); writer->setColumn(0, "x", odc::api::INTEGER); writer->setColumn(1, "v", odc::api::REAL); writer->setColumn(2, "y", odc::api::REAL); writer->setColumn(3, "z", odc::api::REAL); writer->writeHeader(); for (size_t i = 1; i <= 2; i++) { (*writer)[0] = i * 10; // col 0 (*writer)[1] = i * 1000; // col 1 (*writer)[2] = i * 100; // col 2 (*writer)[3] = 13; // col 3 ++writer; } } }; // Decrease number of columns class TemporaryODB4 : public TemporaryFile { public: TemporaryODB4() { odc::Writer<> oda(path()); odc::Writer<>::iterator writer = oda.begin(); writer->setNumberOfColumns(2); writer->setColumn(0, "x", odc::api::REAL); writer->setColumn(1, "v", odc::api::REAL); writer->writeHeader(); for (size_t i = 1; i <= 2; i++) { (*writer)[0] = i * 5; // col 0 (*writer)[1] = i * 7; // col 1 ++writer; } } }; TemporaryODB1 tmpODB1; TemporaryODB2 tmpODB2; TemporaryODB3 tmpODB3; TemporaryODB4 tmpODB4; // Directly concatenate files (append them to the first one) TemporaryFile combinedFile; { std::vector readHandles; readHandles.push_back(new eckit::FileHandle(tmpODB1.path())); readHandles.push_back(new eckit::FileHandle(tmpODB2.path())); readHandles.push_back(new eckit::FileHandle(tmpODB3.path())); readHandles.push_back(new eckit::FileHandle(tmpODB4.path())); eckit::MultiHandle aggregateHandle(readHandles); eckit::FileHandle out_handle(combinedFile.path()); aggregateHandle.saveInto(out_handle); } SECTION("The data in the concatenated files is correct") { odc::Reader in(combinedFile.path()); odc::Reader::iterator it = in.begin(); EXPECT(it->columns().size() == 3); EXPECT(it->columns()[0]->name() == "x"); EXPECT(it->columns()[1]->name() == "y"); EXPECT(it->columns()[2]->name() == "z"); EXPECT(it->columns().rowsNumber() == 2); for (size_t i = 1; i < 3; i++) { for (size_t j = 0; j < 3; j++) { EXPECT((*it)[j] == i); } ++it; } // Check that we have changed the name of a column EXPECT(it->columns().size() == 3); EXPECT(it->columns()[0]->name() == "x"); EXPECT(it->columns()[1]->name() == "y"); EXPECT(it->columns()[2]->name() == "v"); EXPECT(!it->columns().hasColumn("z")); EXPECT(it->columns().rowsNumber() == 2); for (size_t i = 1; i < 3; i++) { EXPECT((*it)[0] == 10 * i); EXPECT((*it)[1] == 100 * i); EXPECT((*it)[2] == 1000 * i); ++it; } // Now we have 4 columns EXPECT(it->columns().size() == 4); EXPECT(it->columns()[0]->name() == "x"); EXPECT(it->columns()[1]->name() == "v"); EXPECT(it->columns()[2]->name() == "y"); EXPECT(it->columns()[3]->name() == "z"); EXPECT(it->columns().rowsNumber() == 2); for (size_t i = 1; i < 3; i++) { EXPECT((*it)[0] == 10 * i); EXPECT((*it)[1] == 1000 * i); EXPECT((*it)[2] == 100 * i); ++it; } // And back to 2 columns EXPECT(it->columns().size() == 2); EXPECT(it->columns()[0]->name() == "x"); EXPECT(it->columns()[1]->name() == "v"); EXPECT(it->columns().rowsNumber() == 2); for (size_t i = 1; i < 3; i++) { EXPECT((*it)[0] == 5 * i); EXPECT((*it)[1] == 7 * i); ++it; } } SECTION("A copy of the concatenated file is identical") { odc::Reader in(combinedFile.path()); odc::Reader::iterator it = in.begin(); odc::Reader::iterator end = in.end(); TemporaryFile copyFile; odc::Writer<> out(copyFile.path()); odc::Writer<>::iterator o = out.begin(); o->pass1(it, end); // Check that ODB-API thinks the files are the same EXPECT_NO_THROW(odc::Comparator().compare(combinedFile.path(), copyFile.path())); } } } CASE("If corrupt data follows a valid ODB this should not be treated as a new ODB") { // See ODB-376 // Construct a valid ODB in a buffer, followed by some invalid data eckit::Buffer buf(4096); eckit::MemoryHandle writeDH(buf); { odc::Writer<> oda(writeDH); odc::Writer<>::iterator writer = oda.begin(); writer->setNumberOfColumns(3); writer->setColumn(0, "x", odc::api::REAL); writer->setColumn(1, "y", odc::api::REAL); writer->setColumn(2, "z", odc::api::INTEGER); writer->writeHeader(); for (size_t i = 1; i <= 2; i++) { (*writer)[0] = i; // col 0 (*writer)[1] = i; // col 1 (*writer)[2] = i; // col 2 ++writer; } } // And write some invalid data on the end of the buffer const uint32_t invalid_data = 0xBAADF00D; writeDH.write(&invalid_data, sizeof(invalid_data)); // Now read the data. We should get the data back, and then an error... eckit::MemoryHandle readDH(buf); readDH.openForRead(); eckit::AutoClose close(readDH); odc::Reader in(readDH); odc::Reader::iterator it = in.begin(); EXPECT(static_cast(it->data()[0]) == 1); EXPECT(static_cast(it->data()[1]) == 1); EXPECT(static_cast(it->data()[2]) == 1); ++it; EXPECT(static_cast(it->data()[0]) == 2); EXPECT(static_cast(it->data()[1]) == 2); EXPECT(static_cast(it->data()[2]) == 2); // Where we would expect an EOF, or a new table, we now have corrupt data. This increment should // NOT succeed, but should complain vociferously!!! EXPECT_THROWS_AS(++it, odc::core::ODBInvalid); } CASE("If a corrupted ODB (with no row data following the header) then report an error") { // See ODB-376 // Construct a valid ODB in a buffer, followed by some invalid data eckit::Buffer buf(4096); eckit::MemoryHandle writeDH(buf); { odc::Writer<> oda(writeDH); odc::Writer<>::iterator writer = oda.begin(); writer->setNumberOfColumns(3); writer->setColumn(0, "x", odc::api::REAL); writer->setColumn(1, "y", odc::api::REAL); writer->setColumn(2, "z", odc::api::INTEGER); writer->writeHeader(); for (size_t i = 1; i <= 2; i++) { (*writer)[0] = i; // col 0 (*writer)[1] = i; // col 1 (*writer)[2] = i; // col 2 ++writer; } } // The header size is 320 bytes. Copy the data from the start... // writeDH.write(buf.data(), 322); writeDH.write(buf.data(), 322 - 80); // -80 == we no longer encode (empty) flags // Now read the data. We should get the data back, and then an error... eckit::MemoryHandle readDH(buf.data(), static_cast(writeDH.position())); readDH.openForRead(); eckit::AutoClose close(readDH); odc::Reader in(readDH); odc::Reader::iterator it = in.begin(); EXPECT(static_cast(it->data()[0]) == 1); EXPECT(static_cast(it->data()[1]) == 1); EXPECT(static_cast(it->data()[2]) == 1); ++it; EXPECT(static_cast(it->data()[0]) == 2); EXPECT(static_cast(it->data()[1]) == 2); EXPECT(static_cast(it->data()[2]) == 2); // Where we would expect an EOF, or a new table, we now have corrupt data. This increment should // NOT succeed, but should complain vociferously!!! EXPECT_THROWS_AS(++it, eckit::SeriousBug); } // ------------------------------------------------------------------------------------------------------ int main(int argc, char* argv[]) { return run_tests(argc, argv); } odc-1.6.3/tests/core/test_text_reader.cc0000664000175000017500000002122715146027420020425 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include #include "eckit/testing/Test.h" #include "odc/csv/TextReader.h" #include "odc/csv/TextReaderIterator.h" using namespace eckit::testing; // ------------------------------------------------------------------------------------------------------ CASE("Read columnar data from CSV") { // n.b. have the first long string not be in the first row, which will force the mechanism // to resize. std::stringstream data; data << "col1:INTEGER,col2:REAL,col3:DOUBLE,col4:STRING,col5:BITFIELD[a:1;b:2;c:5]\n"; data << "1,1.001,0,a-string,0\n"; data << "1234,0,88,b-string,2\n"; data << "-5432,-6.543210,99.999,string-c,4\n"; data << "-2147483648,6.543210,11.63e-37,testing,7\n"; data << "2147483647,NaN,Nan,12345678,8\n"; data << "0,+inf,-inf,this-is-a-longer-string,11\n"; data << "0,-inf,0,short,0\n"; odc::TextReader reader(data, ","); odc::TextReader::iterator it = reader.begin(); std::vector INTEGERS{1, 1234, -5432, -2147483648, 2147483647, 0, 0}; std::vector REALS{1.001, 0.0, -6.543210, 6.543210, std::numeric_limits::quiet_NaN(), std::numeric_limits::infinity(), -std::numeric_limits::infinity()}; std::vector DOUBLES{ 0, 88, 99.999, 11.63e-37, std::numeric_limits::quiet_NaN(), -std::numeric_limits::infinity(), 0}; std::vector STRINGS{ "a-string", "b-string", "string-c", "testing", "12345678", "this-is-a-longer-string", "short"}; std::vector BITFIELDS{0, 2, 4, 7, 8, 11, 0}; EXPECT(it->columns().size() == 5); EXPECT(it->columns()[0]->name() == "col1"); EXPECT(it->columns()[1]->name() == "col2"); EXPECT(it->columns()[2]->name() == "col3"); EXPECT(it->columns()[3]->name() == "col4"); EXPECT(it->columns()[4]->name() == "col5"); EXPECT(it->columns()[0]->type() == odc::api::INTEGER); EXPECT(it->columns()[1]->type() == odc::api::REAL); EXPECT(it->columns()[2]->type() == odc::api::DOUBLE); EXPECT(it->columns()[3]->type() == odc::api::STRING); EXPECT(it->columns()[4]->type() == odc::api::BITFIELD); size_t count = 0; for (; it != reader.end(); ++it) { ++count; // Only resize on new, longer string EXPECT(it->isNewDataset() == (count == 6)); EXPECT(it->data(0) == INTEGERS[count - 1]); EXPECT(it->data(1) == double(REALS[count - 1]) || (std::isnan(it->data(1)) && std::isnan(REALS[count - 1]))); EXPECT(it->data(2) == DOUBLES[count - 1] || (std::isnan(it->data(2)) && std::isnan(DOUBLES[count - 1]))); EXPECT(it->dataSizeDoubles(3) == ((count >= 6) ? 3 : 1)); EXPECT(::strncmp(STRINGS[count - 1].c_str(), (char*)&it->data(3), it->dataSizeDoubles(3) * sizeof(double)) == 0); EXPECT(it->data(4) == BITFIELDS[count - 1]); for (const auto& col : it->columns()) { EXPECT(!col->hasMissing()); } } EXPECT(count == 7); } CASE("Starting with long strings") { // n.b. have the first long string not be in the first row, which will force the mechanism // to resize. std::stringstream data; data << "col4:STRING\n"; data << "a-string-is-long\n"; data << "b-string-is-very-long-indeed-whoah\n"; odc::TextReader reader(data, ","); odc::TextReader::iterator it = reader.begin(); std::vector STRINGS{"a-string-is-long", "b-string-is-very-long-indeed-whoah"}; EXPECT(it->columns().size() == 1); EXPECT(it->columns()[0]->name() == "col4"); EXPECT(it->columns()[0]->type() == odc::api::STRING); size_t count = 0; for (; it != reader.end(); ++it) { ++count; // Only resize on new, longer string EXPECT(it->isNewDataset()); EXPECT(it->dataSizeDoubles(0) == (count == 1 ? 2 : 5)); EXPECT(::strncmp(STRINGS[count - 1].c_str(), (char*)&it->data(0), it->dataSizeDoubles(0) * sizeof(double)) == 0); for (const auto& col : it->columns()) { EXPECT(!col->hasMissing()); } } EXPECT(count == 2); } CASE("Test parsing bitfields") { std::string bitfieldDefinition = "en4_level_flag@hdr:bitfield[TempLevelReject:1;SaltLevelReject:1;LevelVertStability:1;IncreasingDepthCheck:1;" "NotUsed1:1;NotUsed2:1;NotUsed3:1;NotUsed4:1;NotUsed5:1;TempLevelStatList:1;TempLevelArgoQC:1;" "TempLevelOutOfRangeSetToMDI:1;TempLevelEN3List:1;TempLevelVertCheck:1;TempLevelNoBckgrnd:1;TempLevelBays:1;" "TempLevelBaysBud:1;TempLevelBaysBudReinstate:1;TempLevelWaterfallCheck:1;NotUsed6:1;NotUsed7:1;" "SaltLevelStatList:1;SaltLevelArgoQC:1;SaltLevelOutOfRangeSetToMDI:1;SaltLevelEN3List:1;SaltLevelVertCheck:1;" "SaltLevelNoBckgrnd:1;SaltLevelBays:1;SaltLevelBaysBud:1;SaltLevelBaysBudReinstate:1;SaltLevelWaterfallCheck:" "1]"; eckit::sql::BitfieldDef def(odc::TextReaderIterator::parseBitfields(bitfieldDefinition)); eckit::sql::FieldNames names(def.first); eckit::sql::Sizes sizes(def.second); std::vector FIELD_NAMES{"TempLevelReject", "SaltLevelReject", "LevelVertStability", "IncreasingDepthCheck", "NotUsed1", "NotUsed2", "NotUsed3", "NotUsed4", "NotUsed5", "TempLevelStatList", "TempLevelArgoQC", "TempLevelOutOfRangeSetToMDI", "TempLevelEN3List", "TempLevelVertCheck", "TempLevelNoBckgrnd", "TempLevelBays", "TempLevelBaysBud", "TempLevelBaysBudReinstate", "TempLevelWaterfallCheck", "NotUsed6", "NotUsed7", "SaltLevelStatList", "SaltLevelArgoQC", "SaltLevelOutOfRangeSetToMDI", "SaltLevelEN3List", "SaltLevelVertCheck", "SaltLevelNoBckgrnd", "SaltLevelBays", "SaltLevelBaysBud", "SaltLevelBaysBudReinstate", "SaltLevelWaterfallCheck"}; ASSERT(names.size() == 31); ASSERT(sizes.size() == 31); EXPECT(names == FIELD_NAMES); EXPECT(std::all_of(sizes.begin(), sizes.end(), [](int x) { return x == 1; })); } CASE("Test parsing bitfields - 32bit limit") { std::string bitfieldDefinition = "en4_level_flag@hdr:bitfield[TempLevelReject:1;SaltLevelReject:1;LevelVertStability:1;IncreasingDepthCheck:1;" "NotUsed1:1;NotUsed2:1;NotUsed3:1;NotUsed4:1;NotUsed5:1;TempLevelStatList:1;TempLevelArgoQC:1;" "TempLevelOutOfRangeSetToMDI:1;TempLevelEN3List:1;TempLevelVertCheck:1;TempLevelNoBckgrnd:1;TempLevelBays:1;" "TempLevelBaysBud:1;TempLevelBaysBudReinstate:1;TempLevelWaterfallCheck:1;NotUsed6:1;NotUsed7:1;" "SaltLevelStatList:1;SaltLevelArgoQC:1;SaltLevelOutOfRangeSetToMDI:1;SaltLevelEN3List:1;SaltLevelVertCheck:1;" "SaltLevelNoBckgrnd:1;SaltLevelBays:1;SaltLevelBaysBud:1;SaltLevelBaysBudReinstate:1;SaltLevelWaterfallCheck:1;" "NotUsed8:1;NotUsed9:1]"; EXPECT_THROWS_AS(odc::TextReaderIterator::parseBitfields(bitfieldDefinition), eckit::UserError); } // ------------------------------------------------------------------------------------------------------ int main(int argc, char* argv[]) { return run_tests(argc, argv); } odc-1.6.3/tests/core/test_initial_missing.cc0000664000175000017500000003466615146027420021314 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/log/Log.h" #include "eckit/testing/Test.h" #include "odc/Reader.h" #include "odc/api/Odb.h" using namespace eckit::testing; namespace { struct TestIntegerDecoding { TestIntegerDecoding() { odc::api::Settings::treatIntegersAsDoubles(false); } ~TestIntegerDecoding() { odc::api::Settings::treatIntegersAsDoubles(true); } }; } // namespace // ------------------------------------------------------------------------------------------------------ namespace { int64_t integer_cast(const double& val) { static_assert(sizeof(double) == sizeof(int64_t), "Punning is not valid"); auto punnable_val = reinterpret_cast(&val); auto punned_val = reinterpret_cast(punnable_val); return *punned_val; } } // namespace CASE("Test all-missing first row with Decoder integers-as-doubles") { odc::api::Reader reader("odb_533_1.odb", /* aggregated */ false); odc::api::Frame frame = reader.next(); EXPECT(frame.columnCount() == 4); EXPECT(frame.rowCount() == 2); EXPECT(frame.columnInfo()[0].name == "stringval"); EXPECT(frame.columnInfo()[1].name == "intval"); EXPECT(frame.columnInfo()[2].name == "realval"); EXPECT(frame.columnInfo()[3].name == "doubleval"); constexpr size_t nrows = 2; char stringvals[nrows][8]; double intvals[nrows]; double realvals[nrows]; double doublevals[nrows]; // n.b. columns in reverse order, to be awkward std::vector columns{"doubleval", "realval", "intval", "stringval"}; std::vector strides{ {doublevals, nrows, sizeof(doublevals[0]), sizeof(doublevals[0])}, {realvals, nrows, sizeof(realvals[0]), sizeof(realvals[0])}, {intvals, nrows, sizeof(intvals[0]), sizeof(intvals[0])}, {stringvals, nrows, sizeof(stringvals[0]), sizeof(stringvals[0])}, }; odc::api::Decoder decoder(columns, strides); decoder.decode(frame); EXPECT(::memcmp(stringvals[0], "\0\0\0\0\0\0\0\0", 8) == 0); EXPECT(intvals[0] == odc::api::Settings::integerMissingValue()); EXPECT(realvals[0] == odc::api::Settings::doubleMissingValue()); EXPECT(doublevals[0] == odc::api::Settings::doubleMissingValue()); EXPECT(::memcmp(stringvals[1], "testing\0", 8) == 0); EXPECT(intvals[1] == 12345678); EXPECT(realvals[1] == 1234.56); EXPECT(doublevals[1] == 9876.54); EXPECT(!reader.next()); } CASE("Test all-missing first row with Decoder integers-as-integers") { TestIntegerDecoding integers; odc::api::Reader reader("odb_533_1.odb", /* aggregated */ false); odc::api::Frame frame = reader.next(); EXPECT(frame.columnCount() == 4); EXPECT(frame.rowCount() == 2); EXPECT(frame.columnInfo()[0].name == "stringval"); EXPECT(frame.columnInfo()[1].name == "intval"); EXPECT(frame.columnInfo()[2].name == "realval"); EXPECT(frame.columnInfo()[3].name == "doubleval"); constexpr size_t nrows = 2; char stringvals[nrows][8]; int64_t intvals[nrows]; double realvals[nrows]; double doublevals[nrows]; // n.b. columns in reverse order, to be awkward std::vector columns{"doubleval", "realval", "intval", "stringval"}; std::vector strides{ {doublevals, nrows, sizeof(doublevals[0]), sizeof(doublevals[0])}, {realvals, nrows, sizeof(realvals[0]), sizeof(realvals[0])}, {intvals, nrows, sizeof(intvals[0]), sizeof(intvals[0])}, {stringvals, nrows, sizeof(stringvals[0]), sizeof(stringvals[0])}, }; odc::api::Decoder decoder(columns, strides); decoder.decode(frame); EXPECT(::memcmp(stringvals[0], "\0\0\0\0\0\0\0\0", 8) == 0); EXPECT(intvals[0] == odc::api::Settings::integerMissingValue()); EXPECT(realvals[0] == odc::api::Settings::doubleMissingValue()); EXPECT(doublevals[0] == odc::api::Settings::doubleMissingValue()); EXPECT(::memcmp(stringvals[1], "testing\0", 8) == 0); EXPECT(intvals[1] == 12345678); EXPECT(realvals[1] == 1234.56); EXPECT(doublevals[1] == 9876.54); EXPECT(!reader.next()); } CASE("Test some-missing first row with Decoder integers-as-doubles") { odc::api::Reader reader("odb_533_2.odb", /* aggregated */ false); odc::api::Frame frame = reader.next(); EXPECT(frame.columnCount() == 5); EXPECT(frame.rowCount() == 2); EXPECT(frame.columnInfo()[0].name == "stringval"); EXPECT(frame.columnInfo()[1].name == "intval"); EXPECT(frame.columnInfo()[2].name == "realval"); EXPECT(frame.columnInfo()[3].name == "doubleval"); EXPECT(frame.columnInfo()[4].name == "changing"); constexpr size_t nrows = 2; char stringvals[nrows][8]; double intvals[nrows]; double realvals[nrows]; double doublevals[nrows]; double changing[nrows]; // n.b. columns in reverse order, to be awkward std::vector columns{"changing", "doubleval", "realval", "intval", "stringval"}; std::vector strides{ {changing, nrows, sizeof(changing[0]), sizeof(changing[0])}, {doublevals, nrows, sizeof(doublevals[0]), sizeof(doublevals[0])}, {realvals, nrows, sizeof(realvals[0]), sizeof(realvals[0])}, {intvals, nrows, sizeof(intvals[0]), sizeof(intvals[0])}, {stringvals, nrows, sizeof(stringvals[0]), sizeof(stringvals[0])}, }; odc::api::Decoder decoder(columns, strides); decoder.decode(frame); EXPECT(::memcmp(stringvals[0], "\0\0\0\0\0\0\0\0", 8) == 0); EXPECT(intvals[0] == odc::api::Settings::integerMissingValue()); EXPECT(realvals[0] == odc::api::Settings::doubleMissingValue()); EXPECT(doublevals[0] == odc::api::Settings::doubleMissingValue()); EXPECT(changing[0] == 1234); EXPECT(::memcmp(stringvals[1], "testing\0", 8) == 0); EXPECT(intvals[1] == 12345678); EXPECT(realvals[1] == 1234.56); EXPECT(doublevals[1] == 9876.54); EXPECT(changing[1] == 5678); EXPECT(!reader.next()); } CASE("Test some-missing first row with Decoder integers-as-integers") { TestIntegerDecoding integers; odc::api::Reader reader("odb_533_2.odb", /* aggregated */ false); odc::api::Frame frame = reader.next(); EXPECT(frame.columnCount() == 5); EXPECT(frame.rowCount() == 2); EXPECT(frame.columnInfo()[0].name == "stringval"); EXPECT(frame.columnInfo()[1].name == "intval"); EXPECT(frame.columnInfo()[2].name == "realval"); EXPECT(frame.columnInfo()[3].name == "doubleval"); EXPECT(frame.columnInfo()[4].name == "changing"); constexpr size_t nrows = 2; char stringvals[nrows][8]; int64_t intvals[nrows]; double realvals[nrows]; double doublevals[nrows]; int64_t changing[nrows]; // n.b. columns in reverse order, to be awkward std::vector columns{"changing", "doubleval", "realval", "intval", "stringval"}; std::vector strides{ {changing, nrows, sizeof(changing[0]), sizeof(changing[0])}, {doublevals, nrows, sizeof(doublevals[0]), sizeof(doublevals[0])}, {realvals, nrows, sizeof(realvals[0]), sizeof(realvals[0])}, {intvals, nrows, sizeof(intvals[0]), sizeof(intvals[0])}, {stringvals, nrows, sizeof(stringvals[0]), sizeof(stringvals[0])}, }; odc::api::Decoder decoder(columns, strides); decoder.decode(frame); EXPECT(::memcmp(stringvals[0], "\0\0\0\0\0\0\0\0", 8) == 0); EXPECT(intvals[0] == odc::api::Settings::integerMissingValue()); EXPECT(realvals[0] == odc::api::Settings::doubleMissingValue()); EXPECT(doublevals[0] == odc::api::Settings::doubleMissingValue()); EXPECT(changing[0] == 1234); EXPECT(::memcmp(stringvals[1], "testing\0", 8) == 0); EXPECT(intvals[1] == 12345678); EXPECT(realvals[1] == 1234.56); EXPECT(doublevals[1] == 9876.54); EXPECT(changing[1] == 5678); EXPECT(!reader.next()); } CASE("Test all-missing first row with Reader iterator integers-as-doubles") { odc::Reader reader("odb_533_1.odb"); auto it = reader.begin(); auto end = reader.end(); EXPECT(it != end); EXPECT(it->columns().size() == 4); EXPECT(it->columns()[0]->name() == "stringval"); EXPECT(it->columns()[0]->coder().name() == "int8_string"); EXPECT(it->columns()[1]->name() == "intval"); EXPECT(it->columns()[1]->coder().name() == "constant_or_missing"); EXPECT(it->columns()[2]->name() == "realval"); EXPECT(it->columns()[2]->coder().name() == "real_constant_or_missing"); EXPECT(it->columns()[3]->name() == "doubleval"); EXPECT(it->columns()[3]->coder().name() == "real_constant_or_missing"); EXPECT(it->isMissing(1)); EXPECT(it->isMissing(2)); EXPECT(it->isMissing(3)); EXPECT(it->string(0) == ""); EXPECT((*it)[1] == odc::api::Settings::integerMissingValue()); EXPECT(static_cast((*it)[1]) == odc::api::Settings::integerMissingValue()); EXPECT((*it)[2] == odc::api::Settings::doubleMissingValue()); EXPECT((*it)[3] == odc::api::Settings::doubleMissingValue()); ++it; EXPECT(it != end); EXPECT(!it->isMissing(1)); EXPECT(!it->isMissing(2)); EXPECT(!it->isMissing(3)); EXPECT(it->string(0) == "testing"); EXPECT((*it)[1] == 12345678); EXPECT(static_cast((*it)[1]) == 12345678); EXPECT((*it)[2] == 1234.56); EXPECT((*it)[3] == 9876.54); ++it; EXPECT(it == end); } CASE("Test all-missing first row with Reader iterator integers-as-integers") { TestIntegerDecoding integers; odc::Reader reader("odb_533_1.odb"); auto it = reader.begin(); auto end = reader.end(); EXPECT(it != end); EXPECT(it->columns().size() == 4); EXPECT(it->columns()[0]->name() == "stringval"); EXPECT(it->columns()[0]->coder().name() == "int8_string"); EXPECT(it->columns()[1]->name() == "intval"); EXPECT(it->columns()[1]->coder().name() == "constant_or_missing"); EXPECT(it->columns()[2]->name() == "realval"); EXPECT(it->columns()[2]->coder().name() == "real_constant_or_missing"); EXPECT(it->columns()[3]->name() == "doubleval"); EXPECT(it->columns()[3]->coder().name() == "real_constant_or_missing"); EXPECT(it->isMissing(1)); EXPECT(it->isMissing(2)); EXPECT(it->isMissing(3)); EXPECT(it->string(0) == ""); EXPECT(integer_cast((*it)[1]) == odc::api::Settings::integerMissingValue()); EXPECT((*it)[2] == odc::api::Settings::doubleMissingValue()); EXPECT((*it)[3] == odc::api::Settings::doubleMissingValue()); ++it; EXPECT(it != end); EXPECT(!it->isMissing(1)); EXPECT(!it->isMissing(2)); EXPECT(!it->isMissing(3)); EXPECT(it->string(0) == "testing"); EXPECT(integer_cast((*it)[1]) == 12345678); EXPECT((*it)[2] == 1234.56); EXPECT((*it)[3] == 9876.54); ++it; EXPECT(it == end); } CASE("Test some-missing first row with Reader iterator integers-as-doubles") { odc::Reader reader("odb_533_2.odb"); auto it = reader.begin(); auto end = reader.end(); EXPECT(it != end); EXPECT(it->columns().size() == 5); EXPECT(it->columns()[0]->name() == "stringval"); EXPECT(it->columns()[0]->coder().name() == "int8_string"); EXPECT(it->columns()[1]->name() == "intval"); EXPECT(it->columns()[1]->coder().name() == "constant_or_missing"); EXPECT(it->columns()[2]->name() == "realval"); EXPECT(it->columns()[2]->coder().name() == "real_constant_or_missing"); EXPECT(it->columns()[3]->name() == "doubleval"); EXPECT(it->columns()[3]->coder().name() == "real_constant_or_missing"); EXPECT(it->columns()[4]->name() == "changing"); EXPECT(it->columns()[4]->coder().name() == "int16"); EXPECT(it->isMissing(1)); EXPECT(it->isMissing(2)); EXPECT(it->isMissing(3)); EXPECT(!it->isMissing(4)); EXPECT(it->string(0) == ""); EXPECT((*it)[1] == odc::api::Settings::integerMissingValue()); EXPECT(static_cast((*it)[1]) == odc::api::Settings::integerMissingValue()); EXPECT((*it)[2] == odc::api::Settings::doubleMissingValue()); EXPECT((*it)[3] == odc::api::Settings::doubleMissingValue()); EXPECT((*it)[4] == 1234); ++it; EXPECT(it != end); EXPECT(!it->isMissing(1)); EXPECT(!it->isMissing(2)); EXPECT(!it->isMissing(3)); EXPECT(it->string(0) == "testing"); EXPECT((*it)[1] == 12345678); EXPECT(static_cast((*it)[1]) == 12345678); EXPECT((*it)[2] == 1234.56); EXPECT((*it)[3] == 9876.54); EXPECT((*it)[4] == 5678); ++it; EXPECT(it == end); } CASE("Test some-missing first row with Reader iterator integers-as-integers") { TestIntegerDecoding integers; odc::Reader reader("odb_533_2.odb"); auto it = reader.begin(); auto end = reader.end(); EXPECT(it != end); EXPECT(it->columns().size() == 5); EXPECT(it->columns()[0]->name() == "stringval"); EXPECT(it->columns()[0]->coder().name() == "int8_string"); EXPECT(it->columns()[1]->name() == "intval"); EXPECT(it->columns()[1]->coder().name() == "constant_or_missing"); EXPECT(it->columns()[2]->name() == "realval"); EXPECT(it->columns()[2]->coder().name() == "real_constant_or_missing"); EXPECT(it->columns()[3]->name() == "doubleval"); EXPECT(it->columns()[3]->coder().name() == "real_constant_or_missing"); EXPECT(it->columns()[4]->name() == "changing"); EXPECT(it->columns()[4]->coder().name() == "int16"); EXPECT(it->isMissing(1)); EXPECT(it->isMissing(2)); EXPECT(it->isMissing(3)); EXPECT(it->string(0) == ""); EXPECT(integer_cast((*it)[1]) == odc::api::Settings::integerMissingValue()); EXPECT((*it)[2] == odc::api::Settings::doubleMissingValue()); EXPECT((*it)[3] == odc::api::Settings::doubleMissingValue()); EXPECT(integer_cast((*it)[4]) == 1234); ++it; EXPECT(it != end); EXPECT(!it->isMissing(1)); EXPECT(!it->isMissing(2)); EXPECT(!it->isMissing(3)); EXPECT(it->string(0) == "testing"); EXPECT(integer_cast((*it)[1]) == 12345678); EXPECT((*it)[2] == 1234.56); EXPECT((*it)[3] == 9876.54); EXPECT(integer_cast((*it)[4]) == 5678); ++it; EXPECT(it == end); } // ------------------------------------------------------------------------------------------------------ int main(int argc, char* argv[]) { return run_tests(argc, argv); } odc-1.6.3/tests/core/test_codecs_end_to_end2.cc0000664000175000017500000001062415146027420021616 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/io/Buffer.h" #include "eckit/io/DataHandle.h" #include "eckit/io/MemoryHandle.h" #include "eckit/testing/Test.h" #include "odc/Reader.h" #include "odc/Writer.h" #include "odc/codec/Integer.h" #include "odc/codec/String.h" #include "odc/core/MetaData.h" #include "odc/tools/MockReader.h" using namespace eckit::testing; // Constant codecs are a little different from the others, as they can store multiple // different types within the same codec... /// Encoding/decoding using codecs and the reader/writer are tested elsewhere. /// This file is for miscelaneous tests, in case the edge cases elsewhere are insufficient. /// /// @note This is mainly SDS being paranoid about removing apparently duplicate tests when restructuring /// @note This test should be in test_codecs_end_to_end.cc, but it was causing segfaults with the cray /// compiler to put them in the same file. Sigh // ------------------------------------------------------------------------------------------------------ namespace { // This looks-like a read-iterator. It isn't, but that doesn't matter! const int num_rows_to_write = 10; class MockReadIterator { public: MockReadIterator(odc::api::ColumnType type, double data) : columns_(1), type_(type), data_(data), nRows_(num_rows_to_write), refCount_(0), noMore_(false) { columns_[0] = new odc::core::Column(columns_); ASSERT(columns_[0]); columns_[0]->name("a-col"); columns_[0]->type(type_); columns_[0]->hasMissing(false); } odc::core::MetaData& columns() { return columns_; } bool isNewDataset() { return false; } double* data() { return &data_; } bool next() { if (nRows_ == 0) return false; nRows_--; if (nRows_ == 0) noMore_ = true; return true; } protected: odc::core::MetaData columns_; odc::api::ColumnType type_; double data_; int nRows_; public: // Required for IteratorProxy int refCount_; bool noMore_; }; // n.b. Cannot use local classes as template arguments until c++11, so declare it here. // A constant string value (shorter than 8 bytes) // too-big, aligned storage --> undefined sanitizer is happy with access as casted double alignas(sizeof(double)) const char const_string_2[16] = "pies\0\0\0\0"; struct MockReadIteratorConstString2 : public MockReadIterator { MockReadIteratorConstString2() : MockReadIterator(odc::api::STRING, *reinterpret_cast(const_string_2)) { columns_[0]->coder( std::unique_ptr(new odc::codec::CodecChars(odc::api::STRING))); } }; } // namespace CASE("The constant codec can also store strings shorter than 8 bytes") { // Construct the encoded stuff eckit::Buffer buf(4096); eckit::MemoryHandle writeDH(buf); { odc::Writer<> oda(writeDH); odc::Writer<>::iterator outit = oda.begin(); odc::tool::MockReader reader; outit->pass1(reader.begin(), reader.end()); } // And test that this decodes correctly { eckit::MemoryHandle dh(buf.data(), static_cast(writeDH.position())); dh.openForRead(); odc::Reader oda(dh); odc::Reader::iterator it = oda.begin(); odc::Reader::iterator end = oda.end(); EXPECT(it->columns()[0]->name() == "a-col"); size_t count = 0; for (; it != end; ++it) { double val = (*it)[0]; EXPECT(::memcmp(const_string_2, &val, sizeof(val)) == 0); count++; } EXPECT(count == num_rows_to_write); // Check that this has used the constant codec. EXPECT(it->columns()[0]->coder().name() == "constant_string"); EXPECT(it->columns()[0]->type() == odc::api::STRING); } } // ------------------------------------------------------------------------------------------------------ int main(int argc, char* argv[]) { return run_tests(argc, argv); } odc-1.6.3/tests/core/test_codecs_read.cc0000664000175000017500000021343015146027420020351 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/eckit_ecbuild_config.h" #include "eckit/io/DataHandle.h" #include "eckit/system/SystemInfo.h" #include "eckit/testing/Test.h" #include "odc/api/Odb.h" #include "odc/codec/Constant.h" #include "odc/codec/Integer.h" #include "odc/codec/IntegerMissing.h" #include "odc/codec/Real.h" #include "odc/codec/String.h" #include "odc/core/Codec.h" #include #include #include #include using namespace eckit::testing; using namespace odc::core; using namespace odc::codec; namespace { struct TestIntegerDecoding { TestIntegerDecoding() { odc::api::Settings::treatIntegersAsDoubles(false); } ~TestIntegerDecoding() { odc::api::Settings::treatIntegersAsDoubles(true); } }; } // namespace // ------------------------------------------------------------------------------------------------------ // TODO with codecs: // // i) Make them templated on the stream/datahandle directly // ii) Construct them with a specific data handle/stream // iii) Why are we casting data handles via a void* ??? // Given the codec-initialising data, add the header on that is used to construct the // codec. size_t prepend_codec_selection_header(std::vector& data, const std::string& codec_name, bool bigEndian = false) { data.insert(data.begin(), 4, 0); data[bigEndian ? 3 : 0] = static_cast(codec_name.size()); data.insert(data.begin() + 4, codec_name.begin(), codec_name.end()); return 4 + codec_name.length(); } CASE("Constant values are constant") { // Data in little endian format. // "min" value is used for constants const char* source_data[] = { // Codec header "\x00\x00\x00\x00", // no missing value "\xb7\xe6\x87\xb4\x80\x65\xd2\x41", // min (1234567890.1234567) "\x00\x00\x00\x00\x00\x00\x00\x00", // maximum unspecified "\x00\x00\x00\x00\x00\x00\x00\x00", // missing value unspecified }; // Loop throumgh endiannesses for the source data for (int i = 0; i < 2; i++) { bool bigEndianSource = (i == 1); std::vector data; for (size_t j = 0; j < sizeof(source_data) / sizeof(const char*); j++) { size_t len = (j == 0) ? 4 : 8; data.insert(data.end(), source_data[j], source_data[j] + len); if (bigEndianSource) std::reverse(data.end() - len, data.end()); } // Construct codec directly #ifndef _CRAYC { // Skip name of codec GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size()); std::unique_ptr c; if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) { c.reset(new CodecConstant(odc::api::DOUBLE)); } else { c.reset(new CodecConstant(odc::api::DOUBLE)); } c->load(ds); EXPECT(ds.position() == eckit::Offset(28)); EXPECT(c->dataSizeDoubles() == 1); double tmp; c->decode(&tmp); EXPECT(tmp == 1234567890.1234567); c->decode(&tmp); EXPECT(tmp == 1234567890.1234567); c->decode(&tmp); EXPECT(tmp == 1234567890.1234567); c->decode(&tmp); EXPECT(tmp == 1234567890.1234567); // No further data should have been consumed from the data handle. EXPECT(ds.position() == eckit::Offset(28)); } #endif // Construct codec from factory size_t hdrSize = prepend_codec_selection_header(data, "constant", bigEndianSource); { GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size()); std::unique_ptr c; if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) { c = CodecFactory::instance().load(ds.same(), odc::api::DOUBLE); } else { c = CodecFactory::instance().load(ds.other(), odc::api::DOUBLE); } EXPECT(ds.position() == eckit::Offset(hdrSize + 28)); EXPECT(c->dataSizeDoubles() == 1); double tmp; c->decode(&tmp); EXPECT(tmp == 1234567890.1234567); EXPECT(tmp == 1234567890.1234567); EXPECT(tmp == 1234567890.1234567); EXPECT(tmp == 1234567890.1234567); EXPECT(ds.position() == eckit::Offset(hdrSize + 28)); } } } CASE("Constant integer values are constant") { // Set to decode to integers rather than doubles TestIntegerDecoding resetter; // Data in little endian format. // "min" value is used for constants const char* source_data[] = { // Codec header "\x00\x00\x00\x00", // no missing value "\x00\x00\x80\xb4\x80\x65\xd2\x41", // min (1234567890.1234567) "\x00\x00\x00\x00\x00\x00\x00\x00", // maximum unspecified "\x00\x00\x00\x00\x00\x00\x00\x00", // missing value unspecified }; // Loop throumgh endiannesses for the source data for (int i = 0; i < 2; i++) { bool bigEndianSource = (i == 1); std::vector data; for (size_t j = 0; j < sizeof(source_data) / sizeof(const char*); j++) { size_t len = (j == 0) ? 4 : 8; data.insert(data.end(), source_data[j], source_data[j] + len); if (bigEndianSource) std::reverse(data.end() - len, data.end()); } // Construct codec directly #ifndef _CRAYC { // Skip name of codec GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size()); std::unique_ptr c; if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) { c.reset(new CodecConstant(odc::api::BITFIELD)); } else { c.reset(new CodecConstant(odc::api::INTEGER)); } c->load(ds); EXPECT(ds.position() == eckit::Offset(28)); EXPECT(c->dataSizeDoubles() == 1); int64_t tmpi; double& tmpd(reinterpret_cast(tmpi)); c->decode(&tmpd); EXPECT(tmpi == 1234567890); c->decode(&tmpd); EXPECT(tmpi == 1234567890); c->decode(&tmpd); EXPECT(tmpi == 1234567890); c->decode(&tmpd); EXPECT(tmpi == 1234567890); // No further data should have been consumed from the data handle. EXPECT(ds.position() == eckit::Offset(28)); } #endif // Construct codec from factory size_t hdrSize = prepend_codec_selection_header(data, "constant", bigEndianSource); { GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size()); std::unique_ptr c; if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) { c = CodecFactory::instance().load(ds.same(), odc::api::INTEGER); } else { c = CodecFactory::instance().load(ds.other(), odc::api::BITFIELD); } EXPECT(ds.position() == eckit::Offset(hdrSize + 28)); EXPECT(c->dataSizeDoubles() == 1); int64_t tmpi; double& tmp(reinterpret_cast(tmpi)); c->decode(&tmp); EXPECT(tmpi == 1234567890); EXPECT(tmpi == 1234567890); EXPECT(tmpi == 1234567890); EXPECT(tmpi == 1234567890); EXPECT(ds.position() == eckit::Offset(hdrSize + 28)); } } } CASE("constant strings are constant") { // Data in little endian format. // "min" value is used for constants // NOTE that strings are NOT swapped around when things are in the // reverse byte order. const char* source_data[] = { // Codec header "\x00\x00\x00\x00", // no missing value "hi-there", // minimum supplies string "\x00\x00\x00\x00\x00\x00\x00\x00", // maximum unspecified "\x00\x00\x00\x00\x00\x00\x00\x00", // missing value unspecified }; // Loop through endiannesses for the source data for (int i = 0; i < 2; i++) { bool bigEndianSource = (i == 1); std::vector data; for (size_t j = 0; j < sizeof(source_data) / sizeof(const char*); j++) { size_t len = (j == 0) ? 4 : 8; data.insert(data.end(), source_data[j], source_data[j] + len); // n.b. Don't swap string data around with endianness if (bigEndianSource && j != 1) std::reverse(data.end() - len, data.end()); } // Construct codec directly #ifndef _CRAYC { // Skip name of codec GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size()); std::unique_ptr c; if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) { c.reset(new CodecConstantString(odc::api::STRING)); } else { c.reset(new CodecConstantString(odc::api::STRING)); } c->load(ds); EXPECT(ds.position() == eckit::Offset(28)); EXPECT(c->dataSizeDoubles() == 1); double val; c->decode(&val); EXPECT(std::string(reinterpret_cast(&val), 8) == "hi-there"); c->decode(&val); EXPECT(std::string(reinterpret_cast(&val), 8) == "hi-there"); c->decode(&val); EXPECT(std::string(reinterpret_cast(&val), 8) == "hi-there"); c->decode(&val); EXPECT(std::string(reinterpret_cast(&val), 8) == "hi-there"); // No further data should have been consumed from the data handle. EXPECT(ds.position() == eckit::Offset(28)); } #endif // Construct codec from factory size_t hdrSize = prepend_codec_selection_header(data, "constant_string", bigEndianSource); { GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size()); std::unique_ptr c; if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) { c = CodecFactory::instance().load(ds.same(), odc::api::STRING); } else { c = CodecFactory::instance().load(ds.other(), odc::api::STRING); } EXPECT(ds.position() == eckit::Offset(hdrSize + 28)); EXPECT(c->dataSizeDoubles() == 1); double val; c->decode(&val); EXPECT(std::string(reinterpret_cast(&val), 8) == "hi-there"); c->decode(&val); EXPECT(std::string(reinterpret_cast(&val), 8) == "hi-there"); c->decode(&val); EXPECT(std::string(reinterpret_cast(&val), 8) == "hi-there"); c->decode(&val); EXPECT(std::string(reinterpret_cast(&val), 8) == "hi-there"); EXPECT(ds.position() == eckit::Offset(hdrSize + 28)); } } } CASE("Constant integer or missing value behaves a bit oddly") { EXPECT(odc::MDI::integerMDI() == 2147483647); // Note that there is absolutely NOTHING that enforces that these are integers... // --> This test tests the generic case, with a double, which is odd // --> TODO: Really we ought to enforce integers for an integer case... // little const char* source_data[] = { // Codec header "\x01\x00\x00\x00", // has missing value // "\x00\x00\x80\x58\x34\x6f\xcd\x41, // min (little-endian: 987654321) // "\x00\x00\x80\x58\x34\x6f\xcd\x41, // max == min "\xad\x69\xfe\x58\x34\x6f\xcd\x41", // minimum value = 987654321.9876 "\xad\x69\xfe\x58\x34\x6f\xcd\x41", // maximum value "\x00\x00\xc0\xff\xff\xff\xdf\x41" // missingValue = 2147483647 }; // Loop through endiannesses for the source data for (int i = 0; i < 2; i++) { bool bigEndianSource = (i == 1); std::vector data; for (size_t j = 0; j < sizeof(source_data) / sizeof(const char*); j++) { size_t len = (j == 0) ? 4 : 8; data.insert(data.end(), source_data[j], source_data[j] + len); if (bigEndianSource) std::reverse(data.end() - len, data.end()); } // Insert the sequence of test values data.push_back(0); data.push_back(0xff); // missing for (size_t n = 0; n < 255; n++) { data.push_back(static_cast(n)); } data.push_back(0xff); // missing // Construct codec directly #ifndef _CRAYC { // Skip name of codec GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size()); std::unique_ptr c; if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) { c.reset(new CodecConstantOrMissing(odc::api::DOUBLE)); } else { c.reset(new CodecConstantOrMissing(odc::api::INTEGER)); } c->load(ds); c->setDataStream(ds); EXPECT(ds.position() == eckit::Offset(28)); EXPECT(c->dataSizeDoubles() == 1); double baseValue = 987654321.9876; // double baseValue = 987654321; double decoded; c->decode(&decoded); EXPECT(baseValue == decoded); c->decode(&decoded); EXPECT(decoded == odc::MDI::integerMDI()); // missing for (size_t n = 0; n < 255; n++) { double b = baseValue + n; c->decode(&decoded); EXPECT(b == decoded); } c->decode(&decoded); EXPECT(decoded == odc::MDI::integerMDI()); // missing EXPECT(ds.position() == eckit::Offset(28 + 258)); } #endif // Construct codec from factory size_t hdrSize = prepend_codec_selection_header(data, "constant_or_missing", bigEndianSource); { GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size()); std::unique_ptr c; if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) { c = CodecFactory::instance().load(ds.same(), odc::api::INTEGER); } else { c = CodecFactory::instance().load(ds.other(), odc::api::DOUBLE); } c->setDataStream(ds); EXPECT(ds.position() == eckit::Offset(hdrSize + 28)); EXPECT(c->dataSizeDoubles() == 1); double baseValue = 987654321.9876; double decoded; c->decode(&decoded); EXPECT(baseValue == decoded); c->decode(&decoded); EXPECT(decoded == odc::MDI::integerMDI()); // missing for (size_t i = 0; i < 255; i++) { double b = baseValue + i; c->decode(&decoded); EXPECT(b == decoded); } c->decode(&decoded); EXPECT(decoded == odc::MDI::integerMDI()); // missing EXPECT(ds.position() == eckit::Offset(hdrSize + 28 + 258)); } } } CASE("real constant or missing value is not quite constant") { EXPECT(odc::MDI::realMDI() == -2147483647); // TODO: Really something labelled constant ought to be actually constant... // Do this one big-endian just because. // BIG endian data const char* source_data[] = { // Codec header "\x00\x00\x00\x01", // has missing value "\x41\xcd\x6f\x34\x58\xfe\x69\xad", // min = 987654321.9876 (big-endian) "\x41\xcd\x6f\x34\x58\xfe\x69\xad", // max = 987654321.9876 (big-endian) "\xc1\xdf\xff\xff\xff\xc0\x00\x00" // missingValue = -2147483647 }; // Loop through endiannesses for the source data for (int i = 0; i < 2; i++) { bool bigEndianSource = (i == 1); std::vector data; for (size_t j = 0; j < sizeof(source_data) / sizeof(const char*); j++) { size_t len = (j == 0) ? 4 : 8; data.insert(data.end(), source_data[j], source_data[j] + len); if (!bigEndianSource) std::reverse(data.end() - len, data.end()); } // Insert the sequence of test values data.push_back(0); data.push_back(0xff); // missing for (size_t i = 0; i < 255; i++) { data.push_back(static_cast(i)); } data.push_back(0xff); // missing // Construct codec directly #ifndef _CRAYC { // Skip name of codec GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size()); std::unique_ptr c; if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) { c.reset(new CodecRealConstantOrMissing(odc::api::DOUBLE)); } else { c.reset(new CodecRealConstantOrMissing(odc::api::DOUBLE)); } c->load(ds); c->setDataStream(ds); EXPECT(ds.position() == eckit::Offset(28)); EXPECT(c->dataSizeDoubles() == 1); double baseValue = 987654321.9876; // double baseValue = 987654321; double decoded; c->decode(&decoded); EXPECT(baseValue == decoded); c->decode(&decoded); EXPECT(decoded == odc::MDI::realMDI()); // missing for (size_t i = 0; i < 255; i++) { double b = baseValue + i; c->decode(&decoded); EXPECT(b == decoded); } c->decode(&decoded); EXPECT(decoded == odc::MDI::realMDI()); // missing EXPECT(ds.position() == eckit::Offset(28 + 258)); } #endif // Construct codec from factory size_t hdrSize = prepend_codec_selection_header(data, "real_constant_or_missing", bigEndianSource); { GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size()); std::unique_ptr c; if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) { c = CodecFactory::instance().load(ds.same(), odc::api::DOUBLE); } else { c = CodecFactory::instance().load(ds.other(), odc::api::DOUBLE); } c->setDataStream(ds); EXPECT(ds.position() == eckit::Offset(hdrSize + 28)); EXPECT(c->dataSizeDoubles() == 1); double baseValue = 987654321.9876; double decoded; c->decode(&decoded); EXPECT(baseValue == decoded); c->decode(&decoded); EXPECT(decoded == odc::MDI::realMDI()); // missing for (size_t i = 0; i < 255; i++) { double b = baseValue + i; c->decode(&decoded); EXPECT(b == decoded); } c->decode(&decoded); EXPECT(decoded == odc::MDI::realMDI()); // missing EXPECT(ds.position() == eckit::Offset(hdrSize + 28 + 258)); } } } CASE("Character strings are 8-byte sequences coerced into being treated as doubles") { // n.b. there are no missing values for CodecChars // The values here are unused, and endianness is ignored for chars. const char* source_data[] = { // Codec header "\x00\x00\x00\x00", // 0 = hasMissing "\x00\x00\x00\x00\x00\x00\x00\x00", // min = 987654321.9876 (big-endian) "\x00\x00\x00\x00\x00\x00\x00\x00", // max = 987654321.9876 (big-endian) "\x00\x00\x00\x00\x00\x00\x00\x00", // missingValue = -2147483647 "\x00\x00\x00\x00", // Unused 0 value required by chars codec // String data "\0\0\0\0\0\0\0\0", "hi-there", "\0\xff\0\xff\0\xff\0\xff", "a-string", "\xff\xff\xff\xff\xff\xff\xff\xff", }; // Loop throumgh endiannesses for the source data for (int i = 0; i < 2; i++) { bool bigEndianSource = (i == 1); std::vector data; for (size_t j = 0; j < sizeof(source_data) / sizeof(const char*); j++) { size_t len = (j == 0 || j == 4) ? 4 : 8; data.insert(data.end(), source_data[j], source_data[j] + len); // n.b. Don't reverse the endianness of the string data. if (bigEndianSource && j < 5) std::reverse(data.end() - len, data.end()); } // Construct codec directly #ifndef _CRAYC { // Skip name of codec GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size()); std::unique_ptr c; if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) { c.reset(new CodecChars(odc::api::STRING)); } else { c.reset(new CodecChars(odc::api::STRING)); } c->load(ds); c->setDataStream(ds); EXPECT(ds.position() == eckit::Offset(32)); EXPECT(c->dataSizeDoubles() == 1); double val; c->decode(&val); EXPECT(::memcmp(&val, source_data[5], 8) == 0); c->decode(&val); EXPECT(::memcmp(&val, source_data[6], 8) == 0); c->decode(&val); EXPECT(::memcmp(&val, source_data[7], 8) == 0); c->decode(&val); EXPECT(::memcmp(&val, source_data[8], 8) == 0); c->decode(&val); EXPECT(::memcmp(&val, source_data[9], 8) == 0); EXPECT(ds.position() == eckit::Offset(32 + (8 * 5))); } #endif // Construct codec from factory size_t hdrSize = prepend_codec_selection_header(data, "chars", bigEndianSource); { GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size()); std::unique_ptr c; if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) { c = CodecFactory::instance().load(ds.same(), odc::api::STRING); } else { c = CodecFactory::instance().load(ds.other(), odc::api::STRING); } c->setDataStream(ds); EXPECT(ds.position() == eckit::Offset(hdrSize + 32)); EXPECT(c->dataSizeDoubles() == 1); double val; c->decode(&val); EXPECT(::memcmp(&val, source_data[5], 8) == 0); c->decode(&val); EXPECT(::memcmp(&val, source_data[6], 8) == 0); c->decode(&val); EXPECT(::memcmp(&val, source_data[7], 8) == 0); c->decode(&val); EXPECT(::memcmp(&val, source_data[8], 8) == 0); c->decode(&val); EXPECT(::memcmp(&val, source_data[9], 8) == 0); EXPECT(ds.position() == eckit::Offset(hdrSize + 32 + (8 * 5))); } } } CASE("long floating point values can include the missing data value") { const char* source_data[] = { // Codec header "\x00\x00\x00\x00", // no missing value "\x00\x00\x00\x00\x00\x00\x00\x00", // minimum unspecified "\x00\x00\x00\x00\x00\x00\x00\x00", // maximum unspecified "\x00\x00\x00\x00\x00\x00\x00\x00", // missing value unspecified // data to encode "\x00\x00\x00\x00\x00\x00\x00\x00", // 0.0 "\x53\xa4\x0c\x54\x34\x6f\x9d\x41", // 123456789.0123456 "\x9b\xe6\x57\xb7\x80\x65\x02\xc2", // -9876543210.9876 "\x00\x00\x00\x00\x00\x00\xf0\x7f", // +inf "\x00\x00\x00\x00\x00\x00\xf0\xff", // -inf "\x7f\xf7\xff\xff\xff\xff\xff\xff", // NaN (signalling) "\x7f\xff\xff\xff\xff\xff\xff\xff", // NaN (quiet) "\x00\x00\xc0\xff\xff\xff\xdf\xc1" // -2147483647 (otherwise the missing value) }; // Loop through endiannesses for the source data for (int i = 0; i < 2; i++) { bool bigEndianSource = (i == 1); std::vector data; for (size_t j = 0; j < sizeof(source_data) / sizeof(const char*); j++) { size_t len = (j == 0) ? 4 : 8; data.insert(data.end(), source_data[j], source_data[j] + len); if (bigEndianSource) std::reverse(data.end() - len, data.end()); } // Construct codec directly #ifndef _CRAYC { // Skip name of codec GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size()); std::unique_ptr c; if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) { c.reset(new CodecLongReal(odc::api::DOUBLE)); } else { c.reset(new CodecLongReal(odc::api::DOUBLE)); } c->load(ds); c->setDataStream(ds); EXPECT(ds.position() == eckit::Offset(28)); EXPECT(c->dataSizeDoubles() == 1); double val; c->decode(&val); EXPECT(val == 0); c->decode(&val); EXPECT(val == 123456789.0123456); c->decode(&val); EXPECT(val == -9876543210.9876); c->decode(&val); EXPECT(std::isinf(val)); EXPECT(val > 0); c->decode(&val); EXPECT(std::isinf(val)); EXPECT(val < 0); c->decode(&val); EXPECT(std::isnan(val)); c->decode(&val); EXPECT(std::isnan(val)); c->decode(&val); EXPECT(val == -2147483647); EXPECT(ds.position() == eckit::Offset(28 + (8 * 8))); } #endif // Construct codec from factory size_t hdrSize = prepend_codec_selection_header(data, "long_real", bigEndianSource); { GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size()); std::unique_ptr c; if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) { c = CodecFactory::instance().load(ds.same(), odc::api::DOUBLE); } else { c = CodecFactory::instance().load(ds.other(), odc::api::DOUBLE); } c->setDataStream(ds); EXPECT(ds.position() == eckit::Offset(hdrSize + 28)); EXPECT(c->dataSizeDoubles() == 1); double val; c->decode(&val); EXPECT(val == 0); c->decode(&val); EXPECT(val == 123456789.0123456); c->decode(&val); EXPECT(val == -9876543210.9876); c->decode(&val); EXPECT(std::isinf(val)); EXPECT(val > 0); c->decode(&val); EXPECT(std::isinf(val)); EXPECT(val < 0); c->decode(&val); EXPECT(std::isnan(val)); c->decode(&val); EXPECT(std::isnan(val)); c->decode(&val); EXPECT(val == -2147483647); EXPECT(ds.position() == eckit::Offset(hdrSize + 28 + (8 * 8))); } } } CASE("short floating point values can include the missing data value") { // Use a curious, custom missingValue to show it is being used. const char* source_data[] = { // Codec header "\x00\x00\x00\x00", // no missing value "\x00\x00\x00\x00\x00\x00\x00\x00", // minimum unspecified "\x00\x00\x00\x00\x00\x00\x00\x00", // maximum unspecified "\x04\x4f\xab\xa0\xe4\x4e\x91\x26", // missing value = 6.54565456545599971850917315786e-123 // data to encode "\x00\x00\x00\x00", // 0.0 "\x12\xbf\x1f\x49", // 654321.123 "\x00\x00\x80\x00", // Smallest available, missing value for short_real (1.17549435082229e-38) "\xff\xff\x7f\xff", // Lowest available, missing value for short_real2 (-3.40282346638529e+38) "\x00\x00\x80\x7f", // +inf "\x00\x00\x80\xff", // -inf "\xff\xff\xbf\x7f", // NaN (signalling) "\xff\xff\xff\x7f", // NaN (quiet) }; // Loop through endiannesses for the source data for (int i = 0; i < 4; i++) { bool bigEndianSource = (i % 2 == 0); bool secondCodec = (i > 1); std::vector data; for (size_t j = 0; j < sizeof(source_data) / sizeof(const char*); j++) { size_t len = (j == 0 || j > 3) ? 4 : 8; data.insert(data.end(), source_data[j], source_data[j] + len); if (bigEndianSource) std::reverse(data.end() - len, data.end()); } // Construct codec directly #ifndef _CRAYC { // Skip name of codec GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size()); std::unique_ptr c; if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) { if (secondCodec) { c.reset(new CodecShortReal2(odc::api::REAL)); } else { c.reset(new CodecShortReal(odc::api::DOUBLE)); } } else { if (secondCodec) { c.reset(new CodecShortReal2(odc::api::DOUBLE)); } else { c.reset(new CodecShortReal(odc::api::REAL)); } } c->load(ds); c->setDataStream(ds); EXPECT(ds.position() == eckit::Offset(28)); EXPECT(c->dataSizeDoubles() == 1); // n.b. == comparisons for floats as we are testing BIT reproducability of decoding double val; c->decode(&val); EXPECT(val == 0); c->decode(&val); EXPECT(val == float(654321.123)); // Each of the two codecs has a different internal missing value. Check it is correctly recognised. if (secondCodec) { c->decode(&val); EXPECT(val == float(1.17549435082229e-38)); c->decode(&val); EXPECT(val == 6.54565456545599971850917315786e-123); } else { c->decode(&val); EXPECT(val == 6.54565456545599971850917315786e-123); c->decode(&val); EXPECT(val == float(-3.40282346638529e+38)); } c->decode(&val); EXPECT(std::isinf(val)); EXPECT(val > 0); c->decode(&val); EXPECT(std::isinf(val)); EXPECT(val < 0); c->decode(&val); EXPECT(std::isnan(val)); c->decode(&val); EXPECT(std::isnan(val)); EXPECT(ds.position() == eckit::Offset(28 + (8 * 4))); } #endif // Construct codec from factory size_t hdrSize = prepend_codec_selection_header(data, secondCodec ? "short_real2" : "short_real", bigEndianSource); { GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size()); std::unique_ptr c; if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) { c = CodecFactory::instance().load(ds.same(), odc::api::DOUBLE); } else { c = CodecFactory::instance().load(ds.other(), odc::api::REAL); } c->setDataStream(ds); EXPECT(ds.position() == eckit::Offset(hdrSize + 28)); EXPECT(c->dataSizeDoubles() == 1); double val; c->decode(&val); EXPECT(val == 0); c->decode(&val); EXPECT(val == float(654321.123)); // Each of the two codecs has a different internal missing value. Check it is correctly recognised. if (secondCodec) { c->decode(&val); EXPECT(val == float(1.17549435082229e-38)); c->decode(&val); EXPECT(val == 6.54565456545599971850917315786e-123); } else { c->decode(&val); EXPECT(val == 6.54565456545599971850917315786e-123); c->decode(&val); EXPECT(val == float(-3.40282346638529e+38)); } c->decode(&val); EXPECT(std::isinf(val)); EXPECT(val > 0); c->decode(&val); EXPECT(std::isinf(val)); EXPECT(val < 0); c->decode(&val); EXPECT(std::isnan(val)); c->decode(&val); EXPECT(std::isnan(val)); EXPECT(ds.position() == eckit::Offset(hdrSize + 28 + (8 * 4))); } } } CASE("32bit integers can be decoded direct to integers") { // Set to decode to integers rather than doubles TestIntegerDecoding resetter; // Use a curious, custom missingValue to show it is being used. const char* source_data[] = { // Codec header "\x00\x00\x00\x00", // no missing value "\x00\x00\x00\x00\x00\x00\x00\x00", // minimum unspecified "\x00\x00\x00\x00\x00\x00\x00\x00", // maximum unspecified "\x00\x00\x00\x00\x00\x00\x00\x00", // missing value unspecified // data to encode "\x00\x00\x00\x00", // 0.0 "\xff\xff\xff\xff", // -1 "\xff\xff\xff\x7f", // 2147483647 == largest "\x00\x00\x00\x80", // -2147483648 == smallest "\x96\x28\x9c\xff" // -6543210 }; // Loop through endiannesses for the source data for (int i = 0; i < 2; i++) { bool bigEndianSource = (i == 1); std::vector data; for (size_t j = 0; j < sizeof(source_data) / sizeof(const char*); j++) { size_t len = (j == 0 || j > 3) ? 4 : 8; data.insert(data.end(), source_data[j], source_data[j] + len); if (bigEndianSource) std::reverse(data.end() - len, data.end()); } // Construct codec directly #ifndef _CRAYC { // Skip name of codec GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size()); std::unique_ptr c; if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) { c.reset(new CodecInt32(odc::api::INTEGER)); } else { c.reset(new CodecInt32(odc::api::INTEGER)); } c->load(ds); c->setDataStream(ds); EXPECT(ds.position() == eckit::Offset(28)); EXPECT(c->dataSizeDoubles() == 1); double val; int64_t& intVal = reinterpret_cast(val); c->decode(&val); EXPECT(intVal == 0); c->decode(&val); EXPECT(intVal == -1); c->decode(&val); EXPECT(intVal == 2147483647); c->decode(&val); EXPECT(intVal == -2147483648); c->decode(&val); EXPECT(intVal == -6543210); EXPECT(ds.position() == eckit::Offset(28 + (5 * 4))); } #endif // Construct codec from factory size_t hdrSize = prepend_codec_selection_header(data, "int32", bigEndianSource); { GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size()); std::unique_ptr c; if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) { c = CodecFactory::instance().load(ds.same(), odc::api::INTEGER); } else { c = CodecFactory::instance().load(ds.other(), odc::api::INTEGER); } c->setDataStream(ds); EXPECT(ds.position() == eckit::Offset(hdrSize + 28)); EXPECT(c->dataSizeDoubles() == 1); double val; int64_t& intVal = reinterpret_cast(val); c->decode(&val); EXPECT(intVal == 0); c->decode(&val); EXPECT(intVal == -1); c->decode(&val); EXPECT(intVal == 2147483647); c->decode(&val); EXPECT(intVal == -2147483648); c->decode(&val); EXPECT(intVal == -6543210); EXPECT(ds.position() == eckit::Offset(hdrSize + 28 + (5 * 4))); } } } CASE("32bit integers are as-is") { // Use a curious, custom missingValue to show it is being used. const char* source_data[] = { // Codec header "\x00\x00\x00\x00", // no missing value "\x00\x00\x00\x00\x00\x00\x00\x00", // minimum unspecified "\x00\x00\x00\x00\x00\x00\x00\x00", // maximum unspecified "\x00\x00\x00\x00\x00\x00\x00\x00", // missing value unspecified // data to encode "\x00\x00\x00\x00", // 0.0 "\xff\xff\xff\xff", // -1 "\xff\xff\xff\x7f", // 2147483647 == largest "\x00\x00\x00\x80", // -2147483648 == smallest "\x96\x28\x9c\xff" // -6543210 }; // Loop through endiannesses for the source data for (int i = 0; i < 2; i++) { bool bigEndianSource = (i == 1); std::vector data; for (size_t j = 0; j < sizeof(source_data) / sizeof(const char*); j++) { size_t len = (j == 0 || j > 3) ? 4 : 8; data.insert(data.end(), source_data[j], source_data[j] + len); if (bigEndianSource) std::reverse(data.end() - len, data.end()); } // Construct codec directly #ifndef _CRAYC { // Skip name of codec GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size()); std::unique_ptr c; if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) { c.reset(new CodecInt32(odc::api::INTEGER)); } else { c.reset(new CodecInt32(odc::api::INTEGER)); } c->load(ds); c->setDataStream(ds); EXPECT(ds.position() == eckit::Offset(28)); EXPECT(c->dataSizeDoubles() == 1); double val; c->decode(&val); EXPECT(val == 0); c->decode(&val); EXPECT(val == -1); c->decode(&val); EXPECT(val == 2147483647); c->decode(&val); EXPECT(val == -2147483648); c->decode(&val); EXPECT(val == -6543210); EXPECT(ds.position() == eckit::Offset(28 + (5 * 4))); } #endif // Construct codec from factory size_t hdrSize = prepend_codec_selection_header(data, "int32", bigEndianSource); { GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size()); std::unique_ptr c; if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) { c = CodecFactory::instance().load(ds.same(), odc::api::INTEGER); } else { c = CodecFactory::instance().load(ds.other(), odc::api::INTEGER); } c->setDataStream(ds); EXPECT(ds.position() == eckit::Offset(hdrSize + 28)); EXPECT(c->dataSizeDoubles() == 1); double val; c->decode(&val); EXPECT(val == 0); c->decode(&val); EXPECT(val == -1); c->decode(&val); EXPECT(val == 2147483647); c->decode(&val); EXPECT(val == -2147483648); c->decode(&val); EXPECT(val == -6543210); EXPECT(ds.position() == eckit::Offset(hdrSize + 28 + (5 * 4))); } } } CASE("16bit integers are stored with an offset. This need not (strictly) be integral!!") { // n.b. we use a non-standard, non-integral minimum to demonstrate the offset behaviour. // Use a curious, custom missingValue to show it is being used. const char* source_data[] = { // Codec header "\x00\x00\x00\x00", // no missing value "\xcd\xcc\xcc\xcc\xcc\xdc\x5e\xc0", // minimum = -123.45 "\x00\x00\x00\x00\x00\x00\x00\x00", // maximum unspecified "\x04\x4f\xab\xa0\xe4\x4e\x91\x26", // missing value = 6.54565456545599971850917315786e-123 // data to encode "\x00\x00", // 0.0 "\xff\xff", // 65535 and the missing value "\xff\x7f", // 32767 (no negatives) "\x00\x80", // 32768 (no negatives) "\x39\x30" // 12345 }; // Loop through endiannesses for the source data for (int i = 0; i < 4; i++) { bool bigEndianSource = (i % 2 == 0); bool withMissing = (i > 1); std::vector data; for (size_t j = 0; j < sizeof(source_data) / sizeof(const char*); j++) { size_t len = (j == 0) ? 4 : (j > 3) ? 2 : 8; data.insert(data.end(), source_data[j], source_data[j] + len); if (bigEndianSource) std::reverse(data.end() - len, data.end()); } // Construct codec directly #ifndef _CRAYC { // Skip name of codec GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size()); std::unique_ptr c; if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) { if (withMissing) { c.reset(new CodecInt16Missing(odc::api::INTEGER)); } else { c.reset(new CodecInt16(odc::api::INTEGER)); } } else { if (withMissing) { c.reset(new CodecInt16Missing(odc::api::INTEGER)); } else { c.reset(new CodecInt16(odc::api::INTEGER)); } } c->load(ds); c->setDataStream(ds); EXPECT(ds.position() == eckit::Offset(28)); EXPECT(c->dataSizeDoubles() == 1); double val; c->decode(&val); EXPECT(val == (double(-123.45) + 0)); c->decode(&val); if (withMissing) { EXPECT(val == 6.54565456545599971850917315786e-123); } else { EXPECT(val == (double(-123.45) + 65535)); } c->decode(&val); EXPECT(val == (double(-123.45) + 32767)); c->decode(&val); EXPECT(val == (double(-123.45) + 32768)); c->decode(&val); EXPECT(val == (double(-123.45) + 12345)); EXPECT(ds.position() == eckit::Offset(28 + (5 * 2))); } #endif // Construct codec from factory size_t hdrSize = prepend_codec_selection_header(data, withMissing ? "int16_missing" : "int16", bigEndianSource); { GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size()); std::unique_ptr c; if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) { c = CodecFactory::instance().load(ds.same(), odc::api::INTEGER); } else { c = CodecFactory::instance().load(ds.other(), odc::api::INTEGER); } c->setDataStream(ds); EXPECT(ds.position() == eckit::Offset(hdrSize + 28)); EXPECT(c->dataSizeDoubles() == 1); double val; c->decode(&val); EXPECT(val == (double(-123.45) + 0)); c->decode(&val); if (withMissing) { EXPECT(val == 6.54565456545599971850917315786e-123); } else { EXPECT(val == (double(-123.45) + 65535)); } c->decode(&val); EXPECT(val == (double(-123.45) + 32767)); c->decode(&val); EXPECT(val == (double(-123.45) + 32768)); c->decode(&val); EXPECT(val == (double(-123.45) + 12345)); EXPECT(ds.position() == eckit::Offset(hdrSize + 28 + (5 * 2))); } } } CASE("16bit integers are stored with an offset and can be decoded to integers") { // Set to decode to integers rather than doubles TestIntegerDecoding resetter; // n.b. we use a non-standard, non-integral minimum to demonstrate the offset behaviour. // Use a curious, custom missingValue to show it is being used. const char* source_data[] = { // Codec header "\x00\x00\x00\x00", // no missing value "\x00\x00\x00\x00\x00\xc0\x5e\xc0", // minimum = -123 "\x00\x00\x00\x00\x00\x00\x00\x00", // maximum unspecified "\x04\x4f\xab\xa0\xe4\x4e\x91\x26", // missing value = 6.54565456545599971850917315786e-123 // data to encode "\x00\x00", // 0.0 "\xff\xff", // 65535 and the missing value "\xff\x7f", // 32767 (no negatives) "\x00\x80", // 32768 (no negatives) "\x39\x30" // 12345 }; // Loop through endiannesses for the source data for (int i = 0; i < 4; i++) { bool bigEndianSource = (i % 2 == 0); bool withMissing = (i > 1); std::vector data; for (size_t j = 0; j < sizeof(source_data) / sizeof(const char*); j++) { size_t len = (j == 0) ? 4 : (j > 3) ? 2 : 8; data.insert(data.end(), source_data[j], source_data[j] + len); if (bigEndianSource) std::reverse(data.end() - len, data.end()); } // Construct codec directly #ifndef _CRAYC { // Skip name of codec GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size()); std::unique_ptr c; if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) { if (withMissing) { c.reset(new CodecInt16Missing(odc::api::INTEGER)); } else { c.reset(new CodecInt16(odc::api::INTEGER)); } } else { if (withMissing) { c.reset(new CodecInt16Missing(odc::api::INTEGER)); } else { c.reset(new CodecInt16(odc::api::INTEGER)); } } c->load(ds); c->setDataStream(ds); EXPECT(ds.position() == eckit::Offset(28)); EXPECT(c->dataSizeDoubles() == 1); double val; int64_t& intVal(reinterpret_cast(val)); c->decode(&val); EXPECT(intVal == -123 + 0); c->decode(&val); if (withMissing) { // missing Value returned unchanged. TODO: change this behaviour... // EXPECT(val == 6.54565456545599971850917315786e-123); } else { EXPECT(intVal == -123 + 65535); } c->decode(&val); EXPECT(intVal == -123 + 32767); c->decode(&val); EXPECT(intVal == -123 + 32768); c->decode(&val); EXPECT(intVal == -123 + 12345); EXPECT(ds.position() == eckit::Offset(28 + (5 * 2))); } #endif // Construct codec from factory size_t hdrSize = prepend_codec_selection_header(data, withMissing ? "int16_missing" : "int16", bigEndianSource); { GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size()); std::unique_ptr c; if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) { c = CodecFactory::instance().load(ds.same(), odc::api::INTEGER); } else { c = CodecFactory::instance().load(ds.other(), odc::api::INTEGER); } c->setDataStream(ds); EXPECT(ds.position() == eckit::Offset(hdrSize + 28)); EXPECT(c->dataSizeDoubles() == 1); double val; int64_t& intVal(reinterpret_cast(val)); c->decode(&val); EXPECT(intVal == -123 + 0); c->decode(&val); if (withMissing) { // EXPECT(val == 6.54565456545599971850917315786e-123); } else { EXPECT(intVal == -123 + 65535); } c->decode(&val); EXPECT(intVal == -123 + 32767); c->decode(&val); EXPECT(intVal == -123 + 32768); c->decode(&val); EXPECT(intVal == -123 + 12345); EXPECT(ds.position() == eckit::Offset(hdrSize + 28 + (5 * 2))); } } } CASE("8bit integers are stored with an offset. This need not (strictly) be integral!!") { // n.b. we use a non-standard, non-integral minimum to demonstrate the offset behaviour. // Use a curious, custom missingValue to show it is being used. const char* source_data[] = { // Codec header "\x00\x00\x00\x00", // no missing value "\x00\x00\x00\x00\x80\x88\xb3\xc0", // minimum = -5000.5 "\x00\x00\x00\x00\x00\x00\x00\x00", // maximum unspecified "\x04\x4f\xab\xa0\xe4\x4e\x91\x26", // missing value = 6.54565456545599971850917315786e-123 }; // Loop through endiannesses for the source data for (int i = 0; i < 4; i++) { bool bigEndianSource = (i % 2 == 0); bool withMissing = (i > 1); std::vector data; for (size_t j = 0; j < sizeof(source_data) / sizeof(const char*); j++) { size_t len = (j == 0) ? 4 : 8; data.insert(data.end(), source_data[j], source_data[j] + len); if (bigEndianSource) std::reverse(data.end() - len, data.end()); } // Add all of the data values for (int n = 0; n < 256; n++) { data.push_back(static_cast(n)); } // Construct codec directly #ifndef _CRAYC { // Skip name of codec GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size()); std::unique_ptr c; if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) { if (withMissing) { c.reset(new CodecInt8Missing(odc::api::INTEGER)); } else { c.reset(new CodecInt8(odc::api::INTEGER)); } } else { if (withMissing) { c.reset(new CodecInt8Missing(odc::api::INTEGER)); } else { c.reset(new CodecInt8(odc::api::INTEGER)); } } c->load(ds); c->setDataStream(ds); EXPECT(ds.position() == eckit::Offset(28)); EXPECT(c->dataSizeDoubles() == 1); double val; for (int n = 0; n < 255; n++) { c->decode(&val); EXPECT(val == (double(-5000.5) + n)); } c->decode(&val); EXPECT(val == (withMissing ? 6.54565456545599971850917315786e-123 : (-5000.5 + 255))); EXPECT(ds.position() == eckit::Offset(28 + 256)); } #endif // Construct codec from factory size_t hdrSize = prepend_codec_selection_header(data, withMissing ? "int8_missing" : "int8", bigEndianSource); { GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size()); std::unique_ptr c; if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) { c = CodecFactory::instance().load(ds.same(), odc::api::INTEGER); } else { c = CodecFactory::instance().load(ds.other(), odc::api::INTEGER); } c->setDataStream(ds); EXPECT(ds.position() == eckit::Offset(hdrSize + 28)); EXPECT(c->dataSizeDoubles() == 1); double val; for (int n = 0; n < 255; n++) { c->decode(&val); EXPECT(val == (double(-5000.5) + n)); } c->decode(&val); EXPECT(val == (withMissing ? 6.54565456545599971850917315786e-123 : (-5000.5 + 255))); EXPECT(ds.position() == eckit::Offset(hdrSize + 28 + 256)); } } } CASE("8bit integers are stored with an offset and can be decoded to integers") { // Set to decode to integers rather than doubles TestIntegerDecoding resetter; // n.b. we use a non-standard, non-integral minimum to demonstrate the offset behaviour. // Use a curious, custom missingValue to show it is being used. const char* source_data[] = { // Codec header "\x00\x00\x00\x00", // no missing value "\x00\x00\x00\x00\x00\x88\xb3\xc0", // minimum = -5000 "\x00\x00\x00\x00\x00\x00\x00\x00", // maximum unspecified "\x04\x4f\xab\xa0\xe4\x4e\x91\x26", // missing value = 6.54565456545599971850917315786e-123 }; // Loop through endiannesses for the source data for (int i = 0; i < 4; i++) { bool bigEndianSource = (i % 2 == 0); bool withMissing = (i > 1); std::vector data; for (size_t j = 0; j < sizeof(source_data) / sizeof(const char*); j++) { size_t len = (j == 0) ? 4 : 8; data.insert(data.end(), source_data[j], source_data[j] + len); if (bigEndianSource) std::reverse(data.end() - len, data.end()); } // Add all of the data values for (int n = 0; n < 256; n++) { data.push_back(static_cast(n)); } // Construct codec directly #ifndef _CRAYC { // Skip name of codec GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size()); std::unique_ptr c; if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) { if (withMissing) { c.reset(new CodecInt8Missing(odc::api::INTEGER)); } else { c.reset(new CodecInt8(odc::api::INTEGER)); } } else { if (withMissing) { c.reset(new CodecInt8Missing(odc::api::INTEGER)); } else { c.reset(new CodecInt8(odc::api::INTEGER)); } } c->load(ds); c->setDataStream(ds); EXPECT(ds.position() == eckit::Offset(28)); EXPECT(c->dataSizeDoubles() == 1); double val; int64_t& intVal = reinterpret_cast(val); for (int n = 0; n < 255; n++) { c->decode(&val); EXPECT(intVal == (-5000 + n)); } c->decode(&val); // TODO: Missing // EXPECT(val == (withMissing ? 6.54565456545599971850917315786e-123 : (-5000.5 + 255))); EXPECT(ds.position() == eckit::Offset(28 + 256)); } #endif // Construct codec from factory size_t hdrSize = prepend_codec_selection_header(data, withMissing ? "int8_missing" : "int8", bigEndianSource); { GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size()); std::unique_ptr c; if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) { c = CodecFactory::instance().load(ds.same(), odc::api::INTEGER); } else { c = CodecFactory::instance().load(ds.other(), odc::api::INTEGER); } c->setDataStream(ds); EXPECT(ds.position() == eckit::Offset(hdrSize + 28)); EXPECT(c->dataSizeDoubles() == 1); double val; int64_t& intVal = reinterpret_cast(val); for (int n = 0; n < 255; n++) { c->decode(&val); EXPECT(intVal == (-5000 + n)); } c->decode(&val); // TODO: Missing // EXPECT(val == (withMissing ? 6.54565456545599971850917315786e-123 : (-5000.5 + 255))); EXPECT(ds.position() == eckit::Offset(hdrSize + 28 + 256)); } } } CASE("Character strings can be stored in a flat list, and indexed") { // n.b. no missing values const char* source_data[] = { // Codec header "\x00\x00\x00\x00", // 0 = hasMissing "\x00\x00\x00\x00\x00\x00\x00\x00", // min unspecified "\x00\x00\x00\x00\x00\x00\x00\x00", // max unspecified "\x00\x00\x00\x00\x00\x00\x00\x00", // missingValue unspecified // How many strings are there in the table? "\x06\x00\x00\x00", // String data (prepended with lengths) // length, data, "cnt (discarded)", index "\x02\x00\x00\x00", "ab", "\x00\x00\x00\x00", "\x03\x00\x00\x00", // This string is too short "\x06\x00\x00\x00", "ghijkl", "\x00\x00\x00\x00", "\x04\x00\x00\x00", "\x08\x00\x00\x00", "mnopqrst", "\x00\x00\x00\x00", "\x05\x00\x00\x00", // 8-byte length "\x08\x00\x00\x00", "uvwxyzab", "\x00\x00\x00\x00", "\x01\x00\x00\x00", // too long "\x08\x00\x00\x00", "ghijklmn", "\x00\x00\x00\x00", "\x00\x00\x00\x00", "\x08\x00\x00\x00", "opqrstuv", "\x00\x00\x00\x00", "\x02\x00\x00\x00"}; // Loop throumgh endiannesses for the source data for (int i = 0; i < 4; i++) { bool bigEndianSource = (i % 2 == 0); bool bits16 = (i > 1); std::vector data; for (size_t j = 0; j < sizeof(source_data) / sizeof(const char*); j++) { size_t len = (j < 5) ? ((j == 0 || j == 4) ? 4 : 8) : ((j + 2) % 4 == 0 ? ::strlen(source_data[j]) : 4); data.insert(data.end(), source_data[j], source_data[j] + len); // n.b. Don't reverse the endianness of the string data. if (bigEndianSource && !((j > 5) && ((j + 2) % 4 == 0))) std::reverse(data.end() - len, data.end()); } // Which strings do we wish to decode (look at them in reverse. nb refers to index column) for (int n = 5; n >= 0; n--) { if (bits16 && bigEndianSource) data.push_back(0); data.push_back(static_cast(n)); if (bits16 && !bigEndianSource) data.push_back(0); } // Construct codec directly #ifndef _CRAYC { // Skip name of codec GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size()); std::unique_ptr c; if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) { if (bits16) { c.reset(new CodecInt16String(odc::api::STRING)); } else { c.reset(new CodecInt8String(odc::api::STRING)); } } else { if (bits16) { c.reset(new CodecInt16String(odc::api::STRING)); } else { c.reset(new CodecInt8String(odc::api::STRING)); } } c->load(ds); c->setDataStream(ds); EXPECT(ds.position() == eckit::Offset(144)); EXPECT(c->dataSizeDoubles() == 1); double val; c->decode(&val); EXPECT(std::string(reinterpret_cast(&val), 8) == "mnopqrst"); c->decode(&val); EXPECT(std::string(reinterpret_cast(&val), 6) == "ghijkl"); // silently works for shorter strings c->decode(&val); EXPECT(std::string(reinterpret_cast(&val), 2) == "ab"); // silently works for shorter strings c->decode(&val); EXPECT(std::string(reinterpret_cast(&val), 8) == "opqrstuv"); c->decode(&val); EXPECT(std::string(reinterpret_cast(&val), 8) == "uvwxyzab"); // gets truncated to 8 c->decode(&val); EXPECT(std::string(reinterpret_cast(&val), 8) == "ghijklmn"); EXPECT(ds.position() == eckit::Offset(144 + (6 * (bits16 ? 2 : 1)))); } #endif // Construct codec from factory size_t hdrSize = prepend_codec_selection_header(data, bits16 ? "int16_string" : "int8_string", bigEndianSource); { GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size()); std::unique_ptr c; if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) { c = CodecFactory::instance().load(ds.same(), odc::api::STRING); } else { c = CodecFactory::instance().load(ds.other(), odc::api::STRING); } c->setDataStream(ds); EXPECT(ds.position() == eckit::Offset(hdrSize + 144)); EXPECT(c->dataSizeDoubles() == 1); double val; c->decode(&val); EXPECT(std::string(reinterpret_cast(&val), 8) == "mnopqrst"); c->decode(&val); EXPECT(std::string(reinterpret_cast(&val), 6) == "ghijkl"); // silently works for shorter strings c->decode(&val); EXPECT(std::string(reinterpret_cast(&val), 2) == "ab"); // silently works for shorter strings c->decode(&val); EXPECT(std::string(reinterpret_cast(&val), 8) == "opqrstuv"); c->decode(&val); EXPECT(std::string(reinterpret_cast(&val), 8) == "uvwxyzab"); // gets truncated to 8 c->decode(&val); EXPECT(std::string(reinterpret_cast(&val), 8) == "ghijklmn"); EXPECT(ds.position() == eckit::Offset(hdrSize + 144 + (6 * (bits16 ? 2 : 1)))); } } } CASE("Character strings can be stored in a flat list, and indexed, and be longer than 8 bytes") { // n.b. no missing values const char* source_data[] = { // Codec header "\x00\x00\x00\x00", // 0 = hasMissing "\x00\x00\x00\x00\x00\x00\x00\x00", // min unspecified "\x00\x00\x00\x00\x00\x00\x00\x00", // max unspecified "\x00\x00\x00\x00\x00\x00\x00\x00", // missingValue unspecified // How many strings are there in the table? "\x06\x00\x00\x00", // String data (prepended with lengths) // length, data, "cnt (discarded)", index "\x02\x00\x00\x00", "ab", "\x00\x00\x00\x00", "\x03\x00\x00\x00", // This string is too short "\x06\x00\x00\x00", "ghijkl", "\x00\x00\x00\x00", "\x04\x00\x00\x00", "\x08\x00\x00\x00", "mnopqrst", "\x00\x00\x00\x00", "\x05\x00\x00\x00", // 8-byte length "\x0c\x00\x00\x00", "uvwxyzabcdef", "\x00\x00\x00\x00", "\x01\x00\x00\x00", // 12-byte "\x10\x00\x00\x00", "ghijklmnopqrstuv", "\x00\x00\x00\x00", "\x00\x00\x00\x00", // 16-byte "\x08\x00\x00\x00", "opqrstuv", "\x00\x00\x00\x00", "\x02\x00\x00\x00"}; // Loop throumgh endiannesses for the source data for (int i = 0; i < 4; i++) { bool bigEndianSource = (i % 2 == 0); bool bits16 = (i > 1); std::vector data; for (size_t j = 0; j < sizeof(source_data) / sizeof(const char*); j++) { size_t len = (j < 5) ? ((j == 0 || j == 4) ? 4 : 8) : ((j + 2) % 4 == 0 ? ::strlen(source_data[j]) : 4); data.insert(data.end(), source_data[j], source_data[j] + len); // n.b. Don't reverse the endianness of the string data. if (bigEndianSource && !((j > 5) && ((j + 2) % 4 == 0))) std::reverse(data.end() - len, data.end()); } // Which strings do we wish to decode (look at them in reverse. nb refers to index column) for (int n = 5; n >= 0; n--) { if (bits16 && bigEndianSource) data.push_back(0); data.push_back(static_cast(n)); if (bits16 && !bigEndianSource) data.push_back(0); } // Construct codec directly #ifndef _CRAYC { // Skip name of codec GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size()); std::unique_ptr c; if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) { if (bits16) { c.reset(new CodecInt16String(odc::api::STRING)); } else { c.reset(new CodecInt8String(odc::api::STRING)); } } else { if (bits16) { c.reset(new CodecInt16String(odc::api::STRING)); } else { c.reset(new CodecInt8String(odc::api::STRING)); } } c->load(ds); c->setDataStream(ds); EXPECT(ds.position() == eckit::Offset(156)); // n.b. This is different. 16 bytes possible!!! EXPECT(c->dataSizeDoubles() == 2); double val[2]; const char* val_c = reinterpret_cast(val); c->decode(val); EXPECT(std::string(val_c, ::strnlen(val_c, 16)) == "mnopqrst"); c->decode(val); EXPECT(std::string(val_c, ::strnlen(val_c, 16)) == "ghijkl"); c->decode(val); EXPECT(std::string(val_c, ::strnlen(val_c, 16)) == "ab"); c->decode(val); EXPECT(std::string(val_c, ::strnlen(val_c, 16)) == "opqrstuv"); c->decode(val); EXPECT(std::string(val_c, ::strnlen(val_c, 16)) == "uvwxyzabcdef"); c->decode(val); EXPECT(std::string(val_c, ::strnlen(val_c, 16)) == "ghijklmnopqrstuv"); EXPECT(ds.position() == eckit::Offset(156 + (6 * (bits16 ? 2 : 1)))); } #endif // Construct codec from factory size_t hdrSize = prepend_codec_selection_header(data, bits16 ? "int16_string" : "int8_string", bigEndianSource); { GeneralDataStream ds(bigEndianSource != eckit::system::SystemInfo::isBigEndian(), &data[0], data.size()); std::unique_ptr c; if (bigEndianSource == eckit::system::SystemInfo::isBigEndian()) { c = CodecFactory::instance().load(ds.same(), odc::api::STRING); } else { c = CodecFactory::instance().load(ds.other(), odc::api::STRING); } c->setDataStream(ds); EXPECT(ds.position() == eckit::Offset(hdrSize + 156)); // n.b. This is different. 16 bytes possible!!! EXPECT(c->dataSizeDoubles() == 2); double val[2]; const char* val_c = reinterpret_cast(val); c->decode(val); EXPECT(std::string(val_c, ::strnlen(val_c, 16)) == "mnopqrst"); c->decode(val); EXPECT(std::string(val_c, ::strnlen(val_c, 16)) == "ghijkl"); c->decode(val); EXPECT(std::string(val_c, ::strnlen(val_c, 16)) == "ab"); c->decode(val); EXPECT(std::string(val_c, ::strnlen(val_c, 16)) == "opqrstuv"); c->decode(val); EXPECT(std::string(val_c, ::strnlen(val_c, 16)) == "uvwxyzabcdef"); c->decode(val); EXPECT(std::string(val_c, ::strnlen(val_c, 16)) == "ghijklmnopqrstuv"); EXPECT(ds.position() == eckit::Offset(hdrSize + 156 + (6 * (bits16 ? 2 : 1)))); } } } // ------------------------------------------------------------------------------------------------------ int main(int argc, char* argv[]) { return run_tests(argc, argv); } odc-1.6.3/tests/core/test_table_iterator.cc0000664000175000017500000000371715146027420021123 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/config/Resource.h" #include "eckit/filesystem/PathName.h" #include "eckit/testing/Test.h" #include "odc/core/TablesReader.h" using namespace eckit::testing; using eckit::Log; // ------------------------------------------------------------------------------------------------------ eckit::Resource testDataPath("$TEST_DATA_DIRECTORY", ".."); CASE("Test access Table iterator") { eckit::PathName filename = testDataPath / "2000010106-reduced.odb"; std::unique_ptr dh(filename.fileHandle()); dh->openForRead(); eckit::AutoClose close(*dh); odc::core::TablesReader reader(*dh); auto it = reader.begin(); auto end = reader.end(); size_t numRows = 0; size_t tableCount = 0; eckit::Offset lastOffset = 0; EXPECT(dh->estimate() == eckit::Length(2449313)); while (it != end) { tableCount++; EXPECT(it->rowCount() == (tableCount == 333 ? 1753 : 10000)); EXPECT(it->nextPosition() > lastOffset); EXPECT(it->nextPosition() <= dh->estimate()); EXPECT(dh->estimate() == eckit::Length(2449313)); lastOffset = it->nextPosition(); numRows += it->rowCount(); ++it; } EXPECT(dh->estimate() == eckit::Length(2449313)); EXPECT(lastOffset == eckit::Offset(2449313)); EXPECT(numRows == 50000); dh->close(); } // ------------------------------------------------------------------------------------------------------ int main(int argc, char* argv[]) { return run_tests(argc, argv); } odc-1.6.3/tests/core/CMakeLists.txt0000664000175000017500000000154315146027420017310 0ustar alastairalastair list( APPEND core_data_files odb_533_1.odb odb_533_2.odb ) ecbuild_get_test_multidata( TARGET odc_get_core_data NAMES ${core_data_files} NOCHECK ) list( APPEND _core_odc_tests test_encode_odb test_decode_odb test_codecs_write test_codecs_read test_codecs_end_to_end test_codecs_end_to_end2 test_reencode_string_table test_concatenated_odbs test_minmax test_metadata test_select_iterator test_text_reader test_table_iterator test_initial_missing ) foreach( _test ${_core_odc_tests} ) ecbuild_add_test( TARGET odc_${_test} SOURCES ${_test}.cc ../TemporaryFiles.h TEST_DEPENDS odc_get_test_data odc_get_core_data ENVIRONMENT ${test_environment} LIBS eckit odccore ) endforeach() odc-1.6.3/tests/core/test_minmax.cc0000664000175000017500000000473015146027420017410 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/testing/Test.h" #include "odc/Reader.h" #include "TemporaryFiles.h" #include using namespace eckit::testing; // ------------------------------------------------------------------------------------------------------ CASE("Test that the numeric limits are what we expect") { EXPECT(std::numeric_limits::max() == 2147483647); EXPECT(std::numeric_limits::min() == -2147483648); } CASE("Test reading min, max and intermediate values") { SETUP("An odb file containing min/max/intermediate values") { class TemporaryMinMaxODB : public TemporaryFile { public: TemporaryMinMaxODB() { odc::Writer<> oda(path()); odc::Writer<>::iterator writer = oda.begin(); writer->setNumberOfColumns(2); writer->setColumn(0, "intcol", odc::api::INTEGER); writer->setColumn(1, "realcol", odc::api::REAL); writer->writeHeader(); (*writer)[0] = std::numeric_limits::min(); (*writer)[1] = 1; ++writer; (*writer)[0] = std::numeric_limits::max(); (*writer)[1] = 1; ++writer; (*writer)[0] = std::numeric_limits::min() + std::numeric_limits::max(); (*writer)[1] = 1; ++writer; } }; TemporaryMinMaxODB tmpODB; SECTION("Read the values back in") { odc::Reader oda(tmpODB.path()); odc::Reader::iterator it = oda.begin(); EXPECT((*it)[0] == std::numeric_limits::min()); ++it; EXPECT((*it)[0] == std::numeric_limits::max()); ++it; EXPECT((*it)[0] == std::numeric_limits::max() + std::numeric_limits::min()); ++it; } } } // ------------------------------------------------------------------------------------------------------ int main(int argc, char* argv[]) { return run_tests(argc, argv); } odc-1.6.3/tests/core/test_codecs_end_to_end.cc0000664000175000017500000002216415146027420021536 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/io/Buffer.h" #include "eckit/io/DataHandle.h" #include "eckit/io/MemoryHandle.h" #include "eckit/testing/Test.h" #include "odc/Reader.h" #include "odc/Writer.h" #include "odc/codec/Integer.h" #include "odc/codec/String.h" #include "odc/core/MetaData.h" #include "odc/tools/MockReader.h" using namespace eckit::testing; // Constant codecs are a little different from the others, as they can store multiple // different types within the same codec... /// Encoding/decoding using codecs and the reader/writer are tested elsewhere. /// This file is for miscelaneous tests, in case the edge cases elsewhere are insufficient. /// /// @note This is mainly SDS being paranoid about removing apparently duplicate tests when restructuring // ------------------------------------------------------------------------------------------------------ namespace { // This looks-like a read-iterator. It isn't, but that doesn't matter! const int num_rows_to_write = 10; class MockReadIterator { public: MockReadIterator(odc::api::ColumnType type, double data) : columns_(1), type_(type), data_(data), nRows_(num_rows_to_write), refCount_(0), noMore_(false) { columns_[0] = new odc::core::Column(columns_); ASSERT(columns_[0]); columns_[0]->name("a-col"); columns_[0]->type(type_); columns_[0]->hasMissing(false); } odc::core::MetaData& columns() { return columns_; } bool isNewDataset() { return false; } double* data() { return &data_; } bool next() { if (nRows_ == 0) return false; nRows_--; if (nRows_ == 0) noMore_ = true; return true; } protected: odc::core::MetaData columns_; odc::api::ColumnType type_; double data_; int nRows_; public: // Required for IteratorProxy int refCount_; bool noMore_; }; // n.b. Cannot use local classes as template arguments until c++11, so declare it here. // A constant integer value ... const long the_const_value = 20090624; struct MockReadIteratorConstInt : public MockReadIterator { MockReadIteratorConstInt() : MockReadIterator(odc::api::INTEGER, the_const_value) { columns_[0]->coder(std::unique_ptr( new odc::codec::CodecInt32(odc::api::INTEGER))); } }; // A constant string value (the full 8 bytes) // too-big, aligned storage --> undefined sanitizer is happy with access as casted double alignas(sizeof(double)) const char const_string_1[16] = "a-string"; struct MockReadIteratorConstString1 : public MockReadIterator { MockReadIteratorConstString1() : MockReadIterator(odc::api::STRING, *reinterpret_cast(const_string_1)) { columns_[0]->coder( std::unique_ptr(new odc::codec::CodecChars(odc::api::STRING))); } }; } // namespace CASE("The constant integer codec stores a constant integer") { // Construct the encoded stuff eckit::Buffer buf(4096); eckit::MemoryHandle writeDH(buf); { odc::Writer<> oda(writeDH); odc::Writer<>::iterator outit = oda.begin(); odc::tool::MockReader reader; outit->pass1(reader.begin(), reader.end()); } // And test that this decodes correctly { eckit::MemoryHandle dh(buf.data(), static_cast(writeDH.position())); dh.openForRead(); odc::Reader oda(dh); odc::Reader::iterator it = oda.begin(); odc::Reader::iterator end = oda.end(); EXPECT(it->columns()[0]->name() == "a-col"); size_t count = 0; for (; it != end; ++it) { EXPECT(static_cast((*it)[0]) == the_const_value); EXPECT((*it)[0] == static_cast(the_const_value)); count++; } EXPECT(count == num_rows_to_write); // Check that this has used the constant codec. EXPECT(it->columns()[0]->coder().name() == "constant"); EXPECT(it->columns()[0]->type() == odc::api::INTEGER); } } CASE("The constant codec can also store strings") { // Construct the encoded stuff eckit::Buffer buf(4096); eckit::MemoryHandle writeDH(buf); { odc::Writer<> oda(writeDH); odc::Writer<>::iterator outit = oda.begin(); odc::tool::MockReader reader; outit->pass1(reader.begin(), reader.end()); } // And test that this decodes correctly { eckit::MemoryHandle dh(buf.data(), static_cast(writeDH.position())); dh.openForRead(); odc::Reader oda(dh); odc::Reader::iterator it = oda.begin(); odc::Reader::iterator end = oda.end(); EXPECT(it->columns()[0]->name() == "a-col"); size_t count = 0; for (; it != end; ++it) { double val = (*it)[0]; EXPECT(::memcmp(const_string_1, &val, sizeof(val)) == 0); count++; } EXPECT(count == num_rows_to_write); // Check that this has used the constant codec. EXPECT(it->columns()[0]->coder().name() == "constant_string"); EXPECT(it->columns()[0]->type() == odc::api::STRING); } } CASE("The constant codec can also store doubles") { const double constant_value = -987654321.4321e-34; // Don't use the pass1 mechanism here. Build a writer explicitly to show that we can odc::api::ColumnType types[] = {odc::api::REAL, odc::api::DOUBLE}; for (size_t i = 0; i < 2; i++) { odc::api::ColumnType type = types[i]; // Construct the encoded stuff eckit::Buffer buf(4096); eckit::MemoryHandle writeDH(buf); { odc::Writer<> oda(writeDH); odc::Writer<>::iterator outit = oda.begin(); outit->setNumberOfColumns(1); outit->setColumn(0, "abcdefg", type); outit->writeHeader(); for (size_t i = 0; i < num_rows_to_write; i++) { (*outit)[0] = constant_value; ++outit; } } // And test that this decodes correctly { eckit::MemoryHandle dh(buf.data(), static_cast(writeDH.position())); dh.openForRead(); odc::Reader oda(dh); odc::Reader::iterator it = oda.begin(); odc::Reader::iterator end = oda.end(); EXPECT(it->columns()[0]->name() == "abcdefg"); size_t count = 0; for (; it != end; ++it) { double val = (*it)[0]; EXPECT(val == constant_value); count++; } EXPECT(count == num_rows_to_write); // Check that this has used the constant codec. EXPECT(it->columns()[0]->coder().name() == "constant"); EXPECT(it->columns()[0]->type() == type); } } } CASE("Missing values are encoded and decoded correctly") { // Create a mapping between the codecs, their associated missing values, and encoded data sizes typedef std::map > MapType; MapType codec_value_map; codec_value_map["short_real"] = std::make_pair(odc::MDI::realMDI(), sizeof(float)); codec_value_map["short_real2"] = std::make_pair(odc::MDI::realMDI(), sizeof(float)); codec_value_map["long_real"] = std::make_pair(odc::MDI::realMDI(), sizeof(double)); codec_value_map["int8_missing"] = std::make_pair(odc::MDI::integerMDI(), sizeof(int8_t)); codec_value_map["int16_missing"] = std::make_pair(odc::MDI::integerMDI(), sizeof(int16_t)); for (MapType::const_iterator it = codec_value_map.begin(); it != codec_value_map.end(); ++it) { const std::string& codec_name(it->first); double missing_value = it->second.first; int encoded_size = it->second.second; // Get the appropriate codec std::unique_ptr c( odc::core::CodecFactory::instance().build(codec_name, odc::api::DOUBLE)); EXPECT(c->name() == codec_name); // Write data into a buffer unsigned char buffer[256]; unsigned char* next_pos = c->encode(buffer, missing_value); EXPECT((next_pos - buffer) == encoded_size); // And check that we can decode it again! odc::core::DataStream ds(buffer, sizeof(buffer)); c->setDataStream(ds); double decoded; c->decode(&decoded); ASSERT(ds.position() == eckit::Offset(encoded_size)); ASSERT(decoded == missing_value); } } // ------------------------------------------------------------------------------------------------------ int main(int argc, char* argv[]) { return run_tests(argc, argv); } odc-1.6.3/tests/core/test_codecs_write.cc0000664000175000017500000017207515146027420020601 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include #include #include #include "eckit/eckit_ecbuild_config.h" #include "eckit/io/Buffer.h" #include "eckit/system/SystemInfo.h" #include "eckit/testing/Test.h" #include "odc/api/ColumnType.h" #include "odc/codec/Constant.h" #include "odc/codec/Integer.h" #include "odc/codec/IntegerMissing.h" #include "odc/codec/Real.h" #include "odc/codec/String.h" #include "odc/core/Codec.h" using namespace eckit; using namespace eckit::testing; using namespace odc::codec; using odc::core::Codec; using odc::core::DataStream; using odc::core::OtherByteOrder; using odc::core::SameByteOrder; // Log::info() << "DS: " << ds.position() << std::endl; // for (int i = 0; i < ds.position(); ++i) { // Log::info() << i << " : " << std::hex << (int)ds.data()[i] << std::dec << " -- " << (char)ds.data()[i] << // std::endl; // } // Log::info() << "pos: " << ds.position() << " : " << expectedHdrSize << std::endl; // exit(-1); // ------------------------------------------------------------------------------------------------------ // TODO with codecs: // // i) Make them templated on the stream/datahandle directly // ii) Construct them with a specific data handle/stream // iii) Why are we casting data handles via a void* ??? // iv) Why are load/save not virtual functions? // v) We should ASSERT() that encoded data is constant for constant codecs. Currently it is just ignored. // Given the codec-initialising data, add the header on that is used to construct the // codec. size_t prepend_codec_selection_header(std::vector& data, const std::string& codec_name, bool bigEndian = false) { data.insert(data.begin(), 4, 0); data[bigEndian ? 3 : 0] = static_cast(codec_name.size()); data.insert(data.begin() + 4, codec_name.begin(), codec_name.end()); return 4 + codec_name.length(); } class EndianCodecSave { public: EndianCodecSave(bool bigEndianData, Codec& codec) : buffer_(4095) { if (eckit::system::SystemInfo::isBigEndian() == bigEndianData) { DataStream ds(buffer_); codec.save(ds); position_ = ds.position(); } else { DataStream ds(buffer_); codec.save(ds); position_ = ds.position(); } } const char* data() const { return buffer_; } eckit::Offset position() const { return position_; } char* get() { return &buffer_[position_]; } void set(char* p) { ASSERT(p >= &buffer_[0]); ASSERT(p - &buffer_[0] < static_cast(buffer_.size())); position_ = static_cast(p - &buffer_[0]); } private: eckit::Buffer buffer_; eckit::Offset position_; }; // Normal write process: // // i) Initialise codecs and set missing value if appropriate // ii) Append values into a buffor for multiple rows // iii) Call gatherStats on codecs once for each row // iv) When block of rows is gathered, write header // v) Write the data, by calling encode on the data // [vi) Re-initialise fresh codecs]. CASE("Constant values consume no space in the output data buffer") { // Data in little endian format. // "min" value is used for constants const char* expected_data[] = { // Codec header "\x00\x00\x00\x00", // no missing value "\xb7\xe6\x87\xb4\x80\x65\xd2\x41", // min (1234567890.1234567) "\xb7\xe6\x87\xb4\x80\x65\xd2\x41", // max (1234567890.1234567) "\x00\x00\x00\x00\x00\x00\x00\x00", // missing value unspecified }; // Loop throumgh endiannesses for the source data for (int i = 0; i < 2; i++) { bool bigEndianOutput = (i == 1); std::vector data; for (size_t j = 0; j < sizeof(expected_data) / sizeof(const char*); j++) { size_t len = (j == 0) ? 4 : 8; data.insert(data.end(), expected_data[j], expected_data[j] + len); if (bigEndianOutput) std::reverse(data.end() - len, data.end()); } // Initialise codecs std::unique_ptr c; if (bigEndianOutput == eckit::system::SystemInfo::isBigEndian()) { c.reset(new CodecConstant(odc::api::DOUBLE)); } else { c.reset(new CodecConstant(odc::api::DOUBLE)); } c->missingValue(0.0); // Statistics in writing order c->gatherStats(1234567890.1234567); c->gatherStats(1234567890.1234567); c->gatherStats(1234567890.1234567); c->gatherStats(1234567890.1234567); c->gatherStats(1234567890.1234567); EXPECT(!c->hasMissing()); // Encode the header to the data stream EndianCodecSave ds(bigEndianOutput, *c); EXPECT(ds.position() == eckit::Offset(28)); EXPECT(::memcmp(&data[0], ds.data(), data.size()) == 0); // Encode the data to wherever we want it (in reality would be after the header, via a buffer.). // n.b. We don't produce any data when encoding with this codec std::vector buf(1024, 0); EXPECT(c->encode(&buf[0], 1234567890.1234567) == &buf[0]); EXPECT(c->encode(&buf[0], 1234567890.1234567) == &buf[0]); EXPECT(c->encode(&buf[0], 1234567890.1234567) == &buf[0]); EXPECT(c->encode(&buf[0], 1234567890.1234567) == &buf[0]); EXPECT(c->encode(&buf[0], 1234567890.1234567) == &buf[0]); for (size_t n = 0; n < buf.size(); n++) EXPECT(buf[n] == 0); } } CASE("Constant integer values consume no space in the output data buffer") { // Data in little endian format. // "min" value is used for constants const char* expected_data[] = { // Codec header "\x00\x00\x00\x00", // no missing value "\x00\x00\x80\xb4\x80\x65\xd2\x41", // min (1234567890.) "\x00\x00\x80\xb4\x80\x65\xd2\x41", // max (1234567890.) "\x00\x00\x00\x00\x00\x00\x00\x00", // missing value unspecified }; // Loop throumgh endiannesses for the source data for (int i = 0; i < 2; i++) { bool bigEndianOutput = (i == 1); std::vector data; for (size_t j = 0; j < sizeof(expected_data) / sizeof(const char*); j++) { size_t len = (j == 0) ? 4 : 8; data.insert(data.end(), expected_data[j], expected_data[j] + len); if (bigEndianOutput) std::reverse(data.end() - len, data.end()); } // Initialise codecs std::unique_ptr c; if (bigEndianOutput == eckit::system::SystemInfo::isBigEndian()) { c.reset(new CodecConstant(odc::api::INTEGER)); } else { c.reset(new CodecConstant(odc::api::INTEGER)); } c->missingValue(0.0); // Statistics in writing order int64_t x = 1234567890; double& rx(reinterpret_cast(x)); c->gatherStats(rx); c->gatherStats(rx); c->gatherStats(rx); c->gatherStats(rx); c->gatherStats(rx); EXPECT(!c->hasMissing()); // Encode the header to the data stream EndianCodecSave ds(bigEndianOutput, *c); EXPECT(ds.position() == eckit::Offset(28)); EXPECT(::memcmp(&data[0], ds.data(), data.size()) == 0); // Encode the data to wherever we want it (in reality would be after the header, via a buffer.). // n.b. We don't produce any data when encoding with this codec std::vector buf(1024, 0); EXPECT(c->encode(&buf[0], rx) == &buf[0]); EXPECT(c->encode(&buf[0], rx) == &buf[0]); EXPECT(c->encode(&buf[0], rx) == &buf[0]); EXPECT(c->encode(&buf[0], rx) == &buf[0]); EXPECT(c->encode(&buf[0], rx) == &buf[0]); for (size_t n = 0; n < buf.size(); n++) EXPECT(buf[n] == 0); } } CASE("constant strings consume no output data space") { // Data in little endian format. // "min" value is used for constants // NOTE that strings are NOT swapped around when things are in the // reverse byte order. const char* expected_data[] = { // Codec header "\x00\x00\x00\x00", // no missing value "hi-there", // minimum supplies string "hi-there", // maximum unspecified "\x00\x00\x00\x00\x00\x00\x00\x00", // missing value unspecified }; // Loop through endiannesses for the source data for (int i = 0; i < 2; i++) { bool bigEndianOutput = (i == 1); std::vector data; for (size_t j = 0; j < sizeof(expected_data) / sizeof(const char*); j++) { size_t len = (j == 0) ? 4 : 8; data.insert(data.end(), expected_data[j], expected_data[j] + len); // n.b. Don't swap string data around with endianness } // Initialise codecs std::unique_ptr c; if (bigEndianOutput == eckit::system::SystemInfo::isBigEndian()) { c.reset(new CodecConstantString(odc::api::STRING)); } else { c.reset(new CodecConstantString(odc::api::STRING)); } c->missingValue(0.0); // Statistics in writing order alignas(sizeof(double)) const char str[] = "hi-there"; c->gatherStats(*reinterpret_cast(str)); c->gatherStats(*reinterpret_cast(str)); c->gatherStats(*reinterpret_cast(str)); c->gatherStats(*reinterpret_cast(str)); EXPECT(!c->hasMissing()); // Encode the header to the data stream EndianCodecSave ds(bigEndianOutput, *c); EXPECT(ds.position() == eckit::Offset(28)); EXPECT(::memcmp(&data[0], ds.data(), data.size()) == 0); // Encode the data to wherever we want it (in reality would be after the header, via a buffer.). // n.b. We don't produce any data when encoding with this codec std::vector buf(1024, 0); EXPECT(c->encode(&buf[0], *reinterpret_cast(str)) == &buf[0]); EXPECT(c->encode(&buf[0], *reinterpret_cast(str)) == &buf[0]); EXPECT(c->encode(&buf[0], *reinterpret_cast(str)) == &buf[0]); EXPECT(c->encode(&buf[0], *reinterpret_cast(str)) == &buf[0]); for (size_t n = 0; n < buf.size(); n++) EXPECT(buf[n] == 0); } } CASE("Constant integer or missing value behaves a bit oddly") { EXPECT(odc::MDI::integerMDI() == 2147483647); // Note that there is absolutely NOTHING that enforces that these are integers... // --> This test tests the generic case, with a double, which is odd // --> TODO: Really we ought to enforce integers for an integer case... double customMissingValue = 2222222222; double baseValue = 987654321.9876; const size_t expectedHdrSize = 28; const char* expected_data[] = { // Codec header "\x01\x00\x00\x00", // has missing value // "\x00\x00\x80\x58\x34\x6f\xcd\x41, // min (little-endian: 987654321) // "\x00\x00\x80\x58\x34\x6f\xcd\x41, // max == min "\xad\x69\xfe\x58\x34\x6f\xcd\x41", // minimum value = 987654321.9876 "\xad\x69\xfe\xd7\x34\x6f\xcd\x41", // maximum value = 987654321.9876 + 254 "\x00\x00\xc0\x71\x8d\x8e\xe0\x41" // missingValue = -2222222222 }; // Loop through endiannesses for the source data for (int i = 0; i < 2; i++) { bool bigEndianOutput = (i == 1); std::vector data; for (size_t j = 0; j < sizeof(expected_data) / sizeof(const char*); j++) { size_t len = (j == 0) ? 4 : 8; data.insert(data.end(), expected_data[j], expected_data[j] + len); if (bigEndianOutput) std::reverse(data.end() - len, data.end()); } // Insert the sequence of test values data.push_back(0); data.push_back(0xff); // missing for (size_t n = 0; n < 255; n++) { data.push_back(static_cast(n)); } data.push_back(0xff); // missing // Initialise codecs std::unique_ptr c; if (bigEndianOutput == eckit::system::SystemInfo::isBigEndian()) { c.reset(new CodecConstantOrMissing(odc::api::DOUBLE)); } else { c.reset(new CodecConstantOrMissing(odc::api::DOUBLE)); } c->missingValue(customMissingValue); EXPECT(!c->hasMissing()); // Statistics in writing order c->gatherStats(baseValue + 0); c->gatherStats(customMissingValue); // missing for (size_t n = 0; n < 255; n++) { c->gatherStats(baseValue + n); } c->gatherStats(customMissingValue); // missing // Detects that we have added a missing value EXPECT(c->hasMissing()); // Encode the header to the data stream EndianCodecSave ds(bigEndianOutput, *c); EXPECT(ds.position() == eckit::Offset(expectedHdrSize)); // Encode the data to wherever we want it (in reality would be after the header, via a buffer.). // Expect one byte per element. char* posNext; EXPECT((posNext = c->encode(ds.get(), baseValue + 0)) == (ds.get() + 1)); ds.set(posNext); EXPECT((posNext = c->encode(ds.get(), customMissingValue)) == (ds.get() + 1)); ds.set(posNext); for (size_t n = 0; n < 255; n++) { EXPECT((posNext = c->encode(ds.get(), baseValue + n)) == (ds.get() + 1)); ds.set(posNext); } EXPECT((posNext = c->encode(ds.get(), customMissingValue)) == (ds.get() + 1)); ds.set(posNext); // Check we have the data we expect size_t nelem = 258; EXPECT(ds.position() == eckit::Offset(expectedHdrSize + nelem)); EXPECT(::memcmp(&data[0], ds.data(), expectedHdrSize + nelem) == 0); } } CASE("real constant or missing value is not quite constant") { EXPECT(odc::MDI::realMDI() == -2147483647); // TODO: Really something labelled constant ought to be actually constant... // Do this one big-endian just because. double customMissingValue = -2222222222; double baseValue = 987654321.9876; const size_t expectedHdrSize = 28; const char* expected_data[] = { // Codec header "\x01\x00\x00\x00", // has missing value "\xad\x69\xfe\x58\x34\x6f\xcd\x41", // minimum value = 987654321.9876 "\xad\x69\xfe\xd7\x34\x6f\xcd\x41", // maximum value = 987654321.9876 + 254 "\x00\x00\xc0\x71\x8d\x8e\xe0\xc1" // missingValue = -2222222222 }; // Loop through endiannesses for the source data for (int i = 0; i < 2; i++) { bool bigEndianOutput = (i == 1); std::vector data; for (size_t j = 0; j < sizeof(expected_data) / sizeof(const char*); j++) { size_t len = (j == 0) ? 4 : 8; data.insert(data.end(), expected_data[j], expected_data[j] + len); if (bigEndianOutput) std::reverse(data.end() - len, data.end()); } // Insert the sequence of test values data.push_back(0); data.push_back(0xff); // missing for (size_t n = 0; n < 255; n++) { data.push_back(static_cast(n)); } data.push_back(0xff); // missing // Initialise codecs std::unique_ptr c; if (bigEndianOutput == eckit::system::SystemInfo::isBigEndian()) { c.reset(new CodecRealConstantOrMissing(odc::api::DOUBLE)); } else { c.reset(new CodecRealConstantOrMissing(odc::api::DOUBLE)); } c->missingValue(customMissingValue); EXPECT(!c->hasMissing()); // Statistics in writing order c->gatherStats(baseValue + 0); c->gatherStats(customMissingValue); for (size_t n = 0; n < 255; n++) { c->gatherStats(baseValue + n); } c->gatherStats(customMissingValue); // Detect that we have added a missing value EXPECT(c->hasMissing()); // Encode the header to the data stream EndianCodecSave ds(bigEndianOutput, *c); EXPECT(ds.position() == eckit::Offset(expectedHdrSize)); // Encode the data to wherever we want it // Expect one byte per element. char* posNext; EXPECT((posNext = c->encode(ds.get(), baseValue + 0)) == (ds.get() + 1)); ds.set(posNext); EXPECT((posNext = c->encode(ds.get(), customMissingValue)) == (ds.get() + 1)); ds.set(posNext); for (size_t n = 0; n < 255; n++) { EXPECT((posNext = c->encode(ds.get(), baseValue + n)) == (ds.get() + 1)); ds.set(posNext); } EXPECT((posNext = c->encode(ds.get(), customMissingValue)) == (ds.get() + 1)); ds.set(posNext); // Check we have the data we expect size_t nelem = 258; EXPECT(ds.position() == eckit::Offset(expectedHdrSize + nelem)); // eckit::Log::info() << "DATA: " << std::endl; // for (size_t n = 0; n < expectedHdrSize + nelem; n++) { // eckit::Log::info() << std::hex << int(data[n]) << " " << int(ds.data()[n]) << std::endl; // if (int(data[n]) != int(ds.data()[n])) // eckit::Log::info() << "******************************" << std::endl; // } EXPECT(::memcmp(&data[0], ds.data(), expectedHdrSize + nelem) == 0); } } CASE("Character strings are 8-byte sequences coerced into being treated as doubles") { // n.b. there are no missing values for CodecChars const size_t expectedHdrSize = 32; alignas(sizeof(double)) const std::array expected_data = { // Codec header "\x00\x00\x00\x00", // 0 = hasMissing "\x00\x00\x00\x00\x00\x00\x00\x00", // min = 987654321.9876 (big-endian) // UNUSED "\x00\x00\x00\x00\x00\x00\x00\x00", // max = 987654321.9876 (big-endian) // UNUSED "\x00\x00\x00\x00\x00\x00\x00\x00", // missingValue = -2147483647 // UNUSED "\x00\x00\x00\x00", // Unused 0 value required by chars codec // String data "\0\0\0\0\0\0\0\0", "hi-there", "\0\xff\0\xff\0\xff\0\xff", "a-string", "\xff\xff\xff\xff\xff\xff\xff\xff", }; // Loop throumgh endiannesses for the source data for (int i = 0; i < 2; i++) { bool bigEndianOutput = (i == 1); std::vector data; for (size_t j = 0; j < expected_data.size(); j++) { size_t len = (j == 0 || j == 4) ? 4 : 8; data.insert(data.end(), expected_data[j], expected_data[j] + len); // n.b. Don't reverse the endianness of the string data. if (bigEndianOutput && j < 5) std::reverse(data.end() - len, data.end()); } // Initialise codecs std::unique_ptr c; if (bigEndianOutput == eckit::system::SystemInfo::isBigEndian()) { c.reset(new CodecChars(odc::api::STRING)); } else { c.reset(new CodecChars(odc::api::STRING)); } c->missingValue(0.0); EXPECT(!c->hasMissing()); // Statistics in writing order c->gatherStats(*reinterpret_cast(expected_data[5])); c->gatherStats(*reinterpret_cast(expected_data[6])); c->gatherStats(*reinterpret_cast(expected_data[7])); c->gatherStats(*reinterpret_cast(expected_data[8])); c->gatherStats(*reinterpret_cast(expected_data[9])); EXPECT(!c->hasMissing()); // Encode the header to the data stream EndianCodecSave ds(bigEndianOutput, *c); EXPECT(ds.position() == eckit::Offset(expectedHdrSize)); // Encode the data where we want it (ensuring that the data is written in appropriate // sized blocks. char* posNext; for (size_t n = 5; n < 10; n++) { EXPECT((posNext = c->encode(ds.get(), *reinterpret_cast(expected_data[n]))) == (ds.get() + 8)); ds.set(posNext); } // Check we have the data we expect // n.b. We exclude the min/max/missing section of the header. This is not used for reading // CodecChars (at the moment), and the existing codec does odd things. We don't want // to code this behaviour into a test, as that would be weird. size_t dataSize = (8 * 5); EXPECT(ds.position() == eckit::Offset(expectedHdrSize + dataSize)); EXPECT(::memcmp(&data[0], ds.data(), 4) == 0); EXPECT(::memcmp(&data[28], &ds.data()[28], expectedHdrSize + dataSize - 28) == 0); } } CASE("long floating point values can include the missing data value") { const uint64_t inf_bits = 0x7ff0000000000000; const uint64_t neg_inf_bits = 0xfff0000000000000; const uint64_t sig_nan_bits = 0xfffffffffffff77f; const uint64_t quiet_nan_bits = 0xffffffffffffff7f; const size_t expectedHdrSize = 28; double customMissingValue = 2222222222; const char* expected_data[] = { // Codec header "\x01\x00\x00\x00", // has missing value "\x00\x00\x00\x00\x00\x00\xf0\xff", // minimum (-inf) "\x00\x00\x00\x00\x00\x00\xf0\x7f", // maximum (+inf) "\x00\x00\xc0\x71\x8d\x8e\xe0\x41", // missingValue = 2222222222 // data to encode "\x00\x00\x00\x00\x00\x00\x00\x00", // 0.0 "\x53\xa4\x0c\x54\x34\x6f\x9d\x41", // 123456789.0123456 "\x9b\xe6\x57\xb7\x80\x65\x02\xc2", // -9876543210.9876 "\x00\x00\x00\x00\x00\x00\xf0\x7f", // +inf "\x00\x00\x00\x00\x00\x00\xf0\xff", // -inf "\x7f\xf7\xff\xff\xff\xff\xff\xff", // NaN (signalling) "\x7f\xff\xff\xff\xff\xff\xff\xff", // NaN (quiet) "\x00\x00\xc0\xff\xff\xff\xdf\xc1", // -2147483647 (otherwise the missing value) "\x00\x00\xc0\x71\x8d\x8e\xe0\x41" // missingValue = 2222222222 }; // Loop through endiannesses for the source data for (int i = 0; i < 2; i++) { bool bigEndianOutput = (i == 1); std::vector data; for (size_t j = 0; j < sizeof(expected_data) / sizeof(const char*); j++) { size_t len = (j == 0) ? 4 : 8; data.insert(data.end(), expected_data[j], expected_data[j] + len); if (bigEndianOutput) std::reverse(data.end() - len, data.end()); } // Initialise codecs std::unique_ptr c; if (bigEndianOutput == eckit::system::SystemInfo::isBigEndian()) { c.reset(new CodecLongReal(odc::api::DOUBLE)); } else { c.reset(new CodecLongReal(odc::api::DOUBLE)); } c->missingValue(customMissingValue); EXPECT(!c->hasMissing()); // Statistics in writing order c->gatherStats(0.0); c->gatherStats(123456789.0123456); c->gatherStats(-9876543210.9876); double v = *reinterpret_cast(&inf_bits); EXPECT(std::isinf(v)); EXPECT(v > 0); c->gatherStats(v); v = *reinterpret_cast(&neg_inf_bits); EXPECT(std::isinf(v)); EXPECT(v < 0); c->gatherStats(v); v = *reinterpret_cast(&sig_nan_bits); EXPECT(std::isnan(v)); c->gatherStats(v); v = *reinterpret_cast(&quiet_nan_bits); EXPECT(std::isnan(v)); c->gatherStats(v); c->gatherStats(customMissingValue); c->gatherStats(-2147483647); // Detect that we have added a missing value EXPECT(c->hasMissing()); // Encode the header to the data stream EndianCodecSave ds(bigEndianOutput, *c); EXPECT(ds.position() == eckit::Offset(expectedHdrSize)); // Encode the data to wherever we want it // Expect 8 bytes per element char* posNext; EXPECT((posNext = c->encode(ds.get(), 0.0)) == (ds.get() + 8)); ds.set(posNext); EXPECT((posNext = c->encode(ds.get(), 123456789.0123456)) == (ds.get() + 8)); ds.set(posNext); EXPECT((posNext = c->encode(ds.get(), -9876543210.9876)) == (ds.get() + 8)); ds.set(posNext); EXPECT((posNext = c->encode(ds.get(), *reinterpret_cast(&inf_bits))) == (ds.get() + 8)); ds.set(posNext); EXPECT((posNext = c->encode(ds.get(), *reinterpret_cast(&neg_inf_bits))) == (ds.get() + 8)); ds.set(posNext); EXPECT((posNext = c->encode(ds.get(), *reinterpret_cast(&sig_nan_bits))) == (ds.get() + 8)); ds.set(posNext); EXPECT((posNext = c->encode(ds.get(), *reinterpret_cast(&quiet_nan_bits))) == (ds.get() + 8)); ds.set(posNext); EXPECT((posNext = c->encode(ds.get(), -2147483647)) == (ds.get() + 8)); ds.set(posNext); EXPECT((posNext = c->encode(ds.get(), customMissingValue)) == (ds.get() + 8)); ds.set(posNext); // Check we have the data we expect size_t data_size = (9 * 8); EXPECT(ds.position() == eckit::Offset(expectedHdrSize + data_size)); // eckit::Log::info() << "DATA: " << std::endl; // for (size_t n = 0; n < expectedHdrSize + data_size; n++) { // eckit::Log::info() << std::hex << int(data[n]) << " " << int(ds.data()[n]) << std::endl; // if (int(data[n]) != int(ds.data()[n])) // eckit::Log::info() << "******************************" << std::endl; // } EXPECT(::memcmp(&data[0], ds.data(), expectedHdrSize + data_size) == 0); } } CASE("short floating point values can include the missing data value") { // Use a curious, custom missingValue to show it is being used. // n.b. This cannot be represented with a float! const uint64_t inf_bits = 0x7ff0000000000000; const uint64_t neg_inf_bits = 0xfff0000000000000; const uint32_t sig_nan_bits = 0xffffbf7f; const uint32_t quiet_nan_bits = 0xffffff7f; const size_t expectedHdrSize = 28; double customMissingValue = -22222222222; const char* expected_data[] = { // Codec header "\x01\x00\x00\x00", // has missing value "\x00\x00\x00\x00\x00\x00\xf0\xff", // minimum (-inf) "\x00\x00\x00\x00\x00\x00\xf0\x7f", // maximum (+inf) "\x00\x00\x38\xce\x30\xb2\x14\xc2", // missingValue = -22222222222 // data to encode "\x00\x00\x00\x00", // 0.0 "\x12\xbf\x1f\x49", // 654321.123 "\x00\x00\x80\x00", // Smallest available, internal missing value for short_real (1.17549435082229e-38) "\xff\xff\x7f\xff", // Lowest available, internal missing value for short_real2 (-3.40282346638529e+38) "\x00\x00\x80\x7f", // +inf "\x00\x00\x80\xff", // -inf "\x7f\xbf\xff\xff", // NaN (signalling) "\x7f\xff\xff\xff", // NaN (quiet) }; // Loop through endiannesses for the source data for (int i = 0; i < 4; i++) { bool bigEndianOutput = (i % 2 == 0); bool secondCodec = (i > 1); std::vector data; for (size_t j = 0; j < sizeof(expected_data) / sizeof(const char*); j++) { size_t len = (j == 0 || j > 3) ? 4 : 8; data.insert(data.end(), expected_data[j], expected_data[j] + len); if (bigEndianOutput) std::reverse(data.end() - len, data.end()); } // Add the missing value!!! uint32_t mv = secondCodec ? 0xff7fffff : 0x00800000; data.insert(data.end(), reinterpret_cast(&mv), reinterpret_cast(&mv) + 4); if (bigEndianOutput) std::reverse(data.end() - 4, data.end()); // Initialise codecs std::unique_ptr c; if (bigEndianOutput == eckit::system::SystemInfo::isBigEndian()) { if (secondCodec) { c.reset(new CodecShortReal2(odc::api::REAL)); } else { c.reset(new CodecShortReal(odc::api::REAL)); } } else { if (secondCodec) { c.reset(new CodecShortReal2(odc::api::REAL)); } else { c.reset(new CodecShortReal(odc::api::REAL)); } } c->missingValue(customMissingValue); EXPECT(!c->hasMissing()); // Statistics in writing order c->gatherStats(0.0); c->gatherStats(654321.123); c->gatherStats(1.17549435082229e-38); c->gatherStats(-3.40282346638529e+38); double v = *reinterpret_cast(&inf_bits); EXPECT(std::isinf(v)); EXPECT(v > 0); c->gatherStats(v); v = *reinterpret_cast(&neg_inf_bits); EXPECT(std::isinf(v)); EXPECT(v < 0); c->gatherStats(v); v = static_cast(*reinterpret_cast(&sig_nan_bits)); EXPECT(std::isnan(v)); c->gatherStats(v); v = static_cast(*reinterpret_cast(&quiet_nan_bits)); EXPECT(std::isnan(v)); c->gatherStats(v); c->gatherStats(customMissingValue); // Detect that we have added a missing value EXPECT(c->hasMissing()); // Encode the header to the data stream EndianCodecSave ds(bigEndianOutput, *c); EXPECT(ds.position() == eckit::Offset(expectedHdrSize)); // Encode the data to wherever we want it // Expect 4 bytes per element // n.b. If we directly supply the float that is equivalent to the internal missing value, it // is just treated as missing on read, without flagging hasMissing(). We do this here // just to demonstrate. See ODB-367 char* posNext; EXPECT((posNext = c->encode(ds.get(), 0.0)) == (ds.get() + 4)); ds.set(posNext); EXPECT((posNext = c->encode(ds.get(), 654321.123)) == (ds.get() + 4)); ds.set(posNext); size_t offsetMissing = 0; if (secondCodec) { EXPECT((posNext = c->encode(ds.get(), 1.17549435082229e-38)) == (ds.get() + 4)); ds.set(posNext); EXPECT_THROWS_AS(c->encode(ds.get(), -3.40282346638529e+38), eckit::AssertionFailed); offsetMissing = ds.position(); ds.set(ds.get() + 4); } else { EXPECT_THROWS_AS(c->encode(ds.get(), 1.17549435082229e-38), eckit::AssertionFailed); offsetMissing = ds.position(); ds.set(ds.get() + 4); EXPECT((posNext = c->encode(ds.get(), -3.40282346638529e+38)) == (ds.get() + 4)); ds.set(posNext); } EXPECT((posNext = c->encode(ds.get(), *reinterpret_cast(&inf_bits))) == (ds.get() + 4)); ds.set(posNext); EXPECT((posNext = c->encode(ds.get(), *reinterpret_cast(&neg_inf_bits))) == (ds.get() + 4)); ds.set(posNext); EXPECT((posNext = c->encode(ds.get(), static_cast(*reinterpret_cast(&sig_nan_bits)))) == (ds.get() + 4)); ds.set(posNext); EXPECT((posNext = c->encode(ds.get(), static_cast(*reinterpret_cast(&quiet_nan_bits)))) == (ds.get() + 4)); ds.set(posNext); EXPECT((posNext = c->encode(ds.get(), customMissingValue)) == (ds.get() + 4)); ds.set(posNext); // Check we have the data we expect size_t data_size = (9 * 4); EXPECT(ds.position() == eckit::Offset(expectedHdrSize + data_size)); // eckit::Log::info() << "DATA: " << std::endl; // for (size_t n = 0; n < expectedHdrSize + data_size; n++) { // eckit::Log::info() << std::hex << int(data[n]) << " " << int(ds.data()[n]) << std::endl; // if (int(data[n]) != int(ds.data()[n])) // eckit::Log::info() << "******************************" << std::endl; // } // The missing values won't be encoded when they are hit, so skip them in the data test EXPECT(offsetMissing != 0); EXPECT(::memcmp(&data[0], ds.data(), offsetMissing) == 0); EXPECT(::memcmp(&data[0] + offsetMissing + 4, ds.data() + offsetMissing + 4, expectedHdrSize + data_size - offsetMissing - 4) == 0); } } CASE("32bit integers are as-is") { const size_t expectedHdrSize = 28; const char* expected_data[] = { // Codec header "\x01\x00\x00\x00", // has missing value "\x00\x00\x00\x00\x00\x00\xe0\xc1", // minimum = -2147483648 "\x00\x00\xc0\xff\xff\xff\xdf\x41", // maximum = 2147483647 "\x00\x00\x00\x1c\xaf\x7d\xaa\x41", // missing value = 222222222 // data to encode "\x00\x00\x00\x00", // 0.0 "\xff\xff\xff\xff", // -1 "\xff\xff\xff\x7f", // 2147483647 == largest "\x00\x00\x00\x80", // -2147483648 == smallest "\x8e\xd7\x3e\x0d", // 222222222 == missingValue (unused by codec) "\x96\x28\x9c\xff" // -6543210 }; // Loop through endiannesses for the source data for (int i = 0; i < 2; i++) { bool bigEndianOutput = (i == 1); std::vector data; for (size_t j = 0; j < sizeof(expected_data) / sizeof(const char*); j++) { size_t len = (j == 0 || j > 3) ? 4 : 8; data.insert(data.end(), expected_data[j], expected_data[j] + len); if (bigEndianOutput) std::reverse(data.end() - len, data.end()); } // Initialise codecs std::unique_ptr c; if (bigEndianOutput == eckit::system::SystemInfo::isBigEndian()) { c.reset(new CodecInt32(odc::api::INTEGER)); } else { c.reset(new CodecInt32(odc::api::INTEGER)); } c->missingValue(222222222); EXPECT(!c->hasMissing()); // Statistics in writing order c->gatherStats(0); c->gatherStats(-1); c->gatherStats(2147483647); c->gatherStats(-2147483648); EXPECT(!c->hasMissing()); c->gatherStats(222222222); EXPECT(c->hasMissing()); c->gatherStats(-6543210); // Encode the header to the data stream EndianCodecSave ds(bigEndianOutput, *c); EXPECT(ds.position() == eckit::Offset(expectedHdrSize)); // Encode the data to wherever we want it // Expect 4 bytes per element char* posNext; EXPECT((posNext = c->encode(ds.get(), 0)) == (ds.get() + 4)); ds.set(posNext); EXPECT((posNext = c->encode(ds.get(), -1)) == (ds.get() + 4)); ds.set(posNext); EXPECT((posNext = c->encode(ds.get(), 2147483647)) == (ds.get() + 4)); ds.set(posNext); EXPECT((posNext = c->encode(ds.get(), -2147483648)) == (ds.get() + 4)); ds.set(posNext); EXPECT((posNext = c->encode(ds.get(), 222222222)) == (ds.get() + 4)); ds.set(posNext); EXPECT((posNext = c->encode(ds.get(), -6543210)) == (ds.get() + 4)); ds.set(posNext); // Check we have the data we expect size_t data_size = (6 * 4); EXPECT(ds.position() == eckit::Offset(expectedHdrSize + data_size)); // eckit::Log::info() << "DATA: " << std::endl; // for (size_t n = 0; n < expectedHdrSize + data_size; n++) { // eckit::Log::info() << std::hex << int(data[n]) << " " << int(ds.data()[n]) << std::endl; // if (int(data[n]) != int(ds.data()[n])) // eckit::Log::info() << "******************************" << std::endl; // } EXPECT(::memcmp(&data[0], ds.data(), expectedHdrSize + data_size) == 0); } } CASE("16bit integers are stored with an offset. This need not (strictly) be integral!!") { // n.b. we use a non-standard, non-integral minimum to demonstrate the offset behaviour. // Use a curious, custom missingValue to show it is being used. const double customMissingValue = 6.54565456545599971850917315786e-123; const double baseVal = -123.45; const size_t expectedHdrSize = 28; const char* expected_data[] = { // Codec header "\x01\x00\x00\x00", // has missing value "\xcd\xcc\xcc\xcc\xcc\xdc\x5e\xc0", // minimum = -123.45 "\x9a\x99\x99\x99\x71\xf0\xef\x40", // maximum = -123.45 + 65535 "\x04\x4f\xab\xa0\xe4\x4e\x91\x26", // missing value = 6.54565456545599971850917315786e-123 // data to encode "\x00\x00", // 0.0 "\xff\xff", // 65535 and the missing value "\xff\x7f", // 32767 (no negatives) "\x00\x80", // 32768 (no negatives) "\x39\x30" // 12345 }; // Loop through endiannesses for the source data for (int i = 0; i < 4; i++) { bool bigEndianOutput = (i % 2 == 0); bool withMissing = (i > 1); // eckit::Log::info() << "---------------------------------------------------------" << std::endl; // eckit::Log::info() << "ITERATION: " << i << std::endl; // eckit::Log::info() << "big endian: " << (bigEndianOutput ? "T" : "F") << std::endl; // eckit::Log::info() << "with missing: " << (withMissing ? "T" : "F") << std::endl; // eckit::Log::info() << "---------------------------------------------------------" << std::endl; std::vector data; for (size_t j = 0; j < sizeof(expected_data) / sizeof(const char*); j++) { size_t len = (j == 0) ? 4 : (j > 3) ? 2 : 8; data.insert(data.end(), expected_data[j], expected_data[j] + len); if (bigEndianOutput) std::reverse(data.end() - len, data.end()); } // The missing value is odd. It should be handled properly for the Missing codec, but things are just mangled // if we do the direct codec. // See ODB-370 and ODB-371 uint16_t mv = withMissing ? 0xffff : 0x007b; data.insert(data.end(), reinterpret_cast(&mv), reinterpret_cast(&mv) + 2); if (bigEndianOutput) std::reverse(data.end() - 2, data.end()); // Initialise codecs std::unique_ptr c; if (bigEndianOutput == eckit::system::SystemInfo::isBigEndian()) { if (withMissing) { c.reset(new CodecInt16Missing(odc::api::INTEGER)); } else { c.reset(new CodecInt16(odc::api::INTEGER)); } } else { if (withMissing) { c.reset(new CodecInt16Missing(odc::api::INTEGER)); } else { c.reset(new CodecInt16(odc::api::INTEGER)); } } c->missingValue(customMissingValue); EXPECT(!c->hasMissing()); // Statistics in writing order c->gatherStats(baseVal + 0.0); c->gatherStats(baseVal + 65535); // n.b. This triggers a missingValue in Int16Missing. See ODB-369 c->gatherStats(baseVal + 32767); c->gatherStats(baseVal + 32768); c->gatherStats(baseVal + 12345); EXPECT(!c->hasMissing()); c->gatherStats(customMissingValue); EXPECT(c->hasMissing()); // See ODB-371 // Encode the header to the data stream EndianCodecSave ds(bigEndianOutput, *c); EXPECT(ds.position() == eckit::Offset(expectedHdrSize)); // Encode the data to wherever we want it // Expect 2 bytes per element // n.b. If we directly supply the value that is equivalent to the internal missing value, it // is just treated as missing on read, without flagging hasMissing(). We do this here // just to demonstrate. See ODB-369 char* posNext; EXPECT((posNext = c->encode(ds.get(), baseVal + 0.0)) == (ds.get() + 2)); ds.set(posNext); if (withMissing) { EXPECT_THROWS_AS(c->encode(ds.get(), baseVal + 65535), eckit::AssertionFailed); EXPECT((posNext = c->encode(ds.get(), customMissingValue)) == (ds.get() + 2)); // Ensure data is the same } else { EXPECT((posNext = c->encode(ds.get(), baseVal + 65535)) == (ds.get() + 2)); } ds.set(posNext); EXPECT((posNext = c->encode(ds.get(), baseVal + 32767)) == (ds.get() + 2)); ds.set(posNext); EXPECT((posNext = c->encode(ds.get(), baseVal + 32768)) == (ds.get() + 2)); ds.set(posNext); EXPECT((posNext = c->encode(ds.get(), baseVal + 12345)) == (ds.get() + 2)); ds.set(posNext); EXPECT((posNext = c->encode(ds.get(), customMissingValue)) == (ds.get() + 2)); ds.set(posNext); // Check we have the data we expect size_t data_size = (6 * 2); EXPECT(ds.position() == eckit::Offset(expectedHdrSize + data_size)); // eckit::Log::info() << "DATA: " << std::endl; // for (size_t n = 0; n < expectedHdrSize + data_size; n++) { // eckit::Log::info() << std::hex << int(data[n]) << " " << int(ds.data()[n]) << std::endl; // if (int(data[n]) != int(ds.data()[n])) // eckit::Log::info() << "******************************" << std::endl; // } EXPECT(::memcmp(&data[0], ds.data(), expectedHdrSize + data_size) == 0); } } CASE("8bit integers are stored with an offset. This need not (strictly) be integral!!") { // n.b. we use a non-standard, non-integral minimum to demonstrate the offset behaviour. // Use a curious, custom missingValue to show it is being used. const double customMissingValue = 6.54565456545599971850917315786e-123; const double baseVal = -5000.5; const size_t expectedHdrSize = 28; const char* expected_data[] = { // Codec header "\x01\x00\x00\x00", // has missing value "\x00\x00\x00\x00\x80\x88\xb3\xc0", // minimum = -5000.5 "\x00\x00\x00\x00\x80\x89\xb2\xc0", // maximum = -5000.5 + 255 "\x04\x4f\xab\xa0\xe4\x4e\x91\x26", // missing value = 6.54565456545599971850917315786e-123 }; // Loop through endiannesses for the source data for (int i = 0; i < 4; i++) { bool bigEndianOutput = (i % 2 == 0); bool withMissing = (i > 1); // eckit::Log::info() << "---------------------------------------------------------" << std::endl; // eckit::Log::info() << "ITERATION: " << i << std::endl; // eckit::Log::info() << "big endian: " << (bigEndianOutput ? "T" : "F") << std::endl; // eckit::Log::info() << "with missing: " << (withMissing ? "T" : "F") << std::endl; // eckit::Log::info() << "---------------------------------------------------------" << std::endl; std::vector data; for (size_t j = 0; j < sizeof(expected_data) / sizeof(const char*); j++) { size_t len = (j == 0) ? 4 : 8; data.insert(data.end(), expected_data[j], expected_data[j] + len); if (bigEndianOutput) std::reverse(data.end() - len, data.end()); } // Add all of the data values for (int n = 0; n < 256; n++) { data.push_back(static_cast(n)); } // n.b. we can end up with garbage for the missing value... in this case it will be 0x88... // See ODB-371 ODB-370 data.push_back(withMissing ? 0xff : 0x88); // Initialise the codecs std::unique_ptr c; if (bigEndianOutput == eckit::system::SystemInfo::isBigEndian()) { if (withMissing) { c.reset(new CodecInt8Missing(odc::api::INTEGER)); } else { c.reset(new CodecInt8(odc::api::INTEGER)); } } else { if (withMissing) { c.reset(new CodecInt8Missing(odc::api::INTEGER)); } else { c.reset(new CodecInt8(odc::api::INTEGER)); } } c->missingValue(customMissingValue); EXPECT(!c->hasMissing()); // Statistics in writing order // n.b. n == 255 will be silently promoted to missing if using Int8Missing. See ODB-369 for (size_t n = 0; n < 256; n++) { c->gatherStats(baseVal + n); } EXPECT(!c->hasMissing()); c->gatherStats(customMissingValue); EXPECT(c->hasMissing()); // See ODB-71 // Encode the header to the data stream EndianCodecSave ds(bigEndianOutput, *c); EXPECT(ds.position() == eckit::Offset(expectedHdrSize)); // Encode the data to wherever we want it // Expect 1 bytes per element // n.b. If we directly supply the value that is equivalent to the internal missing value, it // is just treated as missing on read, without flagging hasMissing(). We do this here // just to demonstrate. See ODB-369 char* posNext; for (size_t n = 0; n < 255; n++) { EXPECT((posNext = c->encode(ds.get(), baseVal + n)) == (ds.get() + 1)); ds.set(posNext); } if (withMissing) { EXPECT_THROWS_AS(c->encode(ds.get(), baseVal + 255), eckit::AssertionFailed); EXPECT((posNext = c->encode(ds.get(), customMissingValue)) == (ds.get() + 1)); // Ensure data is the same } else { EXPECT((posNext = c->encode(ds.get(), baseVal + 255)) == (ds.get() + 1)); } ds.set(posNext); EXPECT((posNext = c->encode(ds.get(), customMissingValue)) == (ds.get() + 1)); ds.set(posNext); // Check we have the data we expect size_t data_size = (257 * 1); EXPECT(ds.position() == eckit::Offset(expectedHdrSize + data_size)); // eckit::Log::info() << "DATA: " << std::endl; // for (size_t n = 0; n < expectedHdrSize + data_size; n++) { // eckit::Log::info() << std::hex << int(data[n]) << " " << int(ds.data()[n]) << std::endl; // if (int(data[n]) != int(ds.data()[n])) // eckit::Log::info() << "******************************" << std::endl; // } EXPECT(::memcmp(&data[0], ds.data(), expectedHdrSize + data_size) == 0); } } CASE("Character strings can be stored in a flat list, and indexed") { // n.b. no missing values const size_t expectedHdrSize = 144; const char* expected_data[] = { // Codec header "\x00\x00\x00\x00", // 0 = hasMissing "\x6f\x70\x71\x72\x73\x74\x75\x76", // min unspecified == "opqrstuv" "\x00\x00\x00\x00\x00\x00\x00\x00", // max unspecified "\x00\x00\x00\x00\x00\x00\x00\x00", // missingValue unspecified // How many strings are there in the table? "\x06\x00\x00\x00", // String data (prepended with lengths) // length, data, "cnt (discarded)", index // The order of these is a matter of implementation detail of the internal "hash table" // This is what happens to happen in current ODB-API "\x02\x00\x00\x00", "ab", "\x00\x00\x00\x00", "\x00\x00\x00\x00", // This string is too short "\x06\x00\x00\x00", "ghijkl", "\x00\x00\x00\x00", "\x01\x00\x00\x00", "\x08\x00\x00\x00", "mnopqrst", "\x00\x00\x00\x00", "\x02\x00\x00\x00", // 8-byte length "\x08\x00\x00\x00", "uvwxyzab", "\x00\x00\x00\x00", "\x03\x00\x00\x00", // n.b. truncated. "\x08\x00\x00\x00", "ghijklmn", "\x00\x00\x00\x00", "\x04\x00\x00\x00", "\x08\x00\x00\x00", "opqrstuv", "\x00\x00\x00\x00", "\x05\x00\x00\x00", }; // Loop through endiannesses for the source data for (int i = 0; i < 4; i++) { bool bigEndianOutput = (i % 2 == 0); bool bits16 = (i > 1); // eckit::Log::info() << "---------------------------------------------------------" << std::endl; // eckit::Log::info() << "ITERATION: " << i << std::endl; // eckit::Log::info() << "big endian: " << (bigEndianOutput ? "T" : "F") << std::endl; // eckit::Log::info() << "16-bit: " << (bits16 ? "T" : "F") << std::endl; // eckit::Log::info() << "---------------------------------------------------------" << std::endl; std::vector data; for (size_t j = 0; j < sizeof(expected_data) / sizeof(const char*); j++) { size_t len = (j < 5) ? ((j == 0 || j == 4) ? 4 : 8) : ((j + 2) % 4 == 0 ? ::strlen(expected_data[j]) : 4); data.insert(data.end(), expected_data[j], expected_data[j] + len); // n.b. Don't reverse the endianness of the string data. if (bigEndianOutput && !((j > 5) && ((j + 2) % 4 == 0))) std::reverse(data.end() - len, data.end()); } // Which strings do we wish to encode (look at them in reverse. nb refers to index column) for (int n = 0; n < 6; n++) { if (bits16 && bigEndianOutput) data.push_back(0); data.push_back(static_cast(n)); if (bits16 && !bigEndianOutput) data.push_back(0); } // Initialise codecs std::unique_ptr c; if (bigEndianOutput == eckit::system::SystemInfo::isBigEndian()) { if (bits16) { c.reset(new CodecInt16String(odc::api::STRING)); } else { c.reset(new CodecInt8String(odc::api::STRING)); } } else { if (bits16) { c.reset(new CodecInt16String(odc::api::STRING)); } else { c.reset(new CodecInt8String(odc::api::STRING)); } } c->missingValue(0.0); EXPECT(!c->hasMissing()); // Statistics in writing order // Allocate more, and aligned, storage than required --> the undefined behaviour sanitizer is happy with the // test alignas(sizeof(double)) const char s1[16] = "ab"; // check that we can handle short strings! alignas(sizeof(double)) const char s2[16] = "ghijkl"; alignas(sizeof(double)) const char s3[16] = "mnopqrst"; alignas(sizeof(double)) const char s4[16] = "uvwxyzabcdef"; // n.b. will be trucated to 8-bytes alignas(sizeof(double)) const char s5[16] = "ghijklmn"; alignas(sizeof(double)) const char s6[16] = "opqrstuv"; // n.b. these casts are a bit dubious in terms of memory access. May go beyond ends of s1, s2 c->gatherStats(*reinterpret_cast(s1)); c->gatherStats(*reinterpret_cast(s2)); c->gatherStats(*reinterpret_cast(s3)); c->gatherStats(*reinterpret_cast(s4)); c->gatherStats(*reinterpret_cast(s5)); c->gatherStats(*reinterpret_cast(s6)); EXPECT(!c->hasMissing()); // Encode the header to the data stream EndianCodecSave ds(bigEndianOutput, *c); EXPECT(ds.position() == eckit::Offset(expectedHdrSize)); // Encode the data to wherever we want it // Expect 1 or 2 bytes per element char* posNext; EXPECT((posNext = c->encode(ds.get(), *reinterpret_cast(s1))) == (ds.get() + (bits16 ? 2 : 1))); ds.set(posNext); EXPECT((posNext = c->encode(ds.get(), *reinterpret_cast(s2))) == (ds.get() + (bits16 ? 2 : 1))); ds.set(posNext); EXPECT((posNext = c->encode(ds.get(), *reinterpret_cast(s3))) == (ds.get() + (bits16 ? 2 : 1))); ds.set(posNext); EXPECT((posNext = c->encode(ds.get(), *reinterpret_cast(s4))) == (ds.get() + (bits16 ? 2 : 1))); ds.set(posNext); EXPECT((posNext = c->encode(ds.get(), *reinterpret_cast(s5))) == (ds.get() + (bits16 ? 2 : 1))); ds.set(posNext); EXPECT((posNext = c->encode(ds.get(), *reinterpret_cast(s6))) == (ds.get() + (bits16 ? 2 : 1))); ds.set(posNext); // Check we have the data we expect size_t data_size = (6 * (bits16 ? 2 : 1)); EXPECT(ds.position() == eckit::Offset(expectedHdrSize + data_size)); // eckit::Log::info() << "DATA: " << std::endl; // for (size_t n = 0; n < expectedHdrSize + data_size; n++) { // eckit::Log::info() << std::hex << int(data[n]) << " " << int(ds.data()[n]) << std::endl; // if (int(data[n]) != int(ds.data()[n])) // eckit::Log::info() << "******************************" << std::endl; // } EXPECT(::memcmp(&data[0], ds.data(), expectedHdrSize + data_size) == 0); } } CASE("Character strings can be stored in a flat list, and indexed, and longer than 8 bytes") { // n.b. no missing values const size_t expectedHdrSize = 156; const char* expected_data[] = { // Codec header "\x00\x00\x00\x00", // 0 = hasMissing "\x6f\x70\x71\x72\x73\x74\x75\x76", // min unspecified == "opqrstuv" "\x00\x00\x00\x00\x00\x00\x00\x00", // max unspecified "\x00\x00\x00\x00\x00\x00\x00\x00", // missingValue unspecified // How many strings are there in the table? "\x06\x00\x00\x00", // String data (prepended with lengths) // length, data, "cnt (discarded)", index // The order of these is a matter of implementation detail of the internal "hash table" // This is what happens to happen in current ODB-API "\x02\x00\x00\x00", "ab", "\x00\x00\x00\x00", "\x00\x00\x00\x00", // This string is too short "\x06\x00\x00\x00", "ghijkl", "\x00\x00\x00\x00", "\x01\x00\x00\x00", "\x08\x00\x00\x00", "mnopqrst", "\x00\x00\x00\x00", "\x02\x00\x00\x00", // 8-byte length "\x0c\x00\x00\x00", "uvwxyzabcdef", "\x00\x00\x00\x00", "\x03\x00\x00\x00", // n.b. truncated. "\x10\x00\x00\x00", "ghijklmnopqrstuv", "\x00\x00\x00\x00", "\x04\x00\x00\x00", "\x08\x00\x00\x00", "opqrstuv", "\x00\x00\x00\x00", "\x05\x00\x00\x00", }; // Loop through endiannesses for the source data for (int i = 0; i < 4; i++) { bool bigEndianOutput = (i % 2 == 0); bool bits16 = (i > 1); // eckit::Log::info() << "---------------------------------------------------------" << std::endl; // eckit::Log::info() << "ITERATION: " << i << std::endl; // eckit::Log::info() << "big endian: " << (bigEndianOutput ? "T" : "F") << std::endl; // eckit::Log::info() << "16-bit: " << (bits16 ? "T" : "F") << std::endl; // eckit::Log::info() << "---------------------------------------------------------" << std::endl; std::vector data; for (size_t j = 0; j < sizeof(expected_data) / sizeof(const char*); j++) { size_t len = (j < 5) ? ((j == 0 || j == 4) ? 4 : 8) : ((j + 2) % 4 == 0 ? ::strlen(expected_data[j]) : 4); data.insert(data.end(), expected_data[j], expected_data[j] + len); // n.b. Don't reverse the endianness of the string data. if (bigEndianOutput && !((j > 5) && ((j + 2) % 4 == 0))) std::reverse(data.end() - len, data.end()); } // Which strings do we wish to encode (look at them in reverse. nb refers to index column) for (int n = 0; n < 6; n++) { if (bits16 && bigEndianOutput) data.push_back(0); data.push_back(static_cast(n)); if (bits16 && !bigEndianOutput) data.push_back(0); } // Initialise codecs std::unique_ptr c; if (bigEndianOutput == eckit::system::SystemInfo::isBigEndian()) { if (bits16) { c.reset(new CodecInt16String(odc::api::STRING)); } else { c.reset(new CodecInt8String(odc::api::STRING)); } } else { if (bits16) { c.reset(new CodecInt16String(odc::api::STRING)); } else { c.reset(new CodecInt8String(odc::api::STRING)); } } c->missingValue(0.0); EXPECT(!c->hasMissing()); // Allow strings up to 16 bytes c->dataSizeDoubles(2); // Statistics in writing order // Allocate more, and aligned, storage than required --> the undefined behaviour sanitizer is happy with the // test alignas(sizeof(double)) const char s1[24] = "ab"; // check that we can handle short strings! alignas(sizeof(double)) const char s2[24] = "ghijkl"; alignas(sizeof(double)) const char s3[24] = "mnopqrst"; alignas(sizeof(double)) const char s4[24] = "uvwxyzabcdef"; // n.b. will NOT be trucated to 8-bytes alignas(sizeof(double)) const char s5[24] = "ghijklmnopqrstuvwxyz"; // n.b. will be truncated to 16-bytes alignas(sizeof(double)) const char s6[24] = "opqrstuv"; // n.b. these casts are a bit dubious in terms of memory access. May go beyond ends of s1, s2 c->gatherStats(*reinterpret_cast(s1)); c->gatherStats(*reinterpret_cast(s2)); c->gatherStats(*reinterpret_cast(s3)); c->gatherStats(*reinterpret_cast(s4)); c->gatherStats(*reinterpret_cast(s5)); c->gatherStats(*reinterpret_cast(s6)); EXPECT(!c->hasMissing()); // Encode the header to the data stream EndianCodecSave ds(bigEndianOutput, *c); EXPECT(ds.position() == eckit::Offset(expectedHdrSize)); // Encode the data to wherever we want it // Expect 1 or 2 bytes per element char* posNext; EXPECT((posNext = c->encode(ds.get(), *reinterpret_cast(s1))) == (ds.get() + (bits16 ? 2 : 1))); ds.set(posNext); EXPECT((posNext = c->encode(ds.get(), *reinterpret_cast(s2))) == (ds.get() + (bits16 ? 2 : 1))); ds.set(posNext); EXPECT((posNext = c->encode(ds.get(), *reinterpret_cast(s3))) == (ds.get() + (bits16 ? 2 : 1))); ds.set(posNext); EXPECT((posNext = c->encode(ds.get(), *reinterpret_cast(s4))) == (ds.get() + (bits16 ? 2 : 1))); ds.set(posNext); EXPECT((posNext = c->encode(ds.get(), *reinterpret_cast(s5))) == (ds.get() + (bits16 ? 2 : 1))); ds.set(posNext); EXPECT((posNext = c->encode(ds.get(), *reinterpret_cast(s6))) == (ds.get() + (bits16 ? 2 : 1))); ds.set(posNext); // Check we have the data we expect size_t data_size = (6 * (bits16 ? 2 : 1)); EXPECT(ds.position() == eckit::Offset(expectedHdrSize + data_size)); // eckit::Log::info() << "DATA: " << std::endl; // for (size_t n = 0; n < expectedHdrSize + data_size; n++) { // eckit::Log::info() << std::hex << int(data[n]) << " " << int(ds.data()[n]) << std::endl; // if (int(data[n]) != int(ds.data()[n])) // eckit::Log::info() << "******************************" << std::endl; // } EXPECT(::memcmp(&data[0], ds.data(), expectedHdrSize + data_size) == 0); } } // ------------------------------------------------------------------------------------------------------ int main(int argc, char* argv[]) { return run_tests(argc, argv); } odc-1.6.3/tests/core/test_encode_odb.cc0000664000175000017500000004500215146027420020175 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/eckit_config.h" #include "eckit/exception/Exceptions.h" #include "eckit/io/Buffer.h" #include "eckit/io/MemoryHandle.h" #include "eckit/testing/Test.h" #include "odc/Reader.h" #include "odc/Writer.h" #include "odc/api/ColumnType.h" using namespace eckit::testing; #if __cplusplus <= 199711L const float float_lowest = -std::numeric_limits::max(); const double double_lowest = -std::numeric_limits::max(); #else const float float_lowest = std::numeric_limits::lowest(); const double double_lowest = std::numeric_limits::lowest(); #endif // ------------------------------------------------------------------------------------------------------ CASE("Columns are initialised correctly for writing") { eckit::Buffer buf(4096); eckit::MemoryHandle dh(buf); odc::Writer<> oda(dh); odc::Writer<>::iterator writer = oda.begin(); // Set up the columns writer->setNumberOfColumns(10); // Now we create the columns writer->setColumn(0, "int", odc::api::INTEGER); writer->setColumn(1, "real", odc::api::REAL); writer->setColumn(2, "str", odc::api::STRING); writer->columns()[2]->dataSizeDoubles(3); writer->setColumn(3, "bitf", odc::api::BITFIELD); writer->setColumn(4, "dbl", odc::api::DOUBLE); writer->setColumn(5, "int2", odc::api::INTEGER); writer->setColumn(6, "real2", odc::api::REAL); writer->setColumn(7, "str2", odc::api::STRING); writer->setColumn(8, "bitf2", odc::api::BITFIELD); writer->setColumn(9, "dbl2", odc::api::DOUBLE); // Check that the columns are correctly created EXPECT(writer->columns()[0]->type() == odc::api::INTEGER); EXPECT(writer->columns()[1]->type() == odc::api::REAL); EXPECT(writer->columns()[2]->type() == odc::api::STRING); EXPECT(writer->columns()[3]->type() == odc::api::BITFIELD); EXPECT(writer->columns()[4]->type() == odc::api::DOUBLE); EXPECT(writer->columns()[5]->type() == odc::api::INTEGER); EXPECT(writer->columns()[6]->type() == odc::api::REAL); EXPECT(writer->columns()[7]->type() == odc::api::STRING); EXPECT(writer->columns()[8]->type() == odc::api::BITFIELD); EXPECT(writer->columns()[9]->type() == odc::api::DOUBLE); // ... with the correct name EXPECT(writer->columns()[0]->name() == "int"); EXPECT(writer->columns()[1]->name() == "real"); EXPECT(writer->columns()[2]->name() == "str"); EXPECT(writer->columns()[3]->name() == "bitf"); EXPECT(writer->columns()[4]->name() == "dbl"); EXPECT(writer->columns()[5]->name() == "int2"); EXPECT(writer->columns()[6]->name() == "real2"); EXPECT(writer->columns()[7]->name() == "str2"); EXPECT(writer->columns()[8]->name() == "bitf2"); EXPECT(writer->columns()[9]->name() == "dbl2"); // ... and the correct default codecs EXPECT(writer->columns()[0]->coder().name() == "int32"); EXPECT(writer->columns()[1]->coder().name() == "long_real"); EXPECT(writer->columns()[2]->coder().name() == "chars"); EXPECT(writer->columns()[3]->coder().name() == "int32"); EXPECT(writer->columns()[4]->coder().name() == "long_real"); EXPECT(writer->columns()[5]->coder().name() == "int32"); EXPECT(writer->columns()[6]->coder().name() == "long_real"); EXPECT(writer->columns()[7]->coder().name() == "chars"); EXPECT(writer->columns()[8]->coder().name() == "int32"); EXPECT(writer->columns()[9]->coder().name() == "long_real"); // ... and the correct expected data sizes EXPECT(writer->columns()[0]->coder().dataSizeDoubles() == 1); EXPECT(writer->columns()[1]->coder().dataSizeDoubles() == 1); EXPECT(writer->columns()[2]->coder().dataSizeDoubles() == 3); EXPECT(writer->columns()[3]->coder().dataSizeDoubles() == 1); EXPECT(writer->columns()[4]->coder().dataSizeDoubles() == 1); EXPECT(writer->columns()[5]->coder().dataSizeDoubles() == 1); EXPECT(writer->columns()[6]->coder().dataSizeDoubles() == 1); EXPECT(writer->columns()[7]->coder().dataSizeDoubles() == 1); EXPECT(writer->columns()[8]->coder().dataSizeDoubles() == 1); EXPECT(writer->columns()[9]->coder().dataSizeDoubles() == 1); } CASE("If out-of range columns are created, exceptions are thrown") { eckit::Buffer buf(4096); eckit::MemoryHandle dh(buf); odc::Writer<> oda(dh); odc::Writer<>::iterator writer = oda.begin(); writer->setNumberOfColumns(10); writer->setColumn(2, "str", odc::api::STRING); // This is fine // If we create columns out of range, it throws exceptions EXPECT_THROWS_AS(writer->setColumn(11, "badnum", odc::api::INTEGER), eckit::AssertionFailed); } CASE("If columns are created with invalid types, exceptions are thrown") { eckit::Buffer buf(4096); eckit::MemoryHandle dh(buf); odc::Writer<> oda(dh); odc::Writer<>::iterator writer = oda.begin(); // Set up the columns writer->setNumberOfColumns(10); writer->setColumn(6, "real", odc::api::REAL); // This is fine // We cannot create a column of "IGNORE" type, or any type that is not listed in the enum EXPECT_THROWS_AS(writer->setColumn(0, "ignore", odc::api::IGNORE), eckit::AssertionFailed); EXPECT_THROWS_AS(writer->setColumn(0, "ignore", static_cast(123)), eckit::AssertionFailed); } CASE("Columns names must be unique") { // See issue ODB-372 eckit::Buffer buf(4096); eckit::MemoryHandle dh(buf); odc::Writer<> oda(dh); odc::Writer<>::iterator writer = oda.begin(); // Set up the columns writer->setNumberOfColumns(10); writer->setColumn(0, "int", odc::api::INTEGER); writer->setColumn(1, "real", odc::api::REAL); writer->setColumn(2, "str", odc::api::STRING); writer->setColumn(3, "bitf", odc::api::BITFIELD); writer->setColumn(4, "dbl", odc::api::DOUBLE); EXPECT_THROWS_AS(writer->setColumn(5, "int", odc::api::INTEGER), eckit::SeriousBug); EXPECT_THROWS_AS(writer->setColumn(6, "real", odc::api::REAL), eckit::SeriousBug); EXPECT_THROWS_AS(writer->setColumn(7, "str", odc::api::STRING), eckit::SeriousBug); EXPECT_THROWS_AS(writer->setColumn(8, "bitf", odc::api::BITFIELD), eckit::SeriousBug); EXPECT_THROWS_AS(writer->setColumn(9, "dbl", odc::api::DOUBLE), eckit::SeriousBug); } CASE("Data is encoded and read back correctly") { const int32_t i1 = 987654321; const int32_t i2 = -1; const int32_t i3 = std::numeric_limits::min(); const int32_t i4 = std::numeric_limits::max(); const int32_t i5 = 0; const int32_t i6 = -654321; const float f1 = std::numeric_limits::min(); const float f2 = std::numeric_limits::max(); const float f3 = 0.0; const float f4 = float_lowest; const float f5 = static_cast(654321.123); const float f6 = static_cast(-123456.789e-21); alignas(sizeof(double)) const char s1[] = "a-string-longstrvvvvlong"; alignas(sizeof(double)) const char s2[] = "string-2"; alignas(sizeof(double)) const char s3[] = "string-3-LLong"; alignas(sizeof(double)) const char s4[] = "string-4"; alignas(sizeof(double)) const char s5[] = "string-5-LLong"; alignas(sizeof(double)) const char s6[] = "string-6"; alignas(sizeof(double)) const int32_t b1 = static_cast((uint32_t)std::numeric_limits::min()); alignas(sizeof(double)) const int32_t b2 = static_cast((uint32_t)std::numeric_limits::max()); alignas(sizeof(double)) const int32_t b3 = static_cast((uint32_t)0); alignas(sizeof(double)) const int32_t b4 = static_cast((uint32_t)0xff00ff00); alignas(sizeof(double)) const int32_t b5 = static_cast((uint32_t)0x00ff00ff); alignas(sizeof(double)) const int32_t b6 = static_cast((uint32_t)0xfedcba98); const double d1 = std::numeric_limits::min(); const double d2 = std::numeric_limits::max(); const double d3 = double_lowest; const double d4 = 0.0; const double d5 = -123456789.0123; const double d6 = 987654321.987e-56; // See issue ODB-372 eckit::Buffer buf(4096); eckit::MemoryHandle dhWrite(buf); { odc::Writer<> oda(dhWrite); odc::Writer<>::iterator writer = oda.begin(); // Set up the columns writer->setNumberOfColumns(10); writer->setColumn(0, "int", odc::api::INTEGER); writer->setColumn(1, "real", odc::api::REAL); writer->setColumn(2, "str", odc::api::STRING); writer->columns()[2]->dataSizeDoubles(3); writer->setColumn(3, "bitf", odc::api::BITFIELD); writer->setColumn(4, "dbl", odc::api::DOUBLE); writer->setColumn(5, "int2", odc::api::INTEGER); writer->setColumn(6, "real2", odc::api::REAL); writer->setColumn(7, "str2", odc::api::STRING); writer->setColumn(8, "bitf2", odc::api::BITFIELD); writer->setColumn(9, "dbl2", odc::api::DOUBLE); writer->writeHeader(); // Test that the data offsets are correct EXPECT(writer->dataOffset(0) == 0); EXPECT(writer->dataOffset(1) == 1); EXPECT(writer->dataOffset(2) == 2); EXPECT(writer->dataOffset(3) == 5); EXPECT(writer->dataOffset(4) == 6); EXPECT(writer->dataOffset(5) == 7); EXPECT(writer->dataOffset(6) == 8); EXPECT(writer->dataOffset(7) == 9); EXPECT(writer->dataOffset(8) == 10); EXPECT(writer->dataOffset(9) == 11); // Append 3 rows of data (in two different ways) (*writer)[0] = i1; (*writer)[1] = static_cast(f1); ::strncpy(reinterpret_cast(&(*writer)[2]), s1, 24); // strncpy pads with \0 (*writer)[3] = b1; (*writer)[4] = d1; (*writer)[5] = i2; (*writer)[6] = static_cast(f2); (*writer)[7] = *reinterpret_cast(s2); (*writer)[8] = b2; (*writer)[9] = d2; ++writer; writer->data()[writer->dataOffset(0)] = i3; writer->data()[writer->dataOffset(1)] = static_cast(f3); ::strncpy(reinterpret_cast(&writer->data()[writer->dataOffset(2)]), s3, 24); // strncpy pads with \0 writer->data()[writer->dataOffset(3)] = b3; writer->data()[writer->dataOffset(4)] = d3; writer->data()[writer->dataOffset(5)] = i4; writer->data()[writer->dataOffset(6)] = static_cast(f4); writer->data()[writer->dataOffset(7)] = *reinterpret_cast(s4); writer->data()[writer->dataOffset(8)] = b4; writer->data()[writer->dataOffset(9)] = d4; ++writer; writer->data(0) = i5; writer->data(1) = static_cast(f5); ::strncpy(reinterpret_cast(&writer->data(2)), s5, 24); // strncpy pads with \0 writer->data(3) = b5; writer->data(4) = d5; writer->data(5) = i6; writer->data(6) = static_cast(f6); writer->data(7) = *reinterpret_cast(s6); writer->data(8) = b6; writer->data(9) = d6; ++writer; } // Read everything back { eckit::MemoryHandle dh(buf.data(), static_cast(dhWrite.position())); dh.openForRead(); odc::Reader oda(dh); odc::Reader::iterator reader = oda.begin(); EXPECT(reader->columns().size() == size_t(10)); EXPECT(reader->columns()[0]->dataSizeDoubles() == 1); EXPECT(reader->columns()[1]->dataSizeDoubles() == 1); EXPECT(reader->columns()[2]->dataSizeDoubles() == 3); EXPECT(reader->columns()[3]->dataSizeDoubles() == 1); EXPECT(reader->columns()[4]->dataSizeDoubles() == 1); EXPECT(reader->columns()[5]->dataSizeDoubles() == 1); EXPECT(reader->columns()[6]->dataSizeDoubles() == 1); EXPECT(reader->columns()[7]->dataSizeDoubles() == 1); EXPECT(reader->columns()[8]->dataSizeDoubles() == 1); EXPECT(reader->columns()[9]->dataSizeDoubles() == 1); EXPECT((*reader)[0] == i1); EXPECT((*reader)[1] == static_cast(f1)); EXPECT(::strncmp(reinterpret_cast(&(*reader)[2]), s1, 24) == 0); EXPECT((*reader)[3] == b1); EXPECT((*reader)[4] == d1); EXPECT((*reader)[5] == i2); EXPECT((*reader)[6] == static_cast(f2)); EXPECT((*reader)[7] == *reinterpret_cast(s2)); EXPECT((*reader)[8] == b2); EXPECT((*reader)[9] == d2); ++reader; EXPECT(reader->data()[reader->dataOffset(0)] == i3); EXPECT(reader->data()[reader->dataOffset(1)] == static_cast(f3)); EXPECT(::strncmp(reinterpret_cast(&reader->data()[reader->dataOffset(2)]), s3, 24) == 0); EXPECT(reader->data()[reader->dataOffset(3)] == b3); EXPECT(reader->data()[reader->dataOffset(4)] == d3); EXPECT(reader->data()[reader->dataOffset(5)] == i4); EXPECT(reader->data()[reader->dataOffset(6)] == static_cast(f4)); EXPECT(reader->data()[reader->dataOffset(7)] == *reinterpret_cast(s4)); EXPECT(reader->data()[reader->dataOffset(8)] == b4); EXPECT(reader->data()[reader->dataOffset(9)] == d4); ++reader; EXPECT(reader->data(0) == i5); EXPECT(reader->data(1) == static_cast(f5)); EXPECT(::strncmp(reinterpret_cast(&reader->data(2)), s5, 24) == 0); EXPECT(reader->data(3) == b5); EXPECT(reader->data(4) == d5); EXPECT(reader->data(5) == i6); EXPECT(reader->data(6) == static_cast(f6)); EXPECT(reader->data(7) == *reinterpret_cast(s6)); EXPECT(reader->data(8) == b6); EXPECT(reader->data(9) == d6); } } #if 0 // This test needs to be reassesed -- AssertionFailed is not likely the correct Exception to be thrown CASE("We cannot encode short_real with both possible internal missing values") { const float f1 = std::numeric_limits::min(); const float f2 = float_lowest; eckit::Buffer buf(4096); EXPECT_THROWS_AS({ eckit::MemoryHandle dh(buf); odc::Writer<> oda(dh); odc::Writer<>::iterator writer = oda.begin(); // Set up the columns writer->setNumberOfColumns(1); writer->setColumn(0, "real", odc::api::REAL); writer->writeHeader(); // Append 3 rows of data (in two different ways) (*writer)[0] = f1; ++writer; (*writer)[0] = f2; ++writer; }, eckit::AssertionFailed); } #endif CASE("We ASSERT on cases where we try and use an incompletely configured writer") { eckit::Buffer buf(4096); // Illegal to flush an incompletely finished writer eckit::MemoryHandle dh(buf); odc::Writer<> oda(dh); odc::Writer<>::iterator writer = oda.begin(); // Set up the columns writer->setNumberOfColumns(2); writer->setColumn(0, "real", odc::api::REAL); // Cannot writeHeader until all the columns are initialised EXPECT_THROWS_AS(writer->writeHeader(), eckit::AssertionFailed); // Cannot write to an uninitialised writer EXPECT_THROWS_AS((*writer)[0] = 1234.56, eckit::AssertionFailed); // Cannot increment an incomplete row EXPECT_THROWS_AS(++writer, eckit::AssertionFailed); } CASE("Data is automatically written after a configurable number of rows") { EXPECT(true); } CASE("Pathological data for integral codecs is correctly encoded") { // The reduced-size integral codecs have special internal values for missingValue. // If we try and encode an integer that happens to collide with that value whilst // missing values are enabled, the codec in use needs to be uprated to the next // biggest size, so things are encoded correctly. eckit::Buffer buf(4096); for (int i = 0; i < 4; i++) { bool withMissing = (i % 2 == 1); bool bits16 = (i > 1); // eckit::Log::info() << "iteration: " << i // << (withMissing ? "T":"F") // << (bits16 ? "T":"F") << std::endl; int32_t i1 = 12345; int32_t i2 = 12345 + (bits16 ? 0xffff : 0xff); { eckit::MemoryHandle dh(buf); odc::Writer<> oda(dh); odc::Writer<>::iterator writer = oda.begin(); // Set up the columns writer->setNumberOfColumns(1); writer->setColumn(0, "int", odc::api::INTEGER); writer->writeHeader(); // Append 3 rows of data (in two different ways) (*writer)[0] = i1; ++writer; (*writer)[0] = i2; ++writer; if (withMissing) { (*writer)[0] = odc::MDI::integerMDI(); ++writer; } // We have not supplied missing values EXPECT(writer->columns()[0]->missingValue() == odc::MDI::integerMDI()); EXPECT(writer->columns()[0]->hasMissing() == withMissing); } // Read everything back { eckit::MemoryHandle dh(buf); dh.openForRead(); odc::Reader oda(dh); odc::Reader::iterator reader = oda.begin(); // We have not supplied missing values EXPECT(reader->columns().size() == size_t(1)); EXPECT(reader->columns()[0]->missingValue() == odc::MDI::integerMDI()); EXPECT(reader->columns()[0]->hasMissing() == withMissing); // Promotion to int32 occurs to if (withMissing && bits16) { EXPECT(reader->columns()[0]->coder().name() == "int32"); } else if (withMissing) { EXPECT(reader->columns()[0]->coder().name() == "int16_missing"); } else if (bits16) { EXPECT(reader->columns()[0]->coder().name() == "int16"); } else { EXPECT(reader->columns()[0]->coder().name() == "int8"); } EXPECT((*reader)[0] == i1); ++reader; EXPECT((*reader)[0] == i2); if (withMissing) { ++reader; EXPECT((*reader)[0] == odc::MDI::integerMDI()); } } } } // ------------------------------------------------------------------------------------------------------ int main(int argc, char* argv[]) { return run_tests(argc, argv); } odc-1.6.3/tests/core/test_select_iterator.cc0000664000175000017500000001700115146027420021302 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "odc_ecbuild_config.h" #include "eckit/config/Resource.h" #include "eckit/eckit_config.h" #include "eckit/filesystem/PathName.h" #include "eckit/system/SystemInfo.h" #include "eckit/testing/Test.h" #include "odc/Reader.h" #include "odc/Select.h" #include "TemporaryFiles.h" #include using namespace eckit::testing; // ------------------------------------------------------------------------------------------------------ // TODO: Test with WHERE clause in SELECT? CASE("Test reading with iterators") { SETUP("An odb file containing some pre-prepared data") { // Write (and clean up) a temporary ODB file class TemporaryODB : public TemporaryFile { public: TemporaryODB() { odc::Writer<> oda(path()); odc::Writer<>::iterator writer = oda.begin(); writer->setNumberOfColumns(3); writer->setColumn(0, "ifoo", odc::api::INTEGER); writer->setColumn(1, "nbar", odc::api::REAL); writer->setColumn(2, "string", odc::api::STRING); writer->writeHeader(); for (size_t i = 1; i <= 10; i++) { writer->data()[0] = i; // col 0 writer->data()[1] = i; // col 1 ++writer; } } }; TemporaryODB tmpODB; #if __cplusplus > 199711L // C++11 SECTION("Test select iterator for each") { odc::Select oda("select * from \"" + tmpODB.path() + "\";", tmpODB.path()); long count = 0; std::for_each(oda.begin(), oda.end(), [&](odc::Select::row& row) { ++count; int64_t i = row[0]; double n = row[1]; EXPECT(i == count); EXPECT(int(n) == count); }); EXPECT(count == 10); } SECTION("Test read iterator for_each") { odc::Reader oda(tmpODB.path()); long count = 0; std::for_each(oda.begin(), oda.end(), [&](odc::Reader::row& row) { ++count; int64_t i = row[0]; double n = row[1]; EXPECT(i == count); EXPECT(int(n) == count); }); EXPECT(count == 10); } #endif SECTION("Test select data in explicit loop") { odc::Select oda("select * from \"" + tmpODB.path() + "\";", tmpODB.path()); int count = 0; for (odc::Select::iterator it = oda.begin(); it != oda.end(); ++it) { ++count; EXPECT((*it)[0] == count); EXPECT((*it)[1] == count); } EXPECT(count == 10); } SECTION("Test read data in explicit loop") { odc::Reader oda(tmpODB.path()); int count = 0; for (odc::Reader::iterator it = oda.begin(); it != oda.end(); ++it) { ++count; int i = (*it)[0]; double d = (*it)[1]; EXPECT(i == count); EXPECT(d == count); } EXPECT(count == 10); } } } CASE("Test bugfix 01, quote <>") { unsigned char REF_DATA[] = { 0x0, 0x0, 0x0, 0x60, 0x9b, 0x5e, 0x41, 0x40, // 34.7391 0x0, 0x0, 0x0, 0x20, 0xcc, 0xf, 0x11, 0xc0, // -4.26543 0x0, 0x0, 0x0, 0x60, 0x66, 0x46, 0x6b, 0x40, // 218.2 0x0, 0x0, 0x0, 0x0, 0x77, 0xc8, 0x3b, 0x40, // 27.7831 0x0, 0x0, 0x0, 0x80, 0xed, 0xb, 0xef, 0xbf, // -0.970206 0x0, 0x0, 0x0, 0xc0, 0xcc, 0xcc, 0x6b, 0x40, // 222.4 0x0, 0x0, 0x0, 0x80, 0x6, 0xff, 0x48, 0x40, // 49.9924 0x0, 0x0, 0x0, 0x80, 0x81, 0xec, 0xeb, 0x3f, // 0.87262 0x0, 0x0, 0x0, 0x60, 0x66, 0x26, 0x6b, 0x40, // 217.2 0x0, 0x0, 0x0, 0x0, 0xc7, 0x18, 0x45, 0x40, // 42.1936 0x0, 0x0, 0x0, 0xc0, 0x56, 0x91, 0xe7, 0x3f, // 0.736492 0x0, 0x0, 0x0, 0xc0, 0xcc, 0xec, 0x6a, 0x40, // 215.4 0x0, 0x0, 0x0, 0x0, 0xc7, 0x18, 0x45, 0x40, // 42.1936 0x0, 0x0, 0x0, 0xc0, 0x56, 0x91, 0xe7, 0xbf, // -0.736492 0x0, 0x0, 0x0, 0xc0, 0xcc, 0xec, 0x6a, 0x40, // 215.4 0x0, 0x0, 0x0, 0xa0, 0xa5, 0x7c, 0x45, 0x40, // 42.9738 0x0, 0x0, 0x0, 0x40, 0xc7, 0x2, 0xf8, 0xbf, // -1.50068 0x0, 0x0, 0x0, 0x60, 0x66, 0xe6, 0x6a, 0x40, // 215.2 0x0, 0x0, 0x0, 0x60, 0xf9, 0xcb, 0x45, 0x40, // 43.5935 0x0, 0x0, 0x0, 0x80, 0x9, 0x63, 0x8, 0xc0, // -3.04836 0x0, 0x0, 0x0, 0x60, 0x66, 0xe6, 0x6a, 0x40, // 215.2 0x0, 0x0, 0x0, 0x40, 0xa3, 0x22, 0x36, 0x40 // 22.1353 }; if (eckit::system::SystemInfo::isBigEndian()) { // Swap the data on this big-endian box. for (int i = 0; i < sizeof(REF_DATA) / 8; i++) { for (int j = 0; j < 4; j++) std::swap(REF_DATA[i * 8 + j], REF_DATA[(i + 1) * 8 - 1 - j]); // cout << *(reinterpret_cast(&REF_DATA[i * 8])) << std::endl; } } const double* OBSVALUE = reinterpret_cast(REF_DATA); eckit::Resource testDataPath("$TEST_DATA_DIRECTORY", ".."); std::stringstream ss_select; ss_select << "select obsvalue from \"" << (testDataPath / "2000010106-reduced.odb") << "\";"; odc::Select oda(ss_select.str()); // Only consider the first section of the file... size_t count = 0; for (odc::Select::iterator it = oda.begin(); it != oda.end() && count < sizeof(REF_DATA) / sizeof(double); ++it, ++count) { // eckit::Log::info() << "testBug01: it[" << count << "]=" << (*it)[0] << ", should be " << // OBSVALUE[count] << std::endl; EXPECT((*it)[0] == OBSVALUE[count]); } } CASE("Test bugfix 02, quote <>") { std::vector VALUE; VALUE.push_back(1); VALUE.push_back(2); VALUE.push_back(3); class TemporaryODB : public TemporaryFile { public: TemporaryODB(const std::vector& values) { odc::Writer<> oda(path()); odc::Writer<>::iterator writer = oda.begin(); writer->setNumberOfColumns(1); writer->setColumn(0, "value", odc::api::INTEGER); writer->writeHeader(); for (size_t i = 0; i < values.size(); ++i) { (*writer)[0] = values[i]; ++writer; } } }; // Write (and clean up) a temporary ODB file TemporaryODB tmpODB(VALUE); odc::Select oda("select * from \"" + tmpODB.path() + "\";", tmpODB.path()); size_t count = 0; for (odc::Select::iterator it = oda.begin(); it != oda.end(); ++it, ++count) { // eckit::Log::info() << "testBug02: it[" << count << "]=" << (*it)[0] << ", should be " << // VALUE[count] << std::endl; EXPECT((*it)[0] == VALUE[count]); } EXPECT(count == 3); } // ------------------------------------------------------------------------------------------------------ int main(int argc, char* argv[]) { return run_tests(argc, argv); } odc-1.6.3/tests/core/test_metadata.cc0000664000175000017500000000432015146027420017672 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/testing/Test.h" #include "odc/Reader.h" #include "odc/Writer.h" #include "odc/core/MetaData.h" using namespace eckit::testing; using eckit::Log; // ------------------------------------------------------------------------------------------------------ CASE("Copy metadata preserves data sizes") { odc::core::MetaData md; md.addColumn("col1", "STRING"); md.back()->dataSizeDoubles(2); md.addColumn("col2", "STRING"); EXPECT(md[0]->dataSizeDoubles() == 2); odc::core::MetaData md2; md2 = md; EXPECT(md.size() == 2); EXPECT(md[0]->dataSizeDoubles() == 2); EXPECT(md[1]->dataSizeDoubles() == 1); } CASE("Copy metadata in Writer") { odc::core::MetaData md; md.addColumn("col1", "STRING"); md.back()->dataSizeDoubles(2); md.addColumn("col2", "STRING"); EXPECT(md[0]->dataSizeDoubles() == 2); odc::Writer<> w("example.odb"); odc::Writer<>::iterator it = w.begin(); it->columns(md); eckit::Log::info() << "Found: " << it->columns().at(0)->dataSizeDoubles() << std::endl; EXPECT(it->columns().at(0)->dataSizeDoubles() == 2); EXPECT(it->columns().at(1)->dataSizeDoubles() == 1); } CASE("Missing/ambiguous column names") { odc::Reader reader("../2000010106-reduced.odb"); auto it = reader.begin(); EXPECT(it != reader.end()); const odc::core::MetaData& md = it->columns(); EXPECT(md.columnIndex("event1@hdr") == 11); EXPECT(md.columnIndex("event1@body") == 35); EXPECT_THROWS_AS(md.columnIndex("event1"), odc::core::AmbiguousColumnException); EXPECT_THROWS_AS(md.columnIndex("bogus"), odc::core::ColumnNotFoundException); } // ------------------------------------------------------------------------------------------------------ int main(int argc, char* argv[]) { return run_tests(argc, argv); } odc-1.6.3/tests/core/test_decode_odb.cc0000664000175000017500000002631315146027420020167 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/config/Resource.h" #include "eckit/testing/Test.h" #include "eckit/types/FloatCompare.h" #include "eckit/value/Value.h" #include "odc/Reader.h" // Some of the math.h/cmath functions are not clean when switching to C++11 #if __cplusplus <= 199711L #include #else #include #define fabs(x) std::fabs((x)) #define modf(x, y) std::modf((x), (y)) #endif using namespace eckit::testing; using eckit::types::is_approximately_equal; // ------------------------------------------------------------------------------------------------------ struct CellData { template CellData(const std::string n, const T& v, bool m = false) : name(n), value(v), missing(m) {} std::string name; eckit::Value value; bool missing; }; // Checker picks a couple of rows in 2000010106-reduced.odb to test. // Can be extended to check anything depending on initialisation // TODO: Build a random ODB, and test it. class ODBChecker { public: // types typedef std::map > RowStore; public: // methods ODBChecker() { std::vector rowData; rowData.push_back(CellData("expver@desc", std::string("0018 "))); rowData.push_back(CellData("andate@desc", 20000101)); rowData.push_back(CellData("antime@desc", 60000)); rowData.push_back(CellData("seqno@hdr", 66969)); rowData.push_back(CellData("obstype@hdr", 2)); rowData.push_back(CellData("obschar@hdr", 67132561)); rowData.push_back(CellData("subtype@hdr", 145)); rowData.push_back(CellData("date@hdr", 20000101)); rowData.push_back(CellData("time@hdr", 32200)); rowData.push_back(CellData("rdbflag@hdr", 0)); rowData.push_back(CellData("status@hdr", 4)); rowData.push_back(CellData("event1@hdr", 512)); rowData.push_back(CellData("blacklist@hdr", 0)); rowData.push_back(CellData("sortbox@hdr", 2147483647, true)); rowData.push_back(CellData("sitedep@hdr", 0)); rowData.push_back(CellData("statid@hdr", std::string("MR413SRA"))); rowData.push_back(CellData("ident@hdr", 0)); rowData.push_back(CellData("lat@hdr", 0.831300)); rowData.push_back(CellData("lon@hdr", -2.057394)); rowData.push_back(CellData("stalt@hdr", -2147483647.000000, true)); rowData.push_back(CellData("modoro@hdr", 717.562744)); rowData.push_back(CellData("trlat@hdr", 0.831300)); rowData.push_back(CellData("trlon@hdr", 4.225791)); rowData.push_back(CellData("instspec@hdr", 3095)); rowData.push_back(CellData("event2@hdr", 0)); rowData.push_back(CellData("anemoht@hdr", 0.000000)); rowData.push_back(CellData("baroht@hdr", 0.000000)); rowData.push_back(CellData("sensor@hdr", 0)); rowData.push_back(CellData("numlev@hdr", 1)); rowData.push_back(CellData("varno_presence@hdr", 12)); rowData.push_back(CellData("varno@body", 3)); rowData.push_back(CellData("vertco_type@body", 1)); rowData.push_back(CellData("rdbflag@body", 0)); rowData.push_back(CellData("anflag@body", 0)); rowData.push_back(CellData("status@body", 4)); rowData.push_back(CellData("event1@body", 33554432)); rowData.push_back(CellData("blacklist@body", 0)); rowData.push_back(CellData("entryno@body", 1)); rowData.push_back(CellData("press@body", 23840.000000)); rowData.push_back(CellData("press_rl@body", -2147483647.000000, true)); rowData.push_back(CellData("obsvalue@body", 34.739117)); rowData.push_back(CellData("aux1@body", 35.000000)); rowData.push_back(CellData("event2@body", 0)); rowData.push_back(CellData("ppcode@body", 0)); rowData.push_back(CellData("level@body", 0)); rowData.push_back(CellData("biascorr@body", 0.000000)); rowData.push_back(CellData("final_obs_error@errstat", 2.920646)); rowData.push_back(CellData("obs_error@errstat", 2.920646)); rowData.push_back(CellData("repres_error@errstat", -2147483647.000000, true)); rowData.push_back(CellData("pers_error@errstat", -2147483647.000000, true)); rowData.push_back(CellData("fg_error@errstat", 3.002484)); data_[0] = rowData; rowData.clear(); rowData.push_back(CellData("expver@desc", std::string("0018 "))); rowData.push_back(CellData("andate@desc", 20000101)); rowData.push_back(CellData("antime@desc", 60000)); rowData.push_back(CellData("seqno@hdr", 6020684)); rowData.push_back(CellData("obstype@hdr", 7)); rowData.push_back(CellData("obschar@hdr", 135265490)); rowData.push_back(CellData("subtype@hdr", 54)); rowData.push_back(CellData("date@hdr", 20000101)); rowData.push_back(CellData("time@hdr", 54533)); rowData.push_back(CellData("rdbflag@hdr", 0)); rowData.push_back(CellData("status@hdr", 44)); rowData.push_back(CellData("event1@hdr", 2)); rowData.push_back(CellData("blacklist@hdr", 16777223)); rowData.push_back(CellData("sortbox@hdr", 2147483647, true)); rowData.push_back(CellData("sitedep@hdr", 2147483647, true)); rowData.push_back(CellData("statid@hdr", std::string(" 203"))); rowData.push_back(CellData("ident@hdr", 203)); rowData.push_back(CellData("lat@hdr", -0.933479)); rowData.push_back(CellData("lon@hdr", -2.107894)); rowData.push_back(CellData("stalt@hdr", 870000.000000)); rowData.push_back(CellData("modoro@hdr", 0.963609)); rowData.push_back(CellData("trlat@hdr", -0.933479)); rowData.push_back(CellData("trlon@hdr", 4.175291)); rowData.push_back(CellData("instspec@hdr", 9215)); rowData.push_back(CellData("event2@hdr", 0)); rowData.push_back(CellData("anemoht@hdr", 0.000000)); rowData.push_back(CellData("baroht@hdr", 0.000000)); rowData.push_back(CellData("sensor@hdr", 0)); rowData.push_back(CellData("numlev@hdr", 0)); rowData.push_back(CellData("varno_presence@hdr", 1032)); rowData.push_back(CellData("varno@body", 119)); rowData.push_back(CellData("vertco_type@body", 3)); rowData.push_back(CellData("rdbflag@body", 0)); rowData.push_back(CellData("anflag@body", 48)); rowData.push_back(CellData("status@body", 44)); rowData.push_back(CellData("event1@body", 512)); rowData.push_back(CellData("blacklist@body", 0)); rowData.push_back(CellData("entryno@body", 6)); rowData.push_back(CellData("press@body", 6.000000)); rowData.push_back(CellData("press_rl@body", -2147483647.000000, true)); rowData.push_back(CellData("obsvalue@body", 243.490005)); rowData.push_back(CellData("aux1@body", -2147483647.000000, true)); rowData.push_back(CellData("event2@body", 0)); rowData.push_back(CellData("ppcode@body", 0)); rowData.push_back(CellData("level@body", 0)); rowData.push_back(CellData("biascorr@body", 0.493023)); rowData.push_back(CellData("final_obs_error@errstat", 0.600000)); rowData.push_back(CellData("obs_error@errstat", 0.600000)); rowData.push_back(CellData("repres_error@errstat", -2147483647.000000, true)); rowData.push_back(CellData("pers_error@errstat", 0.958788)); rowData.push_back(CellData("fg_error@errstat", 0.269232)); data_[371426] = rowData; } ~ODBChecker() {} void checkRow(size_t num, const odc::Reader::iterator& row) { if (data_.find(num) != data_.end()) { std::vector& reference(data_[num]); EXPECT(reference.size() == row->columns().size()); for (size_t i = 0; i < reference.size(); i++) { EXPECT(row->columns()[i]->name() == reference[i].name); // Data is always returned in an array of (8-byte) doubles. Actual data is of types // of size <= 8 byte. Needs some casting to access. if (reference[i].value.isString()) { std::string s(reinterpret_cast(&row->data()[i]), 8); EXPECT(reference[i].value == s); } else if (reference[i].value.isNumber()) { double intpart; EXPECT(modf(row->data()[i], &intpart) == 0.0); EXPECT(static_cast(reference[i].value) == static_cast(intpart)); } else if (reference[i].value.isDouble()) { EXPECT(is_approximately_equal(static_cast(reference[i].value), row->data()[i], fabs(1.0e-5 * static_cast(reference[i].value)))); } else { // We don't want unknown data types slipping in here!!! EXPECT(false); } // Check that the missing values are reported correctly EXPECT(reference[i].missing == (row->data()[i] == row->columns()[i]->missingValue())); } } } size_t highestRow() const { size_t biggestRow = 0; for (RowStore::const_iterator it = data_.begin(); it != data_.end(); ++it) { biggestRow = (it->first > biggestRow) ? it->first : biggestRow; } return biggestRow; } private: // members RowStore data_; }; // ------------------------------------------------------------------------------------------------------ eckit::Resource testDataPath("$TEST_DATA_DIRECTORY", ".."); CASE("The correct number of rows are decoded") { eckit::PathName filename = testDataPath / "2000010106-reduced.odb"; odc::Reader in(filename); odc::Reader::iterator it = in.begin(); // Each table in the ODB will contain a maximum of 10000 rows! EXPECT(it->columns().rowsNumber() == 10000); size_t count = 0; for (; it != in.end(); ++it) { if (count < 3320000) { EXPECT(it->columns().rowsNumber() == 10000); } else { EXPECT(it->columns().rowsNumber() == 1753); } count++; } // All of the lines correctly decoded EXPECT(count == 50000); } CASE("The correct data is present in a selection of random rows") { eckit::PathName filename = testDataPath / "2000010106-reduced.odb"; odc::Reader in(filename); odc::Reader::iterator it = in.begin(); ODBChecker checker; size_t biggestRow = checker.highestRow(); size_t count = 0; for (; it != in.end() && count <= biggestRow; ++it) { checker.checkRow(count, it); count++; } } // TODO: Test missing values // TODO: Test reading a randomly created ODB. // ------------------------------------------------------------------------------------------------------ int main(int argc, char* argv[]) { return run_tests(argc, argv); } odc-1.6.3/tests/f_api/0000775000175000017500000000000015146027420014673 5ustar alastairalastairodc-1.6.3/tests/f_api/odc_ls.f900000664000175000017500000001674515146027420016473 0ustar alastairalastair! To build this program, please make sure to reference linked libraries: ! ! gfortran -lodccore -lfodc -o odc-fortran-ls odc_ls.f90 program odc_ls use, intrinsic :: iso_c_binding, only: c_null_char use, intrinsic :: iso_fortran_env, only: error_unit,output_unit use odc implicit none character(255) :: path type(odc_reader) :: reader type(odc_frame) :: frame type(odc_decoder) :: decoder integer(8) :: nrows integer :: err, ncols if (command_argument_count() /= 1) then call usage() stop 1 end if call get_command_argument(1, path) call check_call(odc_initialise_api(), "initialising api") call check_call(reader%open_path(trim(path)), "opening path") call check_call(frame%initialise(reader), "initialising frame") err = frame%next() do while (err == ODC_SUCCESS) call check_call(frame%column_count(ncols), "getting column count") call write_header(frame, ncols) call check_call(decoder%initialise(), "initialising decoder") call check_call(decoder%defaults_from_frame(frame), "setting decoder structure") call check_call(decoder%decode(frame, nrows), "decoding data") call write_data(decoder, frame, nrows, ncols) call check_call(decoder%free(), "cleaning up decoder") err = frame%next() end do if (err /= ODC_ITERATION_COMPLETE) call check_call(err, "get next frame") call check_call(reader%close(), "closing reader") contains subroutine check_call(err, desc) integer, intent(in) :: err character(*), intent(in) :: desc if (err /= ODC_SUCCESS) then write(error_unit, *) '**** An error occurred in ODC library' write(error_unit, *) 'Description: ', desc write(error_unit, *) 'Error: ', odc_error_string(err) stop 1 end if end subroutine subroutine usage() write(output_unit, *) 'Usage:' write(output_unit, *) ' odc-fortran-ls ' end subroutine subroutine write_header(frame, ncols) type(odc_frame), intent(in) :: frame integer, intent(in) :: ncols character(:), allocatable :: name_str integer :: col do col = 1, ncols call write_integer(output_unit, col) call check_call(frame%column_attributes(col, name=name_str), "getting column name") write(output_unit, '(3a)', advance='no') '. ', name_str, char(9) end do write(output_unit,*) end subroutine subroutine write_data(decoder, frame, nrows, ncols) type(odc_decoder), intent(inout) :: decoder type(odc_frame), intent(in) :: frame integer(8), intent(in) :: nrows integer, intent(in) :: ncols integer(8) :: row real(8), pointer :: array_data(:,:) integer :: ncols_decoder, ncols_frame, col, current_index, bitfield_count integer(8) :: missing_integer real(8) :: missing_double integer, dimension(ncols) :: types, sizes, bf_sizes, indexes integer, target :: bf, bf_size call check_call(decoder%data(array_data), "getting access to data") call check_call(decoder%column_count(ncols_decoder), "getting decoder column count") call check_call(frame%column_count(ncols_frame), "getting frame column count") if (ncols_decoder /= ncols_frame .or. ncols_decoder /= ncols) then write(error_unit, *) 'Something went wrong in the decode target initialisation' stop 1 end if current_index = 1 do col = 1, ncols call check_call(frame%column_attributes(col, type=types(col), bitfield_count=bitfield_count), "getting column type") call check_call(decoder%column_data_array(col, element_size_doubles=sizes(col)), "getting element size") indexes(col) = current_index current_index = current_index + sizes(col) if (types(col) == ODC_BITFIELD) then bf_sizes(col) = 0 do bf = 1, bitfield_count call check_call(frame%bitfield_attributes(col, bf, size=bf_size), "getting bitfield size") bf_sizes(col) = bf_sizes(col) + bf_size end do end if end do call check_call(odc_missing_integer(missing_integer), "getting missing integer") call check_call(odc_missing_double(missing_double), "getting missing double") do row = 1, nrows do col = 1, ncols select case(types(col)) case (ODC_INTEGER) if (int(array_data(row, indexes(col))) == missing_integer) then write(output_unit, '(a)', advance='no') '.' else call write_integer(output_unit, int(array_data(row, indexes(col)))) end if case (ODC_BITFIELD) if (int(array_data(row, indexes(col))) == 0) then write(output_unit, '(a)', advance='no') '.' else call write_bitfield(output_unit, int(array_data(row, indexes(col))), bf_sizes(col)) end if case (ODC_REAL, ODC_DOUBLE) if (array_data(row, indexes(col)) == missing_double) then write(output_unit, '(a)', advance='no') '.' else call write_double(output_unit, array_data(row, indexes(col))) end if case (ODC_STRING) call write_string(output_unit, array_data(row, indexes(col):(indexes(col)+sizes(col)-1))) case default write(output_unit, '(a)', advance='no') '' end select write(output_unit, '(a)', advance='no') char(9) end do write(output_unit, *) end do end subroutine subroutine write_integer(iunit, i) integer, intent(in) :: iunit, i character(32) :: val write(val, *) i write(iunit, '(a)', advance='no') trim(adjustl(val)) end subroutine subroutine write_bitfield(iunit, i, size) integer, intent(in) :: iunit, i, size character(32) :: padding_format, val write(padding_format, '(a,i0,a)') '(b32.', size, ')' write(val, padding_format) i write(iunit, '(a)', advance='no') trim(adjustl(val)) end subroutine subroutine write_double(iunit, r) integer, intent(in) :: iunit real(8), intent(in) :: r character(32) :: val write(val, '(f8.4)') r write(iunit, '(a)', advance='no') trim(adjustl(val)) end subroutine subroutine write_string(iunit, double_string) integer, intent(in) :: iunit real(8), intent(in), dimension(:) :: double_string if (all(transfer(double_string, 1_8, size(double_string)) == 0)) then write(iunit, '(a)', advance='no') '.' else write(iunit, '(a)', advance='no') strip_nulls(double_string) end if end subroutine function strip_nulls(dstr) result(fstr) real(8), intent(in) :: dstr(:) character(8 * size(dstr)) :: tmpstr character(:), allocatable :: fstr integer :: i tmpstr = transfer(dstr, tmpstr) do i = 1, len(tmpstr) if (tmpstr(i:i) == c_null_char) then fstr = tmpstr(1:i-1) return end if end do fstr = tmpstr end function end program odc-1.6.3/tests/f_api/general.f900000664000175000017500000002156515146027420016641 0ustar alastairalastair module fapi_general_tests use odc use odc_config use, intrinsic :: iso_fortran_env implicit none integer :: test_error_handler_calls = 0 integer(8) :: test_error_handler_last_context integer :: test_error_handler_last_error contains function test_odc_version() result(success) ! Test that we obtain the expected version number logical :: success character(:), allocatable :: version_str success = .true. if (odc_version(version_str) /= ODC_SUCCESS) then write(error_unit, *) 'getting version string failed' success = .false. end if if (version_str /= odc_version_str) then write(error_unit, *) "Unexpected version: ", version_str write(error_unit, *) "Expected: ", odc_version_str success = .false. endif end function function test_git_sha1() result(success) ! Test that we obtain the expected version number logical :: success character(:), allocatable :: sha1 success = .true. if (odc_vcs_version(sha1) /= ODC_SUCCESS) then write(error_unit, *) 'getting git sha1 string failed' success = .false. end if if (sha1 /= odc_git_sha1_str .and. sha1 /= "not available") then write(error_unit, *) "Unexpected git sha1: ", sha1 write(error_unit, *) "Expected: ", odc_git_sha1_str success = .false. endif end function function test_type_names() result(success) logical :: success integer :: test_types(6) = [ODC_IGNORE, ODC_INTEGER, ODC_REAL, ODC_STRING, ODC_BITFIELD, ODC_DOUBLE] character(8) :: names(6) = [character(8) :: "ignore", "integer", "real", "string", "bitfield", "double"] character(:), allocatable :: name integer :: ntypes, n success = .true. if (odc_column_type_count(ntypes) /= ODC_SUCCESS) then write(error_unit, *) 'Failed to get type count' success = .false. end if if (ntypes /= 6) then write(error_unit, *) 'Unexpected number of types found' success = .false. endif do n = 1, ntypes if (odc_column_type_name(test_types(n), name) /= ODC_SUCCESS) then write(error_unit, *) 'Failed to get type name, type=', test_types(n) success = .false. end if if (name /= trim(names(n))) then write(error_unit, *) 'Unexpected type name "', name, '" for type ', n success = .false. end if end do end function function test_error_handling() result(success) logical :: success type(odc_reader) :: reader integer :: j, err success = .true. ! Check that an error is correctly reported do j = 1, 2 err = reader%open_path("invalid-path") if (err == ODC_SUCCESS) then write(error_unit, *) 'open_path succeeded unexpectedly with "invalid-path"' success = .false. end if if (odc_error_string(err) /= "Cannot open invalid-path (No such file or directory)") then write(error_unit, *) 'unexpected error message: ', odc_error_string(err) success = .false. endif end do end function function test_odc_integer_behaviour() result(success) ! Test that we can set the integer behaviour as both double and long logical :: success success = .true. if (odc_integer_behaviour(ODC_INTEGERS_AS_DOUBLES) /= ODC_SUCCESS) then write(error_unit, *) 'setting integer behaviour to doubles failed' success = .false. end if if (odc_integer_behaviour(ODC_INTEGERS_AS_LONGS) /= ODC_SUCCESS) then write(error_unit, *) 'setting integer behaviour to longs failed' success = .false. endif end function function test_odc_set_failure_handler() result(success) ! Test that we can set failure handler and that it is being called on error with appropriate information logical :: success integer(8) :: original_context = 123456 integer(8) :: context, err type(odc_reader) :: reader character(:), allocatable :: name success = .true. context = original_context ! Set test error handler and its context if (odc_set_failure_handler(test_error_handler, context) /= ODC_SUCCESS) then write(error_unit, *) 'setting failure handler failed' success = .false. end if ! Trigger an error err = reader%open_path("invalid-path") if (err == ODC_SUCCESS) then write(error_unit, *) 'open_path succeeded unexpectedly with "invalid-path"' success = .false. end if ! Check number of error handler calls if (test_error_handler_calls /= 1) then write(error_unit, *) 'error handler not called expected number of times:', test_error_handler_calls, '/=', 1 success = .false. end if ! Check last received error handler context if (test_error_handler_last_context /= context) then write(error_unit, *) 'error handler context differs:', test_error_handler_last_context, '/=', context success = .false. end if ! Check last received error code if (test_error_handler_last_error /= err) then write(error_unit, *) 'error handler error code differs:', test_error_handler_last_error, '/=', err success = .false. end if ! Change context value context = 654321 ! Trigger another error err = odc_integer_behaviour(0) if (err == ODC_SUCCESS) then write(error_unit, *) 'odc_integer_behaviour succeeded unexpectedly with "0"' success = .false. end if ! Check number of error handler calls if (test_error_handler_calls /= 2) then write(error_unit, *) 'error handler not called expected number of times:', test_error_handler_calls, '/=', 2 success = .false. end if ! Check last received error handler context, it should match the original one if (test_error_handler_last_context /= original_context) then write(error_unit, *) 'error handler context differs:', test_error_handler_last_context, '/=', original_context success = .false. end if ! Check last received error code, it should match the latest one if (test_error_handler_last_error /= err) then write(error_unit, *) 'error handler error code differs:', test_error_handler_last_error, '/=', err success = .false. end if ! Change context value another time context = 0 ! Trigger yet another error err = odc_column_type_name(999999, name) if (err == ODC_SUCCESS) then write(error_unit, *) 'odc_column_type_name succeeded unexpectedly with "999999"' success = .false. end if ! Check number of error handler calls if (test_error_handler_calls /= 3) then write(error_unit, *) 'error handler not called expected number of times:', test_error_handler_calls, '/=', 3 success = .false. end if ! Check last received error handler context, it should match the original one if (test_error_handler_last_context /= original_context) then write(error_unit, *) 'error handler context differs:', test_error_handler_last_context, '/=', original_context success = .false. end if ! Check last received error code, it should match the latest one if (test_error_handler_last_error /= err) then write(error_unit, *) 'error handler error code differs:', test_error_handler_last_error, '/=', err success = .false. end if end function subroutine test_error_handler(context, error) integer(8), intent(in) :: context integer, intent(in) :: error test_error_handler_calls = test_error_handler_calls + 1 test_error_handler_last_context = context test_error_handler_last_error = error end subroutine end module program fapi_general use fapi_general_tests implicit none logical :: success success = .true. if (odc_initialise_api() /= ODC_SUCCESS) then write(error_unit, *) 'Failed to initialise ODC api' success = .false. end if success = success .and. test_odc_version() success = success .and. test_git_sha1() success = success .and. test_type_names() success = success .and. test_error_handling() success = success .and. test_odc_integer_behaviour() success = success .and. test_odc_set_failure_handler() if (.not. success) stop -1 end program odc-1.6.3/tests/f_api/odc_encode_row_major.f900000664000175000017500000001566715146027420021373 0ustar alastairalastair! To build this program, please make sure to reference linked libraries: ! ! gfortran -lodccore -lfodc -o odc-fortran-encode-row-major odc_encode_row_major.f90 program odc_ls use, intrinsic :: iso_c_binding, only: c_null_char use odc implicit none character(255) :: path integer, parameter :: ncols = 9 integer(8), parameter :: nrows = 20 real(8), target :: data(ncols, nrows) type(odc_encoder) :: encoder logical, parameter :: column_major = .false. character(10), parameter :: property_key = 'encoded_by' character(11), parameter :: property_value = 'odc_example' integer, target :: outunit integer(8), target :: bytes_written if (command_argument_count() /= 1) then call usage() stop 1 end if ! Get output path from command argument call get_command_argument(1, path) ! Initialise API and set treatment of integers as longs call check_call(odc_initialise_api(), 'initialising api') call check_call(odc_integer_behaviour(ODC_INTEGERS_AS_LONGS), 'setting integer behaviour to longs') ! Set up the allocated array with scratch data call create_scratch_data(data, nrows) ! Initialise encoder call check_call(encoder%initialise(), 'initialising encoder') ! Define all column names and their types call check_call(encoder%add_column('expver', ODC_STRING), 'adding expver column') call check_call(encoder%add_column('date@hdr', ODC_INTEGER), 'adding date@hdr column') call check_call(encoder%add_column('statid@hdr', ODC_STRING), 'adding statid@hdr column') call check_call(encoder%add_column('wigos@hdr', ODC_STRING), 'adding wigos@hdr column') call check_call(encoder%add_column('obsvalue@body', ODC_REAL), 'adding obsvalue@bod column') call check_call(encoder%add_column('integer_missing', ODC_INTEGER), 'adding integer_missing column') call check_call(encoder%add_column('double_missing', ODC_REAL), 'adding double_missing column') call check_call(encoder%add_column('bitfield_column', ODC_BITFIELD), 'adding bitfield_column') ! Column `wigos@hdr` is a 16-byte string column, hence takes 2 columns in the array => ncols=9 call check_call(encoder%column_set_data_size(4, element_size_doubles=2), 'setting column data size') ! Column `bitfield_column` is an integer with 4 bitfield values in it call check_call(encoder%column_add_bitfield(8, "flag_a", 1), 'adding flag_a bitfield') call check_call(encoder%column_add_bitfield(8, "flag_b", 2), 'adding flag_b bitfield') call check_call(encoder%column_add_bitfield(8, "flag_c", 3), 'adding flag_c bitfield') call check_call(encoder%column_add_bitfield(8, "flag_d", 1), 'adding flag_d bitfield') ! Set input data array from which data will be encoded call check_call(encoder%set_data(data, column_major), 'setting encoder data array') ! Add some key/value metadata to the frame call check_call(encoder%add_property(property_key, property_value), 'adding property') ! Encode ODB-2 into a file open(newunit=outunit, file=path, access='stream', form='unformatted', status='replace') call check_call(encoder%encode(outunit, bytes_written), 'encoding data') close(outunit) ! Deallocate memory used up by the encoder call check_call(encoder%free(), 'cleaning up encoder') write(6, '(a,i0,a,a)') 'Written ', nrows, ' rows to ', trim(path) contains subroutine check_call(err, desc) integer, intent(in) :: err character(*), intent(in) :: desc if (err /= ODC_SUCCESS) then write(7, *) '**** An error occurred in ODC library' write(7, *) 'Description: ', desc write(7, *) 'Error: ', odc_error_string(err) stop 1 end if end subroutine subroutine usage() write(6, *) 'Usage:' write(6, *) ' odc-fortran-encode-row-major ' end subroutine subroutine cycle_ints(output, input) integer(8), intent(in) :: input(:) integer(8), intent(out) :: output(:) integer :: i do i = 1, size(output, 1) output(i) = input(mod(i - 1, size(input, 1)) + 1) end do end subroutine subroutine cycle_reals(output, input) real(8), intent(in) :: input(:) real(8), intent(out) :: output(:) integer :: i do i = 1, size(output, 1) output(i) = input(mod(i - 1, size(input, 1)) + 1) end do end subroutine subroutine create_scratch_data(data, nrows) real(8), intent(out) :: data(:,:) integer(8), intent(in) :: nrows character(5) :: expver_str character(8) :: date_str integer(8) :: date character(7) :: statid_str character(16) :: wigos_str integer(8) :: missing_integer integer(8) :: integer_pool(3) integer(8), target :: missing_integers(nrows) real(8) :: missing_double real(8) :: double_pool(3) real(8), target :: missing_doubles(nrows) integer(8) :: bitfield_pool(3) integer(8), target :: bitfield_values(nrows) integer(8) :: i ! Prepare the current date as an integer call date_and_time(date=date_str) read(date_str, *) date ! Prepare the list of integer values, including the missing value call check_call(odc_missing_integer(missing_integer), 'getting missing integer value') integer_pool = (/ int(1234, 8), int(4321, 8), missing_integer /) call cycle_ints(missing_integers, integer_pool) ! Prepare the list of double values, including the missing value call check_call(odc_missing_double(missing_double), 'getting missing double value') double_pool = (/ real(12.34, 8), real(43.21, 8), missing_double /) call cycle_reals(missing_doubles, double_pool) ! Prepare the list of bitfield values bitfield_pool = (/ int(b'00000001'), int(b'00001011'), int(b'01101011') /) call cycle_ints(bitfield_values, bitfield_pool) ! Fill in the passed data array with scratch values do i = 1, nrows write(expver_str, '(a,a)') 'xxxx', '' // c_null_char ! proper string termination data(1, i) = transfer(expver_str, data(1, i)) ! expver data(2, i) = transfer(date, data(2, i)) ! date@hdr write(statid_str, '(a,i0.2,a)') 'stat', i - 1, '' // c_null_char ! proper string termination data(3, i) = transfer(statid_str, data(3, i)) ! statid@hdr write(wigos_str, '(a,i0.2,a)') '0-12345-0-678', i - 1, '' // c_null_char ! proper string termination data(4:5, i) = transfer(wigos_str, data(4:5, i)) ! wigos@hdr data(6, i) = 12.3456 * real(i - 1) ! obsvalue@body data(7, i) = transfer(missing_integers(i), data(7, i)) ! integer_missing data(8, i) = missing_doubles(i) ! double_missing data(9, i) = transfer(bitfield_values(i), data(9, i)) ! bitfield_column end do end subroutine end program odc-1.6.3/tests/f_api/odc_encode_custom.f900000664000175000017500000002007015146027420020666 0ustar alastairalastair! To build this program, please make sure to reference linked libraries: ! ! gfortran -lodccore -lfodc -o odc-fortran-encode-custom odc_encode_custom.f90 program odc_ls use, intrinsic :: iso_c_binding, only: c_loc,c_null_char use odc implicit none character(255) :: path integer(8), parameter :: nrows = 20 character(8), target :: data1(nrows) integer(8), target :: data2(nrows) character(8), target :: data3(nrows) character(16), target :: data4(nrows) real(8), target :: data5(nrows) integer(8), target :: data6(nrows) real(8), target :: data7(nrows) integer(8), target :: data8(nrows) type(odc_encoder) :: encoder character(10), parameter :: property_key = 'encoded_by' character(11), parameter :: property_value = 'odc_example' integer, target :: outunit integer(8), target :: bytes_written if (command_argument_count() /= 1) then call usage() stop 1 end if ! Get output path from command argument call get_command_argument(1, path) ! Initialise API and set treatment of integers as longs call check_call(odc_initialise_api(), 'initialising api') call check_call(odc_integer_behaviour(ODC_INTEGERS_AS_LONGS), 'setting integer behaviour to longs') ! Set up the allocated array with scratch data call create_scratch_data(data1, data2, data3, data4, data5, data6, data7, nrows) ! Initialise encoder call check_call(encoder%initialise(), 'initialising encoder') ! Set number of rows to allocate in the encoder call check_call(encoder%set_row_count(nrows), 'setting number of rows') ! Define all column names and their types call check_call(encoder%add_column('expver', ODC_STRING), 'adding expver column') call check_call(encoder%add_column('date@hdr', ODC_INTEGER), 'adding date@hdr column') call check_call(encoder%add_column('statid@hdr', ODC_STRING), 'adding statid@hdr column') call check_call(encoder%add_column('wigos@hdr', ODC_STRING), 'adding wigos@hdr column') call check_call(encoder%add_column('obsvalue@body', ODC_REAL), 'adding obsvalue@bod column') call check_call(encoder%add_column('integer_missing', ODC_INTEGER), 'adding integer_missing column') call check_call(encoder%add_column('double_missing', ODC_REAL), 'adding double_missing column') call check_call(encoder%add_column('bitfield_column', ODC_BITFIELD), 'adding bitfield_column') ! Column `wigos@hdr` is a 16-byte string column call check_call(encoder%column_set_data_size(4, element_size_doubles=2), 'setting column data size') ! Column `bitfield_column` is an integer with 4 bitfield values in it call check_call(encoder%column_add_bitfield(8, "flag_a", 1), 'adding flag_a bitfield') call check_call(encoder%column_add_bitfield(8, "flag_b", 2), 'adding flag_b bitfield') call check_call(encoder%column_add_bitfield(8, "flag_c", 3), 'adding flag_c bitfield') call check_call(encoder%column_add_bitfield(8, "flag_d", 1), 'adding flag_d bitfield') ! Set a custom data layout and data array for each column call check_call(encoder%column_set_data_array(1, 8, stride=8, data=c_loc(data1)), 'setting expver array') call check_call(encoder%column_set_data_array(2, 8, stride=8, data=c_loc(data2)), 'setting date array') call check_call(encoder%column_set_data_array(3, 8, stride=8, data=c_loc(data3)), 'setting statid array') call check_call(encoder%column_set_data_array(4, 16, stride=16, data=c_loc(data4)), 'setting wigos array') call check_call(encoder%column_set_data_array(5, 8, stride=8, data=c_loc(data5)), 'setting obsvalue array') call check_call(encoder%column_set_data_array(6, 8, stride=8, data=c_loc(data6)), 'setting integer_missing array') call check_call(encoder%column_set_data_array(7, 8, stride=8, data=c_loc(data7)), 'setting double_missing array') call check_call(encoder%column_set_data_array(8, 8, stride=8, data=c_loc(data8)), 'setting bitfield_column array') ! Add some key/value metadata to the frame call check_call(encoder%add_property(property_key, property_value), 'adding property') ! Encode ODB-2 into a file open(newunit=outunit, file=path, access='stream', form='unformatted', status='replace') call check_call(encoder%encode(outunit, bytes_written), 'encoding data') close(outunit) ! Deallocate memory used up by the encoder call check_call(encoder%free(), 'cleaning up encoder') write(6, '(a,i0,a,a)') 'Written ', nrows, ' rows to ', trim(path) contains subroutine check_call(err, desc) integer, intent(in) :: err character(*), intent(in) :: desc if (err /= ODC_SUCCESS) then write(7, *) '**** An error occurred in ODC library' write(7, *) 'Description: ', desc write(7, *) 'Error: ', odc_error_string(err) stop 1 end if end subroutine subroutine usage() write(6, *) 'Usage:' write(6, *) ' odc-fortran-encode-custom ' end subroutine subroutine cycle_ints(output, input) integer(8), intent(in) :: input(:) integer(8), intent(out) :: output(:) integer :: i do i = 1, size(output, 1) output(i) = input(mod(i - 1, size(input, 1)) + 1) end do end subroutine subroutine cycle_reals(output, input) real(8), intent(in) :: input(:) real(8), intent(out) :: output(:) integer :: i do i = 1, size(output, 1) output(i) = input(mod(i - 1, size(input, 1)) + 1) end do end subroutine subroutine create_scratch_data(data1, data2, data3, data4, data5, data6, data7, nrows) character(8), intent(out) :: data1(:) integer(8), intent(out) :: data2(:) character(8), intent(out):: data3(:) character(16), intent(out) :: data4(:) real(8), intent(out) :: data5(:) integer(8), intent(out) :: data6(:) real(8), intent(out) :: data7(:) integer(8), intent(in) :: nrows character(4) :: expver_str = 'xxxx' character(8) :: date_str integer(8) :: date character(6) :: statid_str character(16) :: wigos_str integer(8) :: missing_integer integer(8) :: integer_pool(3) integer(8), target :: missing_integers(nrows) real(8) :: missing_double real(8) :: double_pool(3) real(8), target :: missing_doubles(nrows) integer(8) :: bitfield_pool(3) integer(8), target :: bitfield_values(nrows) integer(8) :: i ! Prepare the current date as an integer call date_and_time(date=date_str) read(date_str, *) date ! Prepare the list of integer values, including the missing value call check_call(odc_missing_integer(missing_integer), 'getting missing integer value') integer_pool = (/ int(1234, 8), int(4321, 8), missing_integer /) call cycle_ints(missing_integers, integer_pool) ! Prepare the list of double values, including the missing value call check_call(odc_missing_double(missing_double), 'getting missing double value') double_pool = (/ real(12.34, 8), real(43.21, 8), missing_double /) call cycle_reals(missing_doubles, double_pool) ! Prepare the list of bitfield values bitfield_pool = (/ int(b'00000001'), int(b'00001011'), int(b'01101011') /) call cycle_ints(bitfield_values, bitfield_pool) ! Fill in the passed data arrays with scratch values do i = 1, nrows data1(i) = expver_str // c_null_char ! expver data2(i) = date ! date@hdr write(statid_str, '(a,i0.2)') 'stat', i - 1 data3(i) = statid_str // c_null_char ! statid@hdr write(wigos_str, '(a,i0.2,a)') '0-12345-0-678', i - 1, '' // c_null_char data4(i) = wigos_str ! wigos@hdr data5(i) = 12.3456 * real(i - 1) ! obsvalue@body data6(i) = missing_integers(i) ! integer_missing data7(i) = missing_doubles(i) ! double_missing data8(i) = bitfield_values(i) ! bitfield_column end do end subroutine end program odc-1.6.3/tests/f_api/CMakeLists.txt0000664000175000017500000000277415146027420017445 0ustar alastairalastair list( APPEND _api_fodc_tests general read encode ) foreach( _test ${_api_fodc_tests} ) ecbuild_add_test( TARGET odc_fapi_${_test} SOURCES ${_test}.f90 ENVIRONMENT ${test_environment} TEST_DEPENDS odc_get_test_data CONDITION HAVE_FORTRAN LINKER_LANGUAGE Fortran LIBS fodc ) endforeach() list( APPEND _api_fodc_example_sources odc_header odc_encode_custom odc_encode_row_major odc_ls ) list( APPEND _api_fodc_example_targets odc-fortran-header odc-fortran-encode-custom odc-fortran-encode-row-major odc-fortran-ls ) list( LENGTH _api_fodc_example_sources _count ) math( EXPR _count "${_count}-1" ) foreach( _i RANGE ${_count} ) list( GET _api_fodc_example_sources ${_i} _sources ) list( GET _api_fodc_example_targets ${_i} _target ) ecbuild_add_executable( TARGET ${_target} SOURCES ${_sources}.f90 LIBS fodc CONDITION HAVE_FORTRAN LINKER_LANGUAGE Fortran NOINSTALL ) endforeach() list( APPEND _fapi_odc_tests_scripts usage_examples.sh ) foreach( _script ${_fapi_odc_tests_scripts} ) ecbuild_add_test( TARGET odc_fapi_${_script} TYPE script COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/${_script} ENVIRONMENT PATH=${CMAKE_BINARY_DIR}/bin:$ENV{PATH} CONDITION HAVE_FORTRAN ) endforeach() odc-1.6.3/tests/f_api/encode.f900000664000175000017500000004613415146027420016460 0ustar alastairalastair module fapi_encode_tests use odc use odc_config use, intrinsic :: iso_fortran_env implicit none integer(8), parameter :: col1_data(7) = [1, 2, 3, 4, 5, 6, 7] integer(8), parameter :: col2_data(7) = [0, 0, 0, 0, 0, 0, 0] integer(8), parameter :: col3_data(7) = [73, 73, 73, 73, 73, 73, 73] real(8), parameter :: col4_data(7) = [1.432, 1.432, 1.432, 1.432, 1.432, 1.432, 1.432] integer(8), parameter :: col5_data(7) = [-17, -7, -7, 999999, 1, 4, 4] character(16), parameter :: col6_data(7) = [character(16) :: "aoeu", "aoeu", "abcdefghijkl", "None", "boo", "squiggle", "a"] character(8), parameter :: col7_data(7) = [character(8) :: "abcd", "abcd", "abcd", "abcd", "abcd", "abcd", "abcd"] real(8), parameter :: col8_data(7) = [2.345, 2.345, 2.345, 2.345, 2.345, 2.345, 2.345] real(8), parameter :: col9_data(7) = [999.99, 888.88, 777.77, 666.66, 999999.0, 444.44, 333.33] real(8), parameter :: col10_data(7) = [999.99, 888.88, 777.77, 666.66, 999999.0, 444.44, 333.33] integer(8), parameter :: col11_data(7) = [1, 999999, 3, 4, 5, 999999, 8] integer(8), parameter :: col12_data(7) = [-512, 999999, 3, 7623, -22000, 999999, 7] integer(8), parameter :: col13_data(7) = [-1234567, 8765432, 999999, 22, 2222222, -81222323, 999999] integer(8), parameter :: col14_data(7) = [999999, 999999, 999999, 999999, 999999, 999999, 999999] character(8), parameter :: property_keys(2) = [character(8) :: "foo", "baz"] character(8), parameter :: property_values(2) = [character(8) :: "bar", "qux"] contains ! TODO: Test missing doubles ! TODO: Test missing REAL ! TODO: Encoding NaN? subroutine check_call(err, msg, success) integer, intent(in) :: err character(*), intent(in) :: msg logical, intent(inout) :: success if (err /= ODC_SUCCESS) then write(error_unit, *) 'Failed API call: ', msg write(error_unit, *) 'Error: ', odc_error_string(err) success = .false. end if end subroutine subroutine check_decoded_column_major(data, success) real(8), intent(in) :: data(:, :) integer :: row character(16) :: str16 character(8) :: str8 logical, intent(inout) :: success if (size(data, 1) /= 7 .or. size(data, 2) /= 15) then write(error_unit, *) 'did not get data shape [7, 15]' success = .false. end if if (any(col1_data /= data(:, 1))) then write(error_unit, *) 'Col 1 differs: ', col1_data, ' vs ', data(:, 1) success = .false. end if if (any(col2_data /= data(:, 2))) then write(error_unit, *) 'Col 2 differs: ', col2_data, ' vs ', data(:, 2) success = .false. end if if (any(col3_data /= data(:, 3))) then write(error_unit, *) 'Col 3 differs: ', col3_data, ' vs ', data(:, 3) success = .false. end if if (any(abs(col4_data - data(:, 4)) > 1.0e-10)) then write(error_unit, *) 'Col 4 differs: ', col4_data, ' vs ', data(:, 4) success = .false. end if if (any(col5_data /= data(:, 5))) then write(error_unit, *) 'Col 5 differs: ', col5_data, ' vs ', data(:, 5) success = .false. end if do row = 1, 7 if (trim(col6_data(row)) /= trim(transfer(data(row, 6:7), str16))) then write(error_unit, *) 'Col 6 differs: ', trim(col6_data(row)), ' vs ', & transfer(data(row, 6:7), str16) success = .false. end if if (trim(col7_data(row)) /= trim(transfer(data(row, 8), str8))) then write(error_unit, *) 'Col 7 differs: ', trim(col6_data(row)), ' vs ', & transfer(data(row, 8), str8) success = .false. end if end do if (any(abs(col8_data - data(:, 9)) > 1.0e-10)) then write(error_unit, *) 'Col 8 differs: ', col8_data, ' vs ', data(:, 9) success = .false. end if if (any(abs(col9_data - data(:, 10)) > 1.0e-10)) then write(error_unit, *) 'Col 9 differs: ', col9_data, ' vs ', data(:, 10) success = .false. end if if (any(abs(col10_data - data(:, 11)) > 1.0e-10)) then write(error_unit, *) 'Col 10 differs: ', col10_data, ' vs ', data(:, 11) success = .false. end if if (any(col11_data /= data(:, 12))) then write(error_unit, *) 'Col 11 differs: ', col11_data, ' vs ', data(:, 12) success = .false. end if if (any(col12_data /= data(:, 13))) then write(error_unit, *) 'Col 12 differs: ', col12_data, ' vs ', data(:, 13) success = .false. end if if (any(col13_data /= data(:, 14))) then write(error_unit, *) 'Col 13 differs: ', col13_data, ' vs ', data(:, 14) success = .false. end if if (any(col14_data /= data(:, 15))) then write(error_unit, *) 'Col 14 differs: ', col14_data, ' vs ', data(:, 15) success = .false. end if end subroutine function construct_data_column_major() result(data) real(8) :: data(7, 15) integer :: row data(:, 1) = col1_data data(:, 2) = col2_data data(:, 3) = col3_data data(:, 4) = col4_data data(:, 5) = col5_data do row = 1, 7 data(row, 6:7) = transfer(col6_data(row), 1.0_8, 2) data(row, 8) = transfer(col7_data(row), 1.0_8) end do data(:, 9) = col8_data data(:, 10) = col9_data data(:, 11) = col10_data data(:, 12) = col11_data data(:, 13) = col12_data data(:, 14) = col13_data data(:, 15) = col14_data end function subroutine initialise_encoder(encoder, success) type(odc_encoder) :: encoder logical, intent(inout) :: success call check_call(encoder%initialise(), "initialise encoder", success) call check_call(encoder%add_column("col1", ODC_INTEGER), "add col1", success) call check_call(encoder%add_column("col2", ODC_INTEGER), "add col2", success) call check_call(encoder%add_column("col3", ODC_BITFIELD), "add col3", success) call check_call(encoder%add_column("col4", ODC_DOUBLE), "add col4", success) call check_call(encoder%add_column("col5", ODC_INTEGER), "add col5", success) call check_call(encoder%add_column("col6", ODC_STRING), "add col6", success) call check_call(encoder%add_column("col7", ODC_STRING), "add col7", success) call check_call(encoder%add_column("col8", ODC_REAL), "add col8", success) call check_call(encoder%add_column("col9", ODC_DOUBLE), "add col9", success) call check_call(encoder%add_column("col10", ODC_REAL), "add col10", success) call check_call(encoder%add_column("col11", ODC_BITFIELD), "add col11", success) call check_call(encoder%add_column("col12", ODC_INTEGER), "add col12", success) call check_call(encoder%add_column("col13", ODC_INTEGER), "add col13", success) call check_call(encoder%add_column("col14", ODC_INTEGER), "add col14", success) call check_call(encoder%column_set_data_size(6, element_size_doubles=2), "column attrs", success) call check_call(encoder%column_add_bitfield(11, "bf1", 3), "add bitfield 1", success) call check_call(encoder%column_add_bitfield(11, "bf2", 2), "add bitfield 2", success) call check_call(encoder%column_add_bitfield(11, "bf3", 1), "add bitfield 3", success) end subroutine function test_encode_column_major() result(success) real(8) :: data(7, 15) integer :: outunit, iter integer(8) :: bytes_written type(odc_encoder) :: encoder character(*), parameter :: test_filename = 'f90_test_encode_column.odb' logical :: success success = .true. data = construct_data_column_major() call check_decoded_column_major(data, success) call initialise_encoder(encoder, success) ! Encode additional property key/value pairs do iter = 1, 2 call check_call(encoder%add_property(property_keys(iter), property_values(iter)), "add property", success) end do ! Put encoding in a loop. Do the encoding twice, to demonstrate that ! we can iterate through tables of data. open(newunit=outunit, file=test_filename, access='stream', form='unformatted') do iter = 0, 1 call check_call(encoder%set_data(data), "set encoder data", success) call check_call(encoder%encode(outunit, bytes_written), "do encode", success) end do close(outunit) call check_call(encoder%free(), "free encoder", success) call check_encoded_odb(test_filename, success) end function function test_encode_row_major() result(success) real(8) :: data(15, 7) integer :: row, outunit, iter integer(8) :: bytes_written type(odc_encoder) :: encoder character(*), parameter :: test_filename = 'f90_test_encode_row.odb' logical :: success success = .true. data = transpose(construct_data_column_major()) call check_decoded_column_major(transpose(data), success) call initialise_encoder(encoder, success) ! Encode additional property key/value pairs do iter = 1, 2 call check_call(encoder%add_property(property_keys(iter), property_values(iter)), "add property", success) end do ! Put encoding in a loop. Do the encoding twice, to demonstrate that ! we can iterate through tables of data. open(newunit=outunit, file=test_filename, access='stream', form='unformatted') do iter = 0, 1 call check_call(encoder%set_data(data, column_major=.false.), "set encoder data", success) call check_call(encoder%encode(outunit, bytes_written), "do encode", success) end do close(outunit) call check_call(encoder%free(), "free encoder", success) call check_encoded_odb(test_filename, success) end function subroutine check_frame_properties(frame, keys, values, success) type(odc_frame), intent(in) :: frame character(*), intent(in) :: keys(:) character(*), intent(in) :: values(:) logical, intent(inout) :: success character(:), allocatable, target :: version, key, val character(255) :: version_str integer :: iter call check_call(odc_version(version), 'getting version number', success) write(version_str, *) 'odc version ', version version_str = trim(adjustl(version_str)) ! Check encoded frame properties do iter = 1, size(keys) call check_call(frame%property(keys(iter), val), 'getting property by key', success) if (val /= values(iter)) then write(error_unit, *) 'unexpected property value for ', trim(adjustl(keys(iter))), ': ', val , ' /= ', values(iter) success = .false. end if end do ! Check common encoder property call check_call(frame%property('encoder', val), 'getting property by key', success) if (val /= version_str) then write(error_unit, *) 'unexpected property value for encoder: ', val , ' /= ', version_str success = .false. end if end subroutine subroutine check_frame_column(frame, col, name, type, success) type(odc_frame), intent(in) :: frame integer, intent(in) :: col, type character(*), intent(in) :: name logical, intent(inout) :: success character(:), allocatable :: column_name, nm integer :: ncols, column_type, element_size, element_size_doubles, bitfield_count integer :: sz, off, i integer :: expected_count, expected_sz character(3) :: expected_bf_names(3) = ['bf1', 'bf2', 'bf3'] integer :: expected_bf_sizes(3) = [3, 2, 1] integer :: expected_bf_offsets(3) = [0, 3, 5] call check_call(frame%column_count(ncols), "column count", success) if (ncols /= 14) then write(error_unit, *) 'Unexpected column count. got ', ncols, ', expected 14' success = .false. end if call check_call(frame%column_attributes(col, & name=column_name, & type=column_type, & element_size=element_size, & element_size_doubles=element_size_doubles, & bitfield_count=bitfield_count), "column attrs", success) if (column_name /= name) then write(error_unit, '(a,i2,4a)') 'Unexpected column name for column ', col, & '. Got ', column_name, ', expected ', name success = .false. end if if (column_type /= type) then write(error_unit, '(3(a,i2))') 'Unexpected column type for column ', col, & '. Got ', column_name, ', expected ', name success = .false. end if if (col == 6) then expected_sz = 2 else expected_sz = 1 end if if (element_size_doubles /= expected_sz) then write(error_unit, '(3(a,i2))') 'Unexpected column element size for column ', col, & '. Got ', element_size_doubles, ', expected ', expected_sz success = .false. end if if (element_size /= 8*expected_sz) then write(error_unit, '(3(a,i2))') 'Unexpected column element size for column ', col, & '. Got ', element_size, ', expected ', 8*expected_sz success = .false. end if if (col == 11) then expected_count = 3 else expected_count = 0 end if if (bitfield_count /= expected_count) then write(error_unit, '(3(a,i2))') 'Unexpected column bitfield_count for column ', col, & '. Got ', bitfield_count, ', expected ', expected_count success = .false. end if if (col == 11) then do i = 1, 3 call check_call(frame%bitfield_attributes(11, i, name=nm, offset=off, size=sz), 'bitfield attrs', success) if (sz /= expected_bf_sizes(i)) then write(error_unit, '(3(a,i2))') 'Unexpected bitfield size for field ', i, & '. Got ', sz, ', expected ', expected_bf_sizes(i) success = .false. end if if (off /= expected_bf_offsets(i)) then write(error_unit, '(3(a,i2))') 'Unexpected bitfield offset for field ', i, & '. Got ', off, ', expected ', expected_bf_offsets(i) success = .false. end if if (nm /= expected_bf_names(i)) then write(error_unit, '(a,i2,4a)') 'Unexpected bitfield name for field ', i, & '. Got ', nm, ', expected ', expected_bf_names(i) success = .false. end if end do end if end subroutine subroutine check_encoded_odb(path, success) character(*), intent(in) :: path logical, intent(inout) :: success type(odc_reader) :: reader type(odc_frame) :: frame type(odc_decoder) :: decoder real(8), pointer :: data(:,:) logical :: column_major integer :: err, iter integer(8) :: nrows call check_call(reader%open_path(path), "open " // path, success) call check_call(frame%initialise(reader), "initialise frame", success) ! We are expecting two frames do iter = 0, 1 call check_call(frame%next(), "get first frame", success) call check_frame_properties(frame, property_keys, property_values, success) call check_frame_column(frame, 1, "col1", ODC_INTEGER, success) call check_frame_column(frame, 2, "col2", ODC_INTEGER, success) call check_frame_column(frame, 3, "col3", ODC_BITFIELD, success) call check_frame_column(frame, 4, "col4", ODC_DOUBLE, success) call check_frame_column(frame, 5, "col5", ODC_INTEGER, success) call check_frame_column(frame, 6, "col6", ODC_STRING, success) call check_frame_column(frame, 7, "col7", ODC_STRING, success) call check_frame_column(frame, 8, "col8", ODC_REAL, success) call check_frame_column(frame, 9, "col9", ODC_DOUBLE, success) call check_frame_column(frame, 10, "col10", ODC_REAL, success) call check_frame_column(frame, 11, "col11", ODC_BITFIELD, success) call check_frame_column(frame, 12, "col12", ODC_INTEGER, success) call check_frame_column(frame, 13, "col13", ODC_INTEGER, success) call check_frame_column(frame, 14, "col14", ODC_INTEGER, success) ! Decode the data call check_call(decoder%initialise(), "initialise decoder", success) call check_call(decoder%defaults_from_frame(frame), "defaults from frame", success) call check_call(decoder%decode(frame, nrows), "decode", success) call check_call(decoder%data(data, column_major), "get data", success) if (.not. column_major) then write(error_unit, *) 'expected column major' success = .false. end if call check_decoded_column_major(data, success) call check_call(decoder%free(), "free decoder", success) end do ! And iterations done err = frame%next() if (err /= ODC_ITERATION_COMPLETE) then write(error_unit, *) 'expected iteration complete' success = .false. end if ! Cleanup call check_call(frame%free(), "free frame", success) call check_call(reader%close(), "free frame", success) end subroutine !funcion test_encode_integers() result(success) ! logical :: success ! success = .true. !end function !function test_encode_columns() result(success) ! logical :: success ! success = .true. !end function end module program fapi_general use fapi_encode_tests implicit none logical :: success success = .true. call check_call(odc_initialise_api(), "initialise api", success) call check_call(odc_set_missing_integer(999999_8), "set missing integer", success) call check_call(odc_set_missing_double(999999.0_8), "set missing double", success) success = test_encode_column_major() .and. success success = test_encode_row_major() .and. success !success = test_encode_integers() .and. success !success = test_encode_columns() .and. success if (.not. success) stop -1 end program odc-1.6.3/tests/f_api/odc_header.f900000664000175000017500000001062115146027420017270 0ustar alastairalastair! To build this program, please make sure to reference linked libraries: ! ! gfortran -lodccore -lfodc -o odc-fortran-header odc_header.f90 program odc_header use, intrinsic :: iso_fortran_env, only: error_unit,output_unit use odc implicit none character(255) :: path type(odc_reader) :: reader type(odc_frame) :: frame integer(8), target :: nrows integer :: err, narg, nframe integer :: nproperties, idx character(:), allocatable, target :: key, val character(:), allocatable, target :: name, type_name, bf_name integer, target :: ncols, col, type, element_size, bitfield_count integer, target :: bf, bf_offset, bf_size if (command_argument_count() < 1) then call usage() stop 1 end if ! Initialise API call check_call(odc_initialise_api(), 'initialising api') ! Iterate over all supplied path arguments do narg = 1, command_argument_count() call get_command_argument(narg, path) ! Open current path and initialise frame call check_call(reader%open_path(trim(path)), 'opening path') call check_call(frame%initialise(reader), 'initialising frame') write(output_unit, '(a,a)') 'File: ', trim(path) flush(output_unit) nframe = 1 ! Advance to the first frame in the stream in non-aggregated mode err = frame%next() do while (err == ODC_SUCCESS) ! Get row and column counts call check_call(frame%row_count(nrows), 'getting row count') call check_call(frame%column_count(ncols), 'getting column count') write(output_unit, '(a,i0,a,i0,a,i0)') ' Frame: ', nframe, ', Row count: ', nrows, & ', Column count: ', ncols flush(output_unit) ! Get number of properties encoded in the frame call check_call(frame%properties_count(nproperties), 'getting property count') do idx = 1, nproperties ! Get property key and value by its index call check_call(frame%property_idx(idx, key, val), 'getting property by index') write(output_unit, '(a,a,a,a)') ' Property: ', key, ' => ', val end do ! Iterate over frame columns do col = 1, ncols ! Get column information call check_call(frame%column_attributes(col, name, type, element_size, bitfield_count=bitfield_count), & 'getting column attributes') ! Lookup column type name call check_call(odc_column_type_name(type, type_name), 'getting column type name') write(output_unit, '(a,i0,a,a,a,a,a,i0)') ' Column: ', col, ', Name: ', name, & ', Type: ', type_name, ', Size: ', element_size flush(output_unit) ! Process bitfields only if (type == ODC_BITFIELD) then do bf = 1, bitfield_count ! Get bitfield information call check_call(frame%bitfield_attributes(col, bf, bf_name, bf_offset, bf_size), & 'getting bitfield attributes') write(output_unit, '(a,i0,a,a,a,i0,a,i0)') ' Bitfield: ', bf, ', Name: ', bf_name, & ', Offset: ', bf_offset, ', Nbits: ', bf_size flush(output_unit) end do end if end do nframe = nframe + 1 write(output_unit, '(a)') '' flush(output_unit) ! Advances to the next frame in the stream in non-aggregated mode err = frame%next() end do if (err /= ODC_ITERATION_COMPLETE) call check_call(err, "get next frame") call check_call(reader%close(), "closing reader") end do contains subroutine check_call(err, desc) integer, intent(in) :: err character(*), intent(in) :: desc if (err /= ODC_SUCCESS) then write(error_unit, *) '**** An error occurred in ODC library' write(error_unit, *) 'Description: ', desc write(error_unit, *) 'Error: ', odc_error_string(err) stop 1 end if end subroutine subroutine usage() write(output_unit, *) 'Usage:' write(output_unit, *) ' odc-fortran-header [ ...]' end subroutine end program odc-1.6.3/tests/f_api/usage_examples.sh0000775000175000017500000001333015146027420020234 0ustar alastairalastair#!/bin/bash set -uex # A unique working directory wd=$(pwd) test_wd=$(pwd)/usage_examples mkdir -p ${test_wd} cd ${test_wd} # In case we are resuming from a previous failed run, which has left output in the directory rm *.odb *.txt || true # Helper functions expect_error () { exit_code="${exit_code-0}" [[ "$exit_code" -eq 1 ]] || (echo "Unexpected exit code: $exit_code != 1" ; false) unset exit_code } expect_success () { exit_code="${exit_code-1}" [[ "$exit_code" -eq 0 ]] || (echo "Unexpected exit code: $exit_code != 0" ; false) unset exit_code } # -- # Test Fortran encode row-major example ${wd}/odc-fortran-encode-row-major > encode-row-major-usage.txt || exit_code=$? ; expect_error cat > encode-row-major-usage-expected.txt < EOF cmp encode-row-major-usage.txt encode-row-major-usage-expected.txt ${wd}/odc-fortran-encode-row-major ${test_wd}/test-1.odb > encode-row-major.txt && exit_code=$? ; expect_success cat > encode-row-major-expected.txt < encode-custom-usage.txt || exit_code=$? ; expect_error cat > encode-custom-usage-expected.txt < EOF cmp encode-custom-usage.txt encode-custom-usage-expected.txt ${wd}/odc-fortran-encode-custom ${test_wd}/test-2.odb > encode-custom.txt && exit_code=$? ; expect_success cat > encode-custom-expected.txt < ls-usage.txt || exit_code=$? ; expect_error cat > ls-usage-expected.txt < EOF cmp ls-usage.txt ls-usage-expected.txt ${wd}/odc-fortran-ls ${test_wd}/test-1.odb > ls.txt && exit_code=$? ; expect_success current_date=$(date +"%Y%m%d") cat > ls-expected.txt < header-usage.txt || exit_code=$? ; expect_error cat > header-usage-expected.txt < [ ...] EOF cmp header-usage.txt header-usage-expected.txt ${wd}/odc-fortran-header ${test_wd}/test-1.odb ${test_wd}/test-2.odb > header.txt && exit_code=$? ; expect_success odc_version=$(odc --version | head -n 1 | sed "s/^ODBAPI Version: \([0-9.]\)/\1/") cat > header-expected.txt < odc_example Property: encoder => odc version $odc_version Column: 1, Name: expver, Type: string, Size: 8 Column: 2, Name: date@hdr, Type: integer, Size: 8 Column: 3, Name: statid@hdr, Type: string, Size: 8 Column: 4, Name: wigos@hdr, Type: string, Size: 16 Column: 5, Name: obsvalue@body, Type: real, Size: 8 Column: 6, Name: integer_missing, Type: integer, Size: 8 Column: 7, Name: double_missing, Type: real, Size: 8 Column: 8, Name: bitfield_column, Type: bitfield, Size: 8 Bitfield: 1, Name: flag_a, Offset: 0, Nbits: 1 Bitfield: 2, Name: flag_b, Offset: 1, Nbits: 2 Bitfield: 3, Name: flag_c, Offset: 3, Nbits: 3 Bitfield: 4, Name: flag_d, Offset: 6, Nbits: 1 File: $test_wd/test-2.odb Frame: 1, Row count: 20, Column count: 8 Property: encoded_by => odc_example Property: encoder => odc version $odc_version Column: 1, Name: expver, Type: string, Size: 8 Column: 2, Name: date@hdr, Type: integer, Size: 8 Column: 3, Name: statid@hdr, Type: string, Size: 8 Column: 4, Name: wigos@hdr, Type: string, Size: 16 Column: 5, Name: obsvalue@body, Type: real, Size: 8 Column: 6, Name: integer_missing, Type: integer, Size: 8 Column: 7, Name: double_missing, Type: real, Size: 8 Column: 8, Name: bitfield_column, Type: bitfield, Size: 8 Bitfield: 1, Name: flag_a, Offset: 0, Nbits: 1 Bitfield: 2, Name: flag_b, Offset: 1, Nbits: 2 Bitfield: 3, Name: flag_c, Offset: 3, Nbits: 3 Bitfield: 4, Name: flag_d, Offset: 6, Nbits: 1 EOF cmp header.txt header-expected.txt # Clean up cd ${wd} rm -rf ${test_wd} odc-1.6.3/tests/f_api/read.f900000664000175000017500000014116615146027420016137 0ustar alastairalastair module fapi_read_tests use odc use odc_config use, intrinsic :: iso_fortran_env use, intrinsic :: iso_c_binding, only: c_loc,c_null_char implicit none contains subroutine check_call(err, msg, success) integer, intent(in) :: err character(*), intent(in) :: msg logical, intent(inout) :: success if (err /= ODC_SUCCESS) then write(error_unit, *) 'Failed API call: ', msg write(error_unit, *) 'Error: ', odc_error_string(err) success = .false. end if end subroutine function test_count_lines() result(success) ! Test that we obtain the expected version number type(odc_reader) :: reader type(odc_frame) :: frame integer(8) :: frame_count, row_count, tmp_8 integer :: err, tmp_4 logical :: success success = .true. call check_call(reader%open_path("../2000010106-reduced.odb"), "open ODB", success) call check_call(frame%initialise(reader), "initialise frame", success) frame_count = 0 row_count = 0 err = frame%next() do while (err == ODC_SUCCESS) call check_call(frame%row_count(tmp_8), "row count", success) frame_count = frame_count + 1 row_count = row_count + tmp_8 call check_call(frame%column_count(tmp_4), "column count", success) if (tmp_4 /= 51) then write(error_unit, *) 'Unexpected column count: ', tmp_4, ' /= 51' success = .false. endif err = frame%next() end do if (err /= ODC_ITERATION_COMPLETE) call check_call(err, "next frame", success) if (frame_count /= 5) then write(error_unit, *) 'Unexpected frame count: ', frame_count, ' /= 5' success = .false. endif if (row_count /= 50000) then write(error_unit, *) 'Unexpected row count: ', row_count, ' /= 50000' success = .false. endif call check_call(reader%close(), "close reader", success) end function function test_column_details() result(success) type(odc_reader) :: reader type(odc_frame) :: frame character(:), allocatable :: column_name, field_name integer :: ncols, col, column_type, field, field_size, expected_offset, field_offset integer :: element_size, element_size_doubles, bitfield_count logical :: success character(23), parameter :: example_column_names(*) = [ character(23) :: & "expver@desc", "andate@desc", "antime@desc", "seqno@hdr", "obstype@hdr", & "obschar@hdr", "subtype@hdr", "date@hdr", "time@hdr", "rdbflag@hdr", & "status@hdr", "event1@hdr", "blacklist@hdr", "sortbox@hdr", "sitedep@hdr", & "statid@hdr", "ident@hdr", "lat@hdr", "lon@hdr", "stalt@hdr", & "modoro@hdr", "trlat@hdr", "trlon@hdr", "instspec@hdr", "event2@hdr", & "anemoht@hdr", "baroht@hdr", "sensor@hdr", "numlev@hdr", "varno_presence@hdr", & "varno@body", "vertco_type@body", "rdbflag@body", "anflag@body", "status@body", & "event1@body", "blacklist@body", "entryno@body", "press@body", "press_rl@body", & "obsvalue@body", "aux1@body", "event2@body", "ppcode@body", "level@body", & "biascorr@body", "final_obs_error@errstat", "obs_error@errstat", "repres_error@errstat", & "pers_error@errstat", "fg_error@errstat"] integer, parameter :: example_column_types(*) = [ & ODC_STRING, ODC_INTEGER, ODC_INTEGER, ODC_INTEGER, ODC_INTEGER, ODC_BITFIELD , & ODC_INTEGER, ODC_INTEGER, ODC_INTEGER, ODC_BITFIELD , ODC_BITFIELD , ODC_BITFIELD , & ODC_BITFIELD , ODC_INTEGER, ODC_INTEGER, ODC_STRING, ODC_INTEGER, ODC_REAL, & ODC_REAL, ODC_REAL, ODC_REAL, ODC_REAL, ODC_REAL, ODC_INTEGER, & ODC_INTEGER, ODC_REAL, ODC_REAL, ODC_INTEGER, ODC_INTEGER, ODC_BITFIELD , & ODC_INTEGER, ODC_INTEGER, ODC_BITFIELD , ODC_BITFIELD , ODC_BITFIELD , ODC_BITFIELD , & ODC_BITFIELD , ODC_INTEGER, ODC_REAL, ODC_REAL, ODC_REAL, ODC_REAL, & ODC_INTEGER, ODC_INTEGER, ODC_BITFIELD , ODC_REAL, ODC_REAL, ODC_REAL, & ODC_REAL, ODC_REAL, ODC_REAL] character(14), parameter :: column_10_bitfield_names(*) = [ character(14) :: & "lat_humon", "lat_qcsub", "lat_override", "lat_flag", "lat_hqc_flag", "lon_humon", "lon_qcsub", & "lon_override", "lon_flag", "lon_hqc_flag", "date_humon", "date_qcsub", "date_override", & "date_flag", "date_hqc_flag", "time_humon", "time_qcsub", "time_override", "time_flag", & "time_hqc_flag", "stalt_humon", "stalt_qcsub", "stalt_override", "stalt_flag", "stalt_hqc_flag" & ] integer, parameter :: column_10_bitfield_sizes(*) = [ & 1, 1, 1, 2, 1, 1, 1, 1, 2, 1, 1, 1, 1, 2, 1, 1, 1, 1, 2, 1, 1, 1, 1, 2, 1] success = .true. call check_call(reader%open_path("../2000010106-reduced.odb"), "open reader", success) call check_call(frame%initialise(reader), "initialise frame", success) call check_call(frame%next(), "get the first frame", success) call check_call(frame%column_count(ncols), "column count", success) if (ncols /= 51) then write(error_unit, *) 'Expected 51 columns' success = .false. endif ! n.b. -- 1-based indexing! do col = 1, ncols call check_call(frame%column_attributes(col, & name=column_name, & type=column_type, & element_size=element_size, & element_size_doubles=element_size_doubles, & bitfield_count=bitfield_count), "column attrs", success) if (column_name /= trim(example_column_names(col))) then write(error_unit,'(3a,i2,3a)') 'Unexpected column name ', column_name, & ' for column ', col, ' (expected ', trim(example_column_names(col)), ')' success = .false. end if if (column_type /= example_column_types(col)) then write(error_unit, '(a,i1,a,i2,a,i1,a)') 'Unexpected column type ', column_type, & ' for column ', col, ' (expected ', example_column_types(col), ')' success = .false. end if if (element_size /= 8) then write(error_unit, '(a,i1,a,i2,a)') 'Unexpected column data size ', element_size, & ' for column ', col, ' (expected 8)' success = .false. end if if (element_size_doubles /= 1) then write(error_unit, '(a,i1,a,i2,a)') 'Unexpected column doubles data size ', & element_size_doubles, ' for column ', col, ' (expected 1)' success = .false. end if if (column_type == ODC_BITFIELD) then if (bitfield_count <= 0) then write(error_unit, *) "Bitfields expected for bitfield column" success = .false. end if else if (bitfield_count /= 0) then write(error_unit, *) "Unexpected bitfields for non-bitfield column" success = .false. end if end if end do ! Test bitfields for column 10 call check_call(frame%column_attributes(10, bitfield_count=bitfield_count), "bitfield count", success) if (bitfield_count /= 25) then write(error_unit, *) "Expected 25 bitfield fields for column 10. Got ", bitfield_count success = .false. end if expected_offset = 0 do field = 1, 25 ! Look at column 10 call check_call(frame%bitfield_attributes(10, field, & name=field_name, & offset=field_offset, & size=field_size), "bitfield attrs", success) if (field_name /= trim(column_10_bitfield_names(field))) then write(error_unit, '(3a,i2,3a)') 'Unexpected field name ', field_name, ' for field ', & field, ' (expected ', trim(column_10_bitfield_names(field)), ')' success = .false. end if if (field_size /= column_10_bitfield_sizes(field)) then write(error_unit, '(a,i2,a,i2,a,i2,a)') 'Unexpected field size ', field_size, & ' for field ', field, ' (expected ', column_10_bitfield_sizes(field), ')' success = .false. end if if (field_offset /= expected_offset) then write(error_unit, '(a,i2,a,i2,a,i2,a)') 'Unexpected field offset ', field_offset, & ' for field ', field, ' (expected ', expected_offset, ')' success = .false. end if expected_offset = expected_offset + field_size end do call check_call(frame%free(), "free frame", success) call check_call(reader%close(), "close reader", success) end function function check_frame_2_values(array_data) result(success) real(8) :: array_data(:,:) logical :: success integer :: row, i integer, parameter :: expected_seqno(*) = [6106691, 6002945, 6003233, 6105819] integer, parameter :: expected_obschar(*) = [537918674, 135265490, 135265490, 537918674] integer(8) :: missing_integer real(8) :: missing_double success = .true. call check_call(odc_missing_integer(missing_integer), "missing integer", success) call check_call(odc_missing_double(missing_double), "missing double", success) do i = 1, 4 row = 1 + ((i-1) * 765) ! Expver if (trim(transfer(array_data(row, 1), " ")) /= "0018") then write(error_unit, *) 'unexpected expver in row ', row, ' (expected 0018, got ', & transfer(array_data(row, 1), " ") ,')' success = .false. end if ! Test seqno (INTEGER) if (int(array_data(row, 4)) /= expected_seqno(i)) then write(error_unit, *) 'Unexpected seqno value. row=', row, ", expected=", & expected_seqno(i), ", got=", int(array_data(row, 4)) success = .false. end if ! obschar (BITFIELD) if (int(array_data(row, 6)) /= expected_obschar(i)) then write(error_unit, *) 'Unexpected obschar value. row=', row, ", expected=", & expected_obschar(i), ", got=", int(array_data(row, 6)) success = .false. end if ! Sortbox (INTEGER, missing) if (int(array_data(row, 14)) /= missing_integer) then write(error_unit, *) 'Expected value with set missing value. Got ', int(array_data(row, 14)), ', & &expected ', missing_integer success = .false. end if ! repres_error (REAL, missing) if (array_data(row, 49) /= missing_double) then write(error_unit, *) 'Expected value with set missing value. Got ', array_data(row, 49), ', & &expected ', missing_double success = .false. end if end do end function function test_decode_columns_allocate() result(success) type(odc_reader) :: reader type(odc_frame) :: frame type(odc_decoder) :: decoder integer(8) :: nrows, nrows2 integer :: ncols logical :: success, column_major real(8), pointer :: array_data(:,:) success =.true. call check_call(reader%open_path("../2000010106-reduced.odb"), "open reader", success) call check_call(frame%initialise(reader), "initialise frame", success) ! Read the second frame, because why not. call check_call(frame%next(), "get first frame", success) call check_call(frame%next(), "get second frame", success) call check_call(decoder%initialise(), "initialise decoder", success) call check_call(decoder%defaults_from_frame(frame), "decoder from frame", success) call check_call(decoder%decode(frame, nrows), "do decode", success) if (nrows /= 10000) then write(error_unit, *) 'Unexpected number of rows decoded' success = .false. end if call check_call(decoder%row_count(nrows2), "decoder row count", success) if (nrows2 /= 10000) then write(error_unit, *) 'Got row count ', nrows, ' not 10000' success = .false. end if call check_call(decoder%column_count(ncols), "decoder column count", success) if (ncols /= 51) then write(error_unit, *) 'Got column count ', ncols, ' not 51' success = .false. end if call check_call(decoder%data(array_data, column_major), "get decoded data", success) if (any(shape(array_data) /= [10000, 51])) then write(error_unit, *) 'Unexpected data dimensions' success = .false. end if if (.not. column_major) then write(error_unit, *) 'Expected column major by default' success = .false. end if success = success .and. check_frame_2_values(array_data) call check_call(decoder%free(), "free decoder", success) call check_call(reader%close(), "free reader", success) end function function test_decode_array_reuse() result(success) use, intrinsic :: iso_c_binding type(odc_reader) :: reader type(odc_frame) :: frame type(odc_decoder) :: decoder integer(8) :: rows_decoded, nrows integer :: ncols logical :: success real(8), target :: array_data(11000, 51) success = .true. call check_call(reader%open_path("../2000010106-reduced.odb"), "open reader", success) call check_call(frame%initialise(reader), "initialise frame", success) call check_call(frame%next(), "get first frame", success) call check_call(decoder%initialise(), "initialise decoder", success) call check_call(decoder%defaults_from_frame(frame), "decoder frame defaults", success) call check_call(decoder%set_data(array_data), "set array data", success) call check_call(decoder%decode(frame, rows_decoded), "decode first frame", success) if (rows_decoded /= 10000) then write(error_unit, *) 'Unexpected number of rows decoded' success = .false. end if call check_call(frame%next(), "get second frame", success) call check_call(decoder%decode(frame, rows_decoded), "decode second frame", success) if (rows_decoded /= 10000) then write(error_unit, *) 'Unexpected number of rows decoded' success = .false. end if call check_call(decoder%row_count(nrows), "decoder row count", success) if (nrows /= 11000) then write(error_unit, *) 'Got row count ', nrows, ' not 11000' write(error_unit, *) 'Row count should be related to the size of the decode target, not the decode data' success = .false. end if call check_call(decoder%column_count(ncols), "decoder column count", success) if (ncols /= 51) then write(error_unit, *) 'Got column count ', ncols, ' not 51' success = .false. end if success = success .and. check_frame_2_values(array_data) call check_call(decoder%free(), "free decoder", success) call check_call(reader%close(), "close reader", success) end function function test_decode_aggregate() result(success) type(odc_reader) :: reader type(odc_frame) :: frame type(odc_decoder) :: decoder integer(8) :: rows_decoded, nrows integer :: ncols logical :: success real(8), pointer :: array_data(:,:) success = .true. call check_call(reader%open_path("../2000010106-reduced.odb"), "open reader", success) call check_call(frame%initialise(reader), "initialise frame", success) call check_call(frame%next(maximum_rows=99999_8), "get first (aggregate) frame", success) call check_call(decoder%initialise(), "initialise decoder", success) call check_call(decoder%defaults_from_frame(frame), "decoder frame defaults", success) call check_call(decoder%decode(frame, rows_decoded, nthreads=4), "decode threaded", success) if (rows_decoded /= 50000) then write(error_unit, *) 'Unexpected number of rows decoded' success = .false. end if call check_call(decoder%row_count(nrows), "decoder row count", success) if (nrows /= 50000) then write(error_unit, *) 'Got row count ', nrows, ' not 50000' success = .false. end if call check_call(decoder%column_count(ncols), "decoder column count", success) if (ncols /= 51) then write(error_unit, *) 'Got column count ', ncols, ' not 51' success = .false. end if call check_call(decoder%data(array_data), "get array data", success) if (any(shape(array_data) /= [50000, 51])) then write(error_unit, *) 'Unexpected data dimensions' success = .false. end if call check_call(decoder%free(), "free decoder", success) call check_call(reader%close(), "close reader", success) end function function test_frame_properties_1_non_aggregated() result(success) ! Where the properties in the two frames are distinct (non-aggregated) type(odc_reader) :: reader type(odc_frame) :: frame integer :: err, nframes = 1, nproperties, idx logical :: aggregated = .false. character(:), allocatable, target :: version, key, val character(255) :: version_str logical :: success, exists success = .true. call test_generate_odb('properties-1.odb', 1, success) call check_call(reader%open_path('properties-1.odb'), 'opening path', success) call check_call(frame%initialise(reader), 'initialising frame', success) call check_call(odc_version(version), 'getting version number', success) write(version_str, *) 'odc version ', version version_str = trim(adjustl(version_str)) ! Advance to the first frame in the stream in non-aggregated mode err = frame%next(aggregated) do while (err == ODC_SUCCESS) call check_call(frame%properties_count(nproperties), 'getting properties count', success) ! Check properties count if (nproperties /= 2) then write(error_unit, *) 'unexpected number of properties:', nproperties, '/=', 2 success = .false. end if do idx = 1, 2 call check_call(frame%property_idx(idx, key, val), 'getting property by index', success) ! Check getting properties by index if ((idx == 1 .and. nframes == 1) .or. (idx == 2 .and. nframes == 2)) then if (key /= 'encoder' .or. val /= version_str) then write(error_unit, *) 'unexpected property: ', key, ' => ', val , ' /= encoder => ', version_str success = .false. end if else if (nframes == 1) then if (key /= 'foo' .or. val /= 'bar') then write(error_unit, *) 'unexpected property: ', key, ' => ', val , ' /= foo => bar' success = .false. end if else if (key /= 'baz' .or. val /= 'qux') then write(error_unit, *) 'unexpected property: ', key, ' => ', val , ' /= baz => qux' success = .false. end if end if ! Check getting property values by key if (nframes == 1) then call check_call(frame%property('encoder', val), 'getting property by key', success) if (val /= version_str) then write(error_unit, *) 'unexpected property value for encoder: ', val , ' /= ', version_str success = .false. end if call check_call(frame%property('foo', val), 'getting property by key', success) if (val /= 'bar') then write(error_unit, *) 'unexpected property value for foo: ', val , ' /= bar' end if else call check_call(frame%property('encoder', val), 'getting property by key', success) if (val /= version_str) then write(error_unit, *) 'unexpected property value for encoder: ', val , ' /= ', version_str success = .false. end if call check_call(frame%property('baz', val), 'getting property by key', success) if (val /= 'qux') then write(error_unit, *) 'unexpected property value for baz: ', val , ' /= qux' success = .false. end if end if ! Check for reading of non-existent properties call check_call(frame%property('non-existent', exists=exists), 'getting property by key', success) if (exists) then write(error_unit, *) 'unexpected non-existent property: ', exists, ' /= .false.' success = .false. end if end do nframes = nframes + 1 ! Advances to the next frame in the stream in non-aggregated mode err = frame%next(aggregated) end do if (err /= ODC_ITERATION_COMPLETE) call check_call(err, 'get next frame', success) call check_call(reader%close(), 'closing reader', success) ! Check number of frames if (nframes - 1 /= 2) then write(error_unit, *) 'unexpected number of frames:', nframes - 1, '/=', 2 success = .false. end if end function function test_frame_properties_1_aggregated() result(success) ! Where the properties in the two frames are distinct (aggregated) type(odc_reader) :: reader type(odc_frame) :: frame integer :: err, nframes = 1, nproperties, idx logical :: aggregated = .true. character(:), allocatable, target :: version, key, val character(255) :: version_str logical :: success, exists success = .true. call test_check_file_exists('properties-1.odb', success) call check_call(reader%open_path('properties-1.odb'), 'opening path', success) call check_call(frame%initialise(reader), 'initialising frame', success) call check_call(odc_version(version), 'getting version number', success) write(version_str, *) 'odc version ', version version_str = trim(adjustl(version_str)) ! Advance to the first frame in the stream in aggregated mode err = frame%next(aggregated) do while (err == ODC_SUCCESS) call check_call(frame%properties_count(nproperties), 'getting properties count', success) ! Check properties count if (nproperties /= 3) then write(error_unit, *) 'unexpected number of properties:', nproperties, '/=', 3 success = .false. end if do idx = 1, 3 call check_call(frame%property_idx(idx, key, val), 'getting property by index', success) ! Check getting properties by index if (idx == 1) then if (key /= 'baz' .or. val /= 'qux') then write(error_unit, *) 'unexpected property: ', key, ' => ', val , ' /= baz => qux' success = .false. end if else if (idx == 2) then if (key /= 'encoder' .or. val /= version_str) then write(error_unit, *) 'unexpected property: ', key, ' => ', val , ' /= encoder => ', version_str success = .false. end if else if (key /= 'foo' .or. val /= 'bar') then write(error_unit, *) 'unexpected property: ', key, ' => ', val , ' /= foo => bar' success = .false. end if end if ! Check getting property values by key call check_call(frame%property('encoder', val), 'getting property by key', success) if (val /= version_str) then write(error_unit, *) 'unexpected property value for encoder: ', val , ' /= ', version_str success = .false. end if call check_call(frame%property('foo', val), 'getting property by key', success) if (val /= 'bar') then write(error_unit, *) 'unexpected property value for foo: ', val , ' /= bar' success = .false. end if call check_call(frame%property('baz', val), 'getting property by key', success) if (val /= 'qux') then write(error_unit, *) 'unexpected property value for baz: ', val , ' /= qux' success = .false. end if ! Check for reading of non-existent properties call check_call(frame%property('non-existent', exists=exists), 'getting property by key', success) if (exists) then write(error_unit, *) 'unexpected non-existent property: ', exists, ' /= .false.' success = .false. end if end do nframes = nframes + 1 ! Advances to the next frame in the stream in aggregated mode err = frame%next(aggregated) end do if (err /= ODC_ITERATION_COMPLETE) call check_call(err, 'get next frame', success) call check_call(reader%close(), 'closing reader', success) ! Check number of frames if (nframes - 1 /= 1) then write(error_unit, *) 'unexpected number of frames:', nframes - 1, '/=', 1 success = .false. end if end function function test_frame_properties_2_non_aggregated() result(success) ! Where the properties in the two frames overlap with entries that are the same (non-aggregated) type(odc_reader) :: reader type(odc_frame) :: frame integer :: err, nframes = 1, nproperties, idx logical :: aggregated = .false. character(:), allocatable, target :: version, key, val character(255) :: version_str logical :: success, exists success = .true. call test_generate_odb('properties-2.odb', 2, success) call check_call(reader%open_path('properties-2.odb'), 'opening path', success) call check_call(frame%initialise(reader), 'initialising frame', success) call check_call(odc_version(version), 'getting version number', success) write(version_str, *) 'odc version ', version version_str = trim(adjustl(version_str)) ! Advance to the first frame in the stream in non-aggregated mode err = frame%next(aggregated) do while (err == ODC_SUCCESS) call check_call(frame%properties_count(nproperties), 'getting properties count', success) ! Check properties count if (nproperties /= 3) then write(error_unit, *) 'unexpected number of properties:', nproperties, '/=', 3 success = .false. end if do idx = 1, 3 call check_call(frame%property_idx(idx, key, val), 'getting property by index', success) ! Check getting properties by index if (idx == 1) then if (key /= 'baz' .or. val /= 'qux') then write(error_unit, *) 'unexpected property: ', key, ' => ', val , ' /= baz => qux' success = .false. end if else if (idx == 2) then if (key /= 'encoder' .or. val /= version_str) then write(error_unit, *) 'unexpected property: ', key, ' => ', val , ' /= encoder => ', version_str success = .false. end if else if (key /= 'foo' .or. val /= 'bar') then write(error_unit, *) 'unexpected property: ', key, ' => ', val , ' /= foo => bar' success = .false. end if end if ! Check getting property values by key call check_call(frame%property('encoder', val), 'getting property by key', success) if (val /= version_str) then write(error_unit, *) 'unexpected property value for encoder: ', val , ' /= ', version_str success = .false. end if call check_call(frame%property('foo', val), 'getting property by key', success) if (val /= 'bar') then write(error_unit, *) 'unexpected property value for foo: ', val , ' /= bar' success = .false. end if call check_call(frame%property('baz', val), 'getting property by key', success) if (val /= 'qux') then write(error_unit, *) 'unexpected property value for baz: ', val , ' /= qux' success = .false. end if ! Check for reading of non-existent properties call check_call(frame%property('non-existent', exists=exists), 'getting property by key', success) if (exists) then write(error_unit, *) 'unexpected non-existent property: ', exists, ' /= .false.' success = .false. end if end do nframes = nframes + 1 ! Advances to the next frame in the stream in non-aggregated mode err = frame%next(aggregated) end do if (err /= ODC_ITERATION_COMPLETE) call check_call(err, 'get next frame', success) call check_call(reader%close(), 'closing reader', success) ! Check number of frames if (nframes - 1 /= 2) then write(error_unit, *) 'unexpected number of frames:', nframes - 1, '/=', 2 success = .false. end if end function function test_frame_properties_2_aggregated() result(success) ! Where the properties in the two frames overlap with entries that are the same (aggregated) type(odc_reader) :: reader type(odc_frame) :: frame integer :: err, nframes = 1, nproperties, idx logical :: aggregated = .true. character(:), allocatable, target :: version, key, val character(255) :: version_str logical :: success, exists success = .true. call test_check_file_exists('properties-2.odb', success) call check_call(reader%open_path('properties-2.odb'), 'opening path', success) call check_call(frame%initialise(reader), 'initialising frame', success) call check_call(odc_version(version), 'getting version number', success) write(version_str, *) 'odc version ', version version_str = trim(adjustl(version_str)) ! Advance to the first frame in the stream in aggregated mode err = frame%next(aggregated) do while (err == ODC_SUCCESS) call check_call(frame%properties_count(nproperties), 'getting properties count', success) ! Check properties count if (nproperties /= 3) then write(error_unit, *) 'unexpected number of properties:', nproperties, '/=', 3 success = .false. end if do idx = 1, 3 call check_call(frame%property_idx(idx, key, val), 'getting property by index', success) ! Check getting properties by index if (idx == 1) then if (key /= 'baz' .or. val /= 'qux') then write(error_unit, *) 'unexpected property: ', key, ' => ', val , ' /= baz => qux' success = .false. end if else if (idx == 2) then if (key /= 'encoder' .or. val /= version_str) then write(error_unit, *) 'unexpected property: ', key, ' => ', val , ' /= encoder => ', version_str success = .false. end if else if (key /= 'foo' .or. val /= 'bar') then write(error_unit, *) 'unexpected property: ', key, ' => ', val , ' /= foo => bar' success = .false. end if end if ! Check getting property values by key call check_call(frame%property('encoder', val), 'getting property by key', success) if (val /= version_str) then write(error_unit, *) 'unexpected property value for encoder: ', val , ' /= ', version_str success = .false. end if call check_call(frame%property('foo', val), 'getting property by key', success) if (val /= 'bar') then write(error_unit, *) 'unexpected property value for foo: ', val , ' /= bar' success = .false. end if call check_call(frame%property('baz', val), 'getting property by key', success) if (val /= 'qux') then write(error_unit, *) 'unexpected property value for baz: ', val , ' /= qux' success = .false. end if ! Check for reading of non-existent properties call check_call(frame%property('non-existent', exists=exists), 'getting property by key', success) if (exists) then write(error_unit, *) 'unexpected non-existent property: ', exists, ' /= .false.' success = .false. end if end do nframes = nframes + 1 ! Advances to the next frame in the stream in aggregated mode err = frame%next(aggregated) end do if (err /= ODC_ITERATION_COMPLETE) call check_call(err, 'get next frame', success) call check_call(reader%close(), 'closing reader', success) ! Check number of frames if (nframes - 1 /= 1) then write(error_unit, *) 'unexpected number of frames:', nframes - 1, '/=', 1 success = .false. end if end function function test_frame_properties_3_non_aggregated() result(success) ! Where the properties overlap with entries whose keys are the same, but the values different (non-aggregated) type(odc_reader) :: reader type(odc_frame) :: frame integer :: err, nframes = 1, nproperties, idx logical :: aggregated = .false. character(:), allocatable, target :: version, key, val character(255) :: version_str logical :: success, exists success = .true. call test_generate_odb('properties-3.odb', 3, success) call check_call(reader%open_path('properties-3.odb'), 'opening path', success) call check_call(frame%initialise(reader), 'initialising frame', success) call check_call(odc_version(version), 'getting version number', success) write(version_str, *) 'odc version ', version version_str = trim(adjustl(version_str)) ! Advance to the first frame in the stream in non-aggregated mode err = frame%next(aggregated) do while (err == ODC_SUCCESS) call check_call(frame%properties_count(nproperties), 'getting properties count', success) ! Check properties count if (nproperties /= 2) then write(error_unit, *) 'unexpected number of properties:', nproperties, '/=', 2 success = .false. end if do idx = 1, 2 call check_call(frame%property_idx(idx, key, val), 'getting property by index', success) ! Check getting properties by index if (idx == 1) then if (key /= 'encoder' .or. val /= version_str) then write(error_unit, *) 'unexpected property: ', key, ' => ', val , ' /= encoder => ', version_str success = .false. end if else if (nframes == 1) then if (key /= 'foo' .or. val /= 'bar') then write(error_unit, *) 'unexpected property: ', key, ' => ', val , ' /= foo => bar' success = .false. end if else if (key /= 'foo' .or. val /= 'baz') then write(error_unit, *) 'unexpected property: ', key, ' => ', val , ' /= foo => baz' success = .false. end if end if ! Check getting property values by key if (nframes == 1) then call check_call(frame%property('encoder', val), 'getting property by key', success) if (val /= version_str) then write(error_unit, *) 'unexpected property value for encoder: ', val , ' /= ', version_str success = .false. end if call check_call(frame%property('foo', val), 'getting property by key', success) if (val /= 'bar') then write(error_unit, *) 'unexpected property value for foo: ', val , ' /= bar' success = .false. end if else call check_call(frame%property('encoder', val), 'getting property by key', success) if (val /= version_str) then write(error_unit, *) 'unexpected property value for encoder: ', val , ' /= ', version_str success = .false. end if call check_call(frame%property('foo', val), 'getting property by key', success) if (val /= 'baz') then write(error_unit, *) 'unexpected property value for foo: ', val , ' /= baz' success = .false. end if end if ! Check for reading of non-existent properties call check_call(frame%property('non-existent', exists=exists), 'getting property by key', success) if (exists) then write(error_unit, *) 'unexpected non-existent property: ', exists, ' /= F' success = .false. end if end do nframes = nframes + 1 ! Advances to the next frame in the stream in non-aggregated mode err = frame%next(aggregated) end do if (err /= ODC_ITERATION_COMPLETE) call check_call(err, 'get next frame', success) call check_call(reader%close(), 'closing reader', success) ! Check number of frames if (nframes - 1 /= 2) then write(error_unit, *) 'unexpected number of frames:', nframes - 1, '/=', 2 success = .false. end if end function function test_frame_properties_3_aggregated() result(success) ! Where the properties overlap with entries whose keys are the same, but the values different (aggregated) type(odc_reader) :: reader type(odc_frame) :: frame integer :: err, nframes = 1, nproperties, idx logical :: aggregated = .true. character(:), allocatable, target :: version, key, val character(255) :: version_str logical :: success, exists success = .true. call test_check_file_exists('properties-3.odb', success) call check_call(reader%open_path('properties-3.odb'), 'opening path', success) call check_call(frame%initialise(reader), 'initialising frame', success) call check_call(odc_version(version), 'getting version number', success) write(version_str, *) 'odc version ', version version_str = trim(adjustl(version_str)) ! Advance to the first frame in the stream in aggregated mode err = frame%next(aggregated) do while (err == ODC_SUCCESS) call check_call(frame%properties_count(nproperties), 'getting properties count', success) ! Check properties count if (nproperties /= 2) then write(error_unit, *) 'unexpected number of properties:', nproperties, '/=', 2 success = .false. end if do idx = 1, 2 call check_call(frame%property_idx(idx, key, val), 'getting property by index', success) ! Check getting properties by index if (idx == 1) then if (key /= 'encoder' .or. val /= version_str) then write(error_unit, *) 'unexpected property: ', key, ' => ', val , ' /= encoder => ', version_str success = .false. end if else ! Value from the first frame will win if (key /= 'foo' .or. val /= 'bar') then write(error_unit, *) 'unexpected property: ', key, ' => ', val , ' /= foo => bar' success = .false. end if end if ! Check getting property values by key call check_call(frame%property('encoder', val), 'getting property by key', success) if (val /= version_str) then write(error_unit, *) 'unexpected property value for encoder: ', val , ' /= ', version_str success = .false. end if call check_call(frame%property('foo', val), 'getting property by key', success) ! Value from the first frame will win if (val /= 'bar') then write(error_unit, *) 'unexpected property value for foo: ', val , ' /= bar' success = .false. end if ! Check for reading of non-existent properties call check_call(frame%property('non-existent', exists=exists), 'getting property by key', success) if (exists) then write(error_unit, *) 'unexpected non-existent property: ', exists, ' /= .false.' success = .false. end if end do nframes = nframes + 1 ! Advances to the next frame in the stream in aggregated mode err = frame%next(aggregated) end do if (err /= ODC_ITERATION_COMPLETE) call check_call(err, 'get next frame', success) call check_call(reader%close(), 'closing reader', success) ! Check number of frames if (nframes - 1 /= 1) then write(error_unit, *) 'unexpected number of frames:', nframes - 1, '/=', 1 success = .false. end if end function subroutine test_check_file_exists(path, success) character(*), intent(in) :: path logical, intent(inout) :: success inquire(file=path, exist=success) if (.not. success) then write(error_unit, *) 'unexpected missing file: ', path end if end subroutine subroutine test_generate_odb(path, properties_mode, success) character(*), intent(in) :: path integer, intent(in) :: properties_mode logical, intent(inout) :: success integer(8), parameter :: nrows = 10 character(8), target :: data1(nrows) integer(8), target :: data2(nrows) real(8), target :: data3(nrows) character(4) :: expver_str = 'xxxx' integer(8) :: date = 20210401 integer :: i type(odc_encoder) :: encoder integer, target :: outunit integer(8), target :: bytes_written ! Set treatment of integers as longs call check_call(odc_integer_behaviour(ODC_INTEGERS_AS_LONGS), 'setting integer behaviour to longs', success) ! Fill in the passed data arrays with scratch values do i = 1, nrows data1(i) = expver_str // c_null_char ! expver data2(i) = date ! date@hdr data3(i) = 12.3456 * (i - 1) ! obsvalue@body end do ! Encode ODB-2 into a file open(newunit=outunit, file=path, access='stream', form='unformatted', status='replace') ! Encode two ODB-2 frames with the same data do i = 1, 2 ! Initialise encoder call check_call(encoder%initialise(), 'initialising encoder', success) ! Set number of rows to allocate in the encoder call check_call(encoder%set_row_count(nrows), 'setting number of rows', success) ! Define all column names and their types call check_call(encoder%add_column('expver', ODC_STRING), 'adding expver column', success) call check_call(encoder%add_column('date@hdr', ODC_INTEGER), 'adding date@hdr column', success) call check_call(encoder%add_column('obsvalue@body', ODC_REAL), 'adding obsvalue@body column', success) ! Set a custom data layout and data array for each column call check_call(encoder%column_set_data_array(1, 8, stride=8, data=c_loc(data1)), 'setting expver array', success) call check_call(encoder%column_set_data_array(2, 8, stride=8, data=c_loc(data2)), 'setting date array', success) call check_call(encoder%column_set_data_array(3, 8, stride=8, data=c_loc(data3)), 'setting obsvalue array', success) ! Encode additional properties depending on the current mode select case(properties_mode) ! Where the properties in the two frames are distinct case (1) if (i == 1) then call check_call(encoder%add_property('foo', 'bar'), 'adding property', success) else call check_call(encoder%add_property('baz', 'qux'), 'adding property', success) end if ! Where the properties in the two frames overlap with entries that are the same case (2) call check_call(encoder%add_property('foo', 'bar'), 'adding property', success) call check_call(encoder%add_property('baz', 'qux'), 'adding property', success) ! Where the properties overlap with entries whose keys are the same, but the values different case (3) if (i == 1) then call check_call(encoder%add_property('foo', 'bar'), 'adding property', success) else call check_call(encoder%add_property('foo', 'baz'), 'adding property', success) end if end select call check_call(encoder%encode(outunit, bytes_written), 'do encode', success) ! Deallocate memory used up by the encoder call check_call(encoder%free(), 'cleaning up encoder', success) end do close(outunit) end subroutine end module program fapi_general use fapi_read_tests implicit none logical :: success success = .true. call check_call(odc_initialise_api(), "initialise api", success) success = test_count_lines() .and. success success = test_column_details() .and. success success = test_decode_columns_allocate() .and. success success = test_decode_array_reuse() .and. success success = test_decode_aggregate() .and. success success = test_frame_properties_1_non_aggregated() .and. success success = test_frame_properties_1_aggregated() .and. success success = test_frame_properties_2_non_aggregated() .and. success success = test_frame_properties_2_aggregated() .and. success success = test_frame_properties_3_non_aggregated() .and. success success = test_frame_properties_3_aggregated() .and. success if (.not. success) stop -1 end program odc-1.6.3/tests/c_api/0000775000175000017500000000000015146027420014670 5ustar alastairalastairodc-1.6.3/tests/c_api/encode.cc0000664000175000017500000006105515146027420016443 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include // TODO: unneeded #include #include #include "eckit/filesystem/TmpFile.h" #include "eckit/io/Buffer.h" #include "eckit/testing/Test.h" #include "odc/api/odc.h" using namespace eckit::testing; // Specialise custom deletion #define CHECK_RETURN(x) EXPECT((x) == ODC_SUCCESS) namespace std { template <> struct default_delete { void operator()(odc_encoder_t* e) { CHECK_RETURN(odc_free_encoder(e)); } }; template <> struct default_delete { void operator()(odc_reader_t* o) { CHECK_RETURN(odc_close(o)); } }; template <> struct default_delete { void operator()(odc_frame_t* t) { CHECK_RETURN(odc_free_frame(t)); } }; template <> struct default_delete { void operator()(odc_decoder_t* t) { CHECK_RETURN(odc_free_decoder(t)); } }; } // namespace std // ------------------------------------------------------------------------------------------------------ // TODO: Row-major // TODO: Column major CASE("Encode data in standard tabular form") { CHECK_RETURN(odc_integer_behaviour(ODC_INTEGERS_AS_DOUBLES)); const int nrows = 15; const int ncols = 8; double data[nrows][ncols]; // Construct some data to encode for (int row = 0; row < nrows; ++row) { for (int col = 0; col < 4; ++col) { data[row][col] = (1000 * row) + (3 * col); } data[row][4] = 0; data[row][5] = 0; ::strncpy(reinterpret_cast(&data[row][4]), "abcdefghijkl", 2 * sizeof(double)); for (int col = 6; col < ncols; ++col) { data[row][col] = (1000 * row) + (3 * col); } } // Configure the encoder to encode said data odc_encoder_t* enc = nullptr; CHECK_RETURN(odc_new_encoder(&enc)); std::unique_ptr enc_deleter(enc); bool columnMajor = false; CHECK_RETURN(odc_encoder_set_row_count(enc, nrows)); CHECK_RETURN(odc_encoder_set_data_array(enc, data, ncols * sizeof(double), nrows, columnMajor)); CHECK_RETURN(odc_encoder_add_column(enc, "col1", ODC_INTEGER)); CHECK_RETURN(odc_encoder_add_column(enc, "col2", ODC_REAL)); CHECK_RETURN(odc_encoder_add_column(enc, "col3", ODC_DOUBLE)); CHECK_RETURN(odc_encoder_add_column(enc, "col4", ODC_DOUBLE)); CHECK_RETURN(odc_encoder_add_column(enc, "col5", ODC_STRING)); int elementSize = 2 * sizeof(double); CHECK_RETURN(odc_encoder_column_set_data_array(enc, 4, elementSize, 0, 0)); CHECK_RETURN(odc_encoder_add_column(enc, "col6", ODC_REAL)); CHECK_RETURN(odc_encoder_add_column(enc, "col7", ODC_BITFIELD)); CHECK_RETURN(odc_encoder_column_add_bitfield(enc, 6, "bits1", 2)); CHECK_RETURN(odc_encoder_column_add_bitfield(enc, 6, "bits2", 3)); CHECK_RETURN(odc_encoder_column_add_bitfield(enc, 6, "bits3", 1)); // Do the encoding eckit::Buffer encoded(1024 * 1024); long sz; CHECK_RETURN(odc_encode_to_buffer(enc, encoded.data(), encoded.size(), &sz)); // Check that the table contains what we expect odc_reader_t* reader = nullptr; CHECK_RETURN(odc_open_buffer(&reader, encoded.data(), sz)); EXPECT(reader); std::unique_ptr reader_deleter(reader); odc_frame_t* frame = nullptr; CHECK_RETURN(odc_new_frame(&frame, reader)); EXPECT(frame); std::unique_ptr frame_deleter(frame); CHECK_RETURN(odc_next_frame(frame)); int column_count; CHECK_RETURN(odc_frame_column_count(frame, &column_count)); EXPECT(column_count == ncols - 1); long row_count; CHECK_RETURN(odc_frame_row_count(frame, &row_count)); EXPECT(row_count == nrows); const char* column_names[] = {"col1", "col2", "col3", "col4", "col5", "col6", "col7"}; int column_types[] = {ODC_INTEGER, ODC_REAL, ODC_DOUBLE, ODC_DOUBLE, ODC_STRING, ODC_REAL, ODC_BITFIELD}; const char* bitfield_names[] = {"bits1", "bits2", "bits3"}; int bitfield_sizes[] = {2, 3, 1}; int bitfield_offsets[] = {0, 2, 5}; for (int col = 0; col < 7; ++col) { const char* name; int type; int elementSize; int bitfieldCount; CHECK_RETURN(odc_frame_column_attributes(frame, col, &name, &type, &elementSize, &bitfieldCount)); EXPECT(name); EXPECT(::strcmp(name, column_names[col]) == 0); EXPECT(type == column_types[col]); EXPECT(elementSize == (col == 4 ? 2 : 1) * int(sizeof(double))); EXPECT(bitfieldCount == (col == 6 ? 3 : 0)); if (col == 6) { for (int bf = 0; bf < 3; ++bf) { const char* bf_name; int bfSize; int bfOffset; CHECK_RETURN(odc_frame_bitfield_attributes(frame, col, bf, &bf_name, &bfOffset, &bfSize)); EXPECT(bf_name); EXPECT(::strcmp(bf_name, bitfield_names[bf]) == 0); EXPECT(bfSize == bitfield_sizes[bf]); EXPECT(bfOffset == bitfield_offsets[bf]); } } } // Test that the data is correctly encoded odc_decoder_t* decoder; CHECK_RETURN(odc_new_decoder(&decoder)); EXPECT(decoder); std::unique_ptr decoder_deleter(decoder); CHECK_RETURN(odc_decoder_defaults_from_frame(decoder, frame)); long rows_decoded; CHECK_RETURN(odc_decode(decoder, frame, &rows_decoded)); EXPECT(rows_decoded == nrows); const void* p; CHECK_RETURN(odc_decoder_column_data_array(decoder, 0, 0, 0, &p)); const void* pdata; long row_stride; bool decodeColumnMajor; CHECK_RETURN(odc_decoder_data_array(decoder, &pdata, &row_stride, 0, &decodeColumnMajor)); EXPECT(!decodeColumnMajor); EXPECT(p != nullptr); EXPECT(pdata != nullptr); EXPECT(p == pdata); EXPECT(row_stride == 8 * sizeof(double)); double vals1[] = {0, 1000, 2000, 3000, 4000, 5000, 6000, 7000, 8000, 9000, 10000, 11000, 12000, 13000, 14000}; double vals2[] = {3, 1003, 2003, 3003, 4003, 5003, 6003, 7003, 8003, 9003, 10003, 11003, 12003, 13003, 14003}; double vals3[] = {6, 1006, 2006, 3006, 4006, 5006, 6006, 7006, 8006, 9006, 10006, 11006, 12006, 13006, 14006}; double vals4[] = {9, 1009, 2009, 3009, 4009, 5009, 6009, 7009, 8009, 9009, 10009, 11009, 12009, 13009, 14009}; double vals6[] = {18, 1018, 2018, 3018, 4018, 5018, 6018, 7018, 8018, 9018, 10018, 11018, 12018, 13018, 14018}; double vals7[] = {21, 1021, 2021, 3021, 4021, 5021, 6021, 7021, 8021, 9021, 10021, 11021, 12021, 13021, 14021}; const double(*row_data)[8] = reinterpret_cast(pdata); for (size_t row = 0; row < nrows; ++row) { EXPECT(vals1[row] == row_data[row][0]); EXPECT(vals2[row] == row_data[row][1]); EXPECT(vals3[row] == row_data[row][2]); EXPECT(vals4[row] == row_data[row][3]); EXPECT(::strncmp("abcdefghijkl", reinterpret_cast(&row_data[row][4]), 2 * sizeof(double)) == 0); EXPECT(vals6[row] == row_data[row][6]); EXPECT(vals7[row] == row_data[row][7]); } EXPECT(odc_next_frame(frame) == ODC_ITERATION_COMPLETE); } CASE("Encode from columnar data") { odc_integer_behaviour(ODC_INTEGERS_AS_LONGS); const int nrows = 10; const int ncols = 5; // Construct some source data long icol[nrows] = {1111, 2222, 3333, 4444, 5555, 6666, 7777, 8888, 9999, 0}; long bcol[nrows] = {1101, 2202, 3303, 4404, 5505, 6606, 7707, 8808, 9909, 0}; char scol[nrows][3 * sizeof(double)] = {0}; double dcol[nrows] = {1131, 2232, 3333, 4434, 5535, 6636, 7737, 8838, 9939, 0}; double rcol[nrows] = {1141, 2242, 3343, 4444, 5545, 6646, 7747, 8848, 9949, 0}; for (size_t i = 0; i < nrows; ++i) { ::strncpy(&scol[i][0], "abcdefghhgfedcbazzzz", 3 * sizeof(double)); } // Configure the encoder to encode said data odc_encoder_t* enc = nullptr; CHECK_RETURN(odc_new_encoder(&enc)); std::unique_ptr enc_deleter(enc); CHECK_RETURN(odc_encoder_set_row_count(enc, nrows)); CHECK_RETURN(odc_encoder_add_column(enc, "col1", ODC_INTEGER)); CHECK_RETURN(odc_encoder_add_column(enc, "col2", ODC_BITFIELD)); CHECK_RETURN(odc_encoder_add_column(enc, "col3", ODC_STRING)); CHECK_RETURN(odc_encoder_add_column(enc, "col4", ODC_DOUBLE)); CHECK_RETURN(odc_encoder_add_column(enc, "col5", ODC_REAL)); int elementSizeCol2 = 3 * sizeof(double); CHECK_RETURN(odc_encoder_column_set_data_array(enc, 0, 0, 0, icol)); CHECK_RETURN(odc_encoder_column_set_data_array(enc, 1, 0, 0, bcol)); CHECK_RETURN(odc_encoder_column_set_data_array(enc, 2, elementSizeCol2, 0, scol)); CHECK_RETURN(odc_encoder_column_set_data_array(enc, 3, 0, 0, dcol)); CHECK_RETURN(odc_encoder_column_set_data_array(enc, 4, 0, 0, rcol)); // Do the encoding eckit::Buffer encoded(1024 * 1024); long sz; CHECK_RETURN(odc_encode_to_buffer(enc, encoded.data(), encoded.size(), &sz)); // Check that we have encoded what we think we have encoded odc_reader_t* reader = nullptr; CHECK_RETURN(odc_open_buffer(&reader, encoded.data(), sz)); EXPECT(reader); std::unique_ptr reader_deleter(reader); odc_frame_t* frame = nullptr; CHECK_RETURN(odc_new_frame(&frame, reader)); EXPECT(frame); std::unique_ptr frame_deleter(frame); CHECK_RETURN(odc_next_frame(frame)); int column_count; CHECK_RETURN(odc_frame_column_count(frame, &column_count)); EXPECT(column_count == ncols); long row_count; CHECK_RETURN(odc_frame_row_count(frame, &row_count)); EXPECT(row_count == nrows); // Test that the data is correctly encoded odc_decoder_t* decoder; CHECK_RETURN(odc_new_decoder(&decoder)); EXPECT(decoder); std::unique_ptr decoder_deleter(decoder); CHECK_RETURN(odc_decoder_defaults_from_frame(decoder, frame)); long rows_decoded; CHECK_RETURN(odc_decode(decoder, frame, &rows_decoded)); EXPECT(rows_decoded == nrows); const void* pdata; long row_stride; bool decodeColumnMajor; CHECK_RETURN(odc_decoder_data_array(decoder, &pdata, &row_stride, 0, &decodeColumnMajor)); EXPECT(!decodeColumnMajor); EXPECT(pdata != nullptr); EXPECT(row_stride == 7 * sizeof(double)); const double(*row_data)[7] = reinterpret_cast(pdata); for (size_t row = 0; row < nrows; ++row) { EXPECT(reinterpret_cast(row_data[row][0]) == icol[row]); EXPECT(reinterpret_cast(row_data[row][1]) == bcol[row]); EXPECT(::strncmp("abcdefghhgfedcbazzzz", reinterpret_cast(&row_data[row][2]), 3 * sizeof(double)) == 0); EXPECT(row_data[row][5] == dcol[row]); EXPECT(row_data[row][6] == rcol[row]); } EXPECT(odc_next_frame(frame) == ODC_ITERATION_COMPLETE); } //// ------------------------------------------------------------------------------------------------------ CASE("Encode data with custom stride") { odc_integer_behaviour(ODC_INTEGERS_AS_LONGS); const int nrows = 5; const int ncols = 5; // Construct some source data long icol[2 * nrows] = {1111, 2222, 3333, 4444, 5555, 6666, 7777, 8888, 9999, 0}; long bcol[2 * nrows] = {1101, 2202, 3303, 4404, 5505, 6606, 7707, 8808, 9909, 0}; char scol[2 * nrows][3 * sizeof(double)] = {0}; double dcol[2 * nrows] = {1131, 2232, 3333, 4434, 5535, 6636, 7737, 8838, 9939, 0}; double rcol[2 * nrows] = {1141, 2242, 3343, 4444, 5545, 6646, 7747, 8848, 9949, 0}; for (size_t i = 0; i < 2 * nrows; ++i) { ::strncpy(&scol[i][0], "abcdefghhgfedcbazzzz", 3 * sizeof(double)); } // Configure the encoder to encode said data odc_encoder_t* enc = nullptr; CHECK_RETURN(odc_new_encoder(&enc)); std::unique_ptr enc_deleter(enc); CHECK_RETURN(odc_encoder_set_row_count(enc, nrows)); CHECK_RETURN(odc_encoder_add_column(enc, "col1", ODC_INTEGER)); CHECK_RETURN(odc_encoder_add_column(enc, "col2", ODC_BITFIELD)); CHECK_RETURN(odc_encoder_add_column(enc, "col3", ODC_STRING)); CHECK_RETURN(odc_encoder_add_column(enc, "col4", ODC_DOUBLE)); CHECK_RETURN(odc_encoder_add_column(enc, "col5", ODC_REAL)); CHECK_RETURN(odc_encoder_column_set_data_array(enc, 0, 0, 2 * sizeof(icol[0]), icol)); CHECK_RETURN(odc_encoder_column_set_data_array(enc, 1, 0, 2 * sizeof(bcol[0]), bcol)); CHECK_RETURN(odc_encoder_column_set_data_array(enc, 2, sizeof(scol[0]), 2 * sizeof(scol[0]), scol)); CHECK_RETURN(odc_encoder_column_set_data_array(enc, 3, 0, 2 * sizeof(dcol[0]), dcol)); CHECK_RETURN(odc_encoder_column_set_data_array(enc, 4, 0, 2 * sizeof(rcol[0]), rcol)); // Do the encoding eckit::Buffer encoded(1024 * 1024); long sz; CHECK_RETURN(odc_encode_to_buffer(enc, encoded.data(), encoded.size(), &sz)); // Check that we have encoded what we think we have encoded odc_reader_t* reader = nullptr; CHECK_RETURN(odc_open_buffer(&reader, encoded.data(), sz)); EXPECT(reader); std::unique_ptr reader_deleter(reader); odc_frame_t* frame = nullptr; CHECK_RETURN(odc_new_frame(&frame, reader)); EXPECT(frame); std::unique_ptr frame_deleter(frame); CHECK_RETURN(odc_next_frame(frame)); int column_count; CHECK_RETURN(odc_frame_column_count(frame, &column_count)); EXPECT(column_count == ncols); long row_count; CHECK_RETURN(odc_frame_row_count(frame, &row_count)); EXPECT(row_count == nrows); // Test that the data is correctly encoded odc_decoder_t* decoder; CHECK_RETURN(odc_new_decoder(&decoder)); EXPECT(decoder); std::unique_ptr decoder_deleter(decoder); CHECK_RETURN(odc_decoder_defaults_from_frame(decoder, frame)); long rows_decoded; CHECK_RETURN(odc_decode(decoder, frame, &rows_decoded)); EXPECT(rows_decoded == nrows); const void* pdata; long row_stride; bool decodeColumnMajor; CHECK_RETURN(odc_decoder_data_array(decoder, &pdata, &row_stride, 0, &decodeColumnMajor)); EXPECT(!decodeColumnMajor); EXPECT(pdata != nullptr); EXPECT(row_stride == 7 * sizeof(double)); const double(*row_data)[7] = reinterpret_cast(pdata); for (size_t row = 0; row < nrows; ++row) { EXPECT(reinterpret_cast(row_data[row][0]) == icol[2 * row]); EXPECT(reinterpret_cast(row_data[row][1]) == bcol[2 * row]); EXPECT(::strncmp("abcdefghhgfedcbazzzz", reinterpret_cast(&row_data[row][2]), 3 * sizeof(double)) == 0); EXPECT(row_data[row][5] == dcol[2 * row]); EXPECT(row_data[row][6] == rcol[2 * row]); } EXPECT(odc_next_frame(frame) == ODC_ITERATION_COMPLETE); } // ------------------------------------------------------------------------------------------------------ CASE("Encode with more rows that fit inside a table") { odc_integer_behaviour(ODC_INTEGERS_AS_LONGS); const int nrows = 10; const int ncols = 5; // Construct some source data long icol[nrows] = {1111, 2222, 3333, 4444, 5555, 6666, 7777, 8888, 9999, 0}; long bcol[nrows] = {1101, 2202, 3303, 4404, 5505, 6606, 7707, 8808, 9909, 0}; char scol[nrows][3 * sizeof(double)] = {0}; double dcol[nrows] = {1131, 2232, 3333, 4434, 5535, 6636, 7737, 8838, 9939, 0}; double rcol[nrows] = {1141, 2242, 3343, 4444, 5545, 6646, 7747, 8848, 9949, 0}; for (size_t i = 0; i < nrows; ++i) { ::strncpy(&scol[i][0], "abcdefghhgfedcbazzzz", 3 * sizeof(double)); } // Configure the encoder to encode said data odc_encoder_t* enc = nullptr; CHECK_RETURN(odc_new_encoder(&enc)); std::unique_ptr enc_deleter(enc); CHECK_RETURN(odc_encoder_set_row_count(enc, nrows)); CHECK_RETURN(odc_encoder_add_column(enc, "col1", ODC_INTEGER)); CHECK_RETURN(odc_encoder_add_column(enc, "col2", ODC_BITFIELD)); CHECK_RETURN(odc_encoder_add_column(enc, "col3", ODC_STRING)); CHECK_RETURN(odc_encoder_add_column(enc, "col4", ODC_DOUBLE)); CHECK_RETURN(odc_encoder_add_column(enc, "col5", ODC_REAL)); CHECK_RETURN(odc_encoder_column_set_data_array(enc, 0, 0, 0, icol)); CHECK_RETURN(odc_encoder_column_set_data_array(enc, 1, 0, 0, bcol)); CHECK_RETURN(odc_encoder_column_set_data_array(enc, 2, sizeof(scol[0]), 0, scol)); CHECK_RETURN(odc_encoder_column_set_data_array(enc, 3, 0, 0, dcol)); CHECK_RETURN(odc_encoder_column_set_data_array(enc, 4, 0, 0, rcol)); // Set reduced number of lines per frame int maxPerFrame = 5; CHECK_RETURN(odc_encoder_set_rows_per_frame(enc, maxPerFrame)); // Do the encoding eckit::Buffer encoded(1024 * 1024); long sz; CHECK_RETURN(odc_encode_to_buffer(enc, encoded.data(), encoded.size(), &sz)); // Check that we have encoded what we think we have encoded odc_reader_t* reader = nullptr; CHECK_RETURN(odc_open_buffer(&reader, encoded.data(), sz)); EXPECT(reader); std::unique_ptr reader_deleter(reader); odc_frame_t* frame = nullptr; CHECK_RETURN(odc_new_frame(&frame, reader)); EXPECT(frame); std::unique_ptr frame_deleter(frame); // We now expect two frames for (int frame_idx = 0; frame_idx < 2; ++frame_idx) { CHECK_RETURN(odc_next_frame(frame)); int row_offset = (frame_idx * maxPerFrame); int column_count; CHECK_RETURN(odc_frame_column_count(frame, &column_count)); EXPECT(column_count == ncols); long row_count; CHECK_RETURN(odc_frame_row_count(frame, &row_count)); EXPECT(row_count == (frame_idx ? nrows - maxPerFrame : maxPerFrame)); // Test that the data is correctly encoded odc_decoder_t* decoder; CHECK_RETURN(odc_new_decoder(&decoder)); EXPECT(decoder); std::unique_ptr decoder_deleter(decoder); CHECK_RETURN(odc_decoder_defaults_from_frame(decoder, frame)); long rows_decoded; CHECK_RETURN(odc_decode(decoder, frame, &rows_decoded)); EXPECT(rows_decoded == row_count); const void* pdata; long row_stride; bool decodeColumnMajor; CHECK_RETURN(odc_decoder_data_array(decoder, &pdata, &row_stride, 0, &decodeColumnMajor)); EXPECT(!decodeColumnMajor); EXPECT(pdata != nullptr); EXPECT(row_stride == 7 * sizeof(double)); const double(*row_data)[7] = reinterpret_cast(pdata); for (int row = 0; row < row_count; ++row) { EXPECT(reinterpret_cast(row_data[row][0]) == icol[row + row_offset]); EXPECT(reinterpret_cast(row_data[row][1]) == bcol[row + row_offset]); EXPECT(::strncmp("abcdefghhgfedcbazzzz", reinterpret_cast(&row_data[row][2]), 3 * sizeof(double)) == 0); EXPECT(row_data[row][5] == dcol[row + row_offset]); EXPECT(row_data[row][6] == rcol[row + row_offset]); } } EXPECT(odc_next_frame(frame) == ODC_ITERATION_COMPLETE); } // ------------------------------------------------------------------------------------------------------ CASE("Encode to a file descriptor") { // Do some trivial encoding odc_integer_behaviour(ODC_INTEGERS_AS_LONGS); const int nrows = 10; long icol[nrows] = {1111, 2222, 3333, 4444, 5555, 6666, 7777, 8888, 9999, 0}; // Configure the encoder to encode said data odc_encoder_t* enc = nullptr; CHECK_RETURN(odc_new_encoder(&enc)); std::unique_ptr enc_deleter(enc); CHECK_RETURN(odc_encoder_set_row_count(enc, nrows)); CHECK_RETURN(odc_encoder_add_column(enc, "col1", ODC_INTEGER)); CHECK_RETURN(odc_encoder_column_set_data_array(enc, 0, 0, 0, icol)); // Do the encoding eckit::TmpFile tf; int fd = ::open(tf.asString().c_str(), O_CREAT | O_WRONLY, 0666); ASSERT(fd != -1); long sz; try { CHECK_RETURN(odc_encode_to_file_descriptor(enc, fd, &sz)); } catch (...) { ::close(fd); throw; } EXPECT(sz > 0); EXPECT(sz == tf.size()); ::close(fd); // Check that we have encoded what we think we have encoded odc_reader_t* reader = nullptr; CHECK_RETURN(odc_open_path(&reader, tf.asString().c_str())); EXPECT(reader); std::unique_ptr reader_deleter(reader); odc_frame_t* frame = nullptr; CHECK_RETURN(odc_new_frame(&frame, reader)); EXPECT(frame); std::unique_ptr frame_deleter(frame); CHECK_RETURN(odc_next_frame(frame)); int column_count; CHECK_RETURN(odc_frame_column_count(frame, &column_count)); EXPECT(column_count == 1); long row_count; CHECK_RETURN(odc_frame_row_count(frame, &row_count)); EXPECT(row_count == nrows); // Test that the data is correctly encoded odc_decoder_t* decoder; CHECK_RETURN(odc_new_decoder(&decoder)); EXPECT(decoder); std::unique_ptr decoder_deleter(decoder); CHECK_RETURN(odc_decoder_defaults_from_frame(decoder, frame)); long rows_decoded; CHECK_RETURN(odc_decode(decoder, frame, &rows_decoded)); EXPECT(rows_decoded == nrows); const void* pdata; CHECK_RETURN(odc_decoder_data_array(decoder, &pdata, 0, 0, 0)); EXPECT(::memcmp(icol, pdata, sizeof(icol)) == 0); EXPECT(odc_next_frame(frame) == ODC_ITERATION_COMPLETE); } // ------------------------------------------------------------------------------------------------------ struct custom_buffer_t { char* data; size_t pos; size_t size; }; long custom_buffer_write(void* h, const void* buffer, long length) { auto handle = reinterpret_cast(h); ASSERT(handle); ASSERT(length + handle->pos <= handle->size); ::memcpy(handle->data + handle->pos, buffer, length); handle->pos += length; return length; } CASE("Encode to a custom output stream") { // Do some trivial encoding odc_integer_behaviour(ODC_INTEGERS_AS_LONGS); const int nrows = 10; long icol[nrows] = {1111, 2222, 3333, 4444, 5555, 6666, 7777, 8888, 9999, 0}; // Configure the encoder to encode said data odc_encoder_t* enc = nullptr; CHECK_RETURN(odc_new_encoder(&enc)); std::unique_ptr enc_deleter(enc); CHECK_RETURN(odc_encoder_set_row_count(enc, nrows)); CHECK_RETURN(odc_encoder_add_column(enc, "col1", ODC_INTEGER)); CHECK_RETURN(odc_encoder_column_set_data_array(enc, 0, 0, 0, icol)); // Do the encoding eckit::Buffer encoded(1024 * 1024); custom_buffer_t handle = {(char*)encoded, 0, encoded.size()}; long sz; CHECK_RETURN(odc_encode_to_stream(enc, &handle, &custom_buffer_write, &sz)); EXPECT(sz > 0); EXPECT(size_t(sz) == handle.pos); // Check that we have encoded what we think we have encoded odc_reader_t* reader = nullptr; CHECK_RETURN(odc_open_buffer(&reader, encoded.data(), sz)); EXPECT(reader); std::unique_ptr reader_deleter(reader); odc_frame_t* frame = nullptr; CHECK_RETURN(odc_new_frame(&frame, reader)); EXPECT(frame); std::unique_ptr frame_deleter(frame); CHECK_RETURN(odc_next_frame(frame)); int column_count; CHECK_RETURN(odc_frame_column_count(frame, &column_count)); EXPECT(column_count == 1); long row_count; CHECK_RETURN(odc_frame_row_count(frame, &row_count)); EXPECT(row_count == nrows); // Test that the data is correctly encoded odc_decoder_t* decoder; CHECK_RETURN(odc_new_decoder(&decoder)); EXPECT(decoder); std::unique_ptr decoder_deleter(decoder); CHECK_RETURN(odc_decoder_defaults_from_frame(decoder, frame)); long rows_decoded; CHECK_RETURN(odc_decode(decoder, frame, &rows_decoded)); EXPECT(rows_decoded == nrows); const void* pdata; CHECK_RETURN(odc_decoder_data_array(decoder, &pdata, 0, 0, 0)); EXPECT(::memcmp(icol, pdata, sizeof(icol)) == 0); EXPECT(odc_next_frame(frame) == ODC_ITERATION_COMPLETE); } // ------------------------------------------------------------------------------------------------------ int main(int argc, char* argv[]) { return run_tests(argc, argv); } odc-1.6.3/tests/c_api/odc_encode_row_major.c0000664000175000017500000001520215146027420021175 0ustar alastairalastair/** * To build this program, please make sure to reference linked library: * * gcc -lodccore -o odc-c-encode-row-major odc_encode_row_major.c */ #include #include #include #include #include #include #include "odc/api/odc.h" // Bitfield constants #define Ob00000001 1 #define Ob00001011 11 #define Ob01101011 107 #define CHECK_RESULT(x) \ do { \ int rc = (x); \ if (rc != ODC_SUCCESS) { \ fprintf(stderr, "Error calling odc function \"%s\": %s\n", #x, odc_error_string(rc)); \ exit(1); \ } \ } while (false); void usage() { fprintf(stderr, "Usage:\n"); fprintf(stderr, " odc-c-encode-row-major \n\n"); } void cycle_longs(long* list, int size, long* pool, int pool_size) { int index = 0; int i; for (i = 0; i < size; i++) { if (index == pool_size) index = 0; list[i] = pool[index]; index++; } } void cycle_doubles(double* list, int size, double* pool, int pool_size) { int index = 0; int i; for (i = 0; i < size; i++) { if (index == pool_size) index = 0; list[i] = pool[index]; index++; } } void create_scratch_data(long nrows, int ncols, double data[nrows][ncols]) { // Prepare the current date as an integer time_t rawtime; time(&rawtime); struct tm* timeinfo; timeinfo = localtime(&rawtime); int64_t date = 10000 * (timeinfo->tm_year + 1900) + 100 * (timeinfo->tm_mon + 1) + timeinfo->tm_mday; // Prepare the list of integer values, including the missing value long missing_integer; CHECK_RESULT(odc_missing_integer(&missing_integer)); long integer_pool[] = {1234, 4321, missing_integer}; int integer_pool_size = sizeof(integer_pool) / sizeof(integer_pool[0]); long missing_integers[nrows]; cycle_longs(missing_integers, nrows, integer_pool, integer_pool_size); // Prepare the list of double values, including the missing value double missing_double; CHECK_RESULT(odc_missing_double(&missing_double)); double double_pool[] = {12.34, 43.21, missing_double}; int double_pool_size = sizeof(double_pool) / sizeof(double_pool[0]); double missing_doubles[nrows]; cycle_doubles(missing_doubles, nrows, double_pool, double_pool_size); // Prepare the list of bitfield values long bitfield_pool[] = {Ob00000001, Ob00001011, Ob01101011}; int bitfield_pool_size = sizeof(bitfield_pool) / sizeof(bitfield_pool[0]); long bitfield_values[nrows]; cycle_longs(bitfield_values, nrows, bitfield_pool, bitfield_pool_size); int i; // Fill in the passed data array with scratch values for (i = 0; i < nrows; i++) { memset(&data[i][0], 0, 5); snprintf((char*)(&data[i][0]), 5, "xxxx"); // expver *(int64_t*)(&data[i][1]) = date; // date@hdr memset(&data[i][2], 0, 8); snprintf((char*)(&data[i][2]), 7, "stat%02d", i); // statid@hdr memset(&data[i][3], 0, 16); snprintf((char*)(&data[i][3]), 16, "0-12345-0-678%02d", i); // wigos@hdr *(double*)(&data[i][5]) = 12.3456 * i; // obsvalue@body *(long*)(&data[i][6]) = missing_integers[i]; // integer_missing *(double*)(&data[i][7]) = missing_doubles[i]; // double_missing *(long*)(&data[i][8]) = bitfield_values[i]; // bitfield_column } } int main(int argc, char* argv[]) { if (argc != 2) { usage(); return 1; } // Get output path from command argument char* path = argv[1]; // Initialise API and set treatment of integers as longs CHECK_RESULT(odc_initialise_api()); CHECK_RESULT(odc_integer_behaviour(ODC_INTEGERS_AS_LONGS)); // Define row and column count const long nrows = 20; const int ncols = 9; // Allocate data array of rows x columns double data[nrows][ncols]; // Set up the allocated array with scratch data create_scratch_data(nrows, ncols, data); // Initialise encoder odc_encoder_t* encoder = NULL; CHECK_RESULT(odc_new_encoder(&encoder)); // Define all column names and their types CHECK_RESULT(odc_encoder_add_column(encoder, "expver", ODC_STRING)); CHECK_RESULT(odc_encoder_add_column(encoder, "date@hdr", ODC_INTEGER)); CHECK_RESULT(odc_encoder_add_column(encoder, "statid@hdr", ODC_STRING)); CHECK_RESULT(odc_encoder_add_column(encoder, "wigos@hdr", ODC_STRING)); CHECK_RESULT(odc_encoder_add_column(encoder, "obsvalue@body", ODC_REAL)); CHECK_RESULT(odc_encoder_add_column(encoder, "integer_missing", ODC_INTEGER)); CHECK_RESULT(odc_encoder_add_column(encoder, "double_missing", ODC_REAL)); CHECK_RESULT(odc_encoder_add_column(encoder, "bitfield_column", ODC_BITFIELD)); // Column `wigos@hdr` is a 16-byte string column, hence takes 2 columns in the array => ncols=9 CHECK_RESULT(odc_encoder_column_set_data_size(encoder, 3, 16)); // Column `bitfield_column` is an integer with 4 bitfield values in it CHECK_RESULT(odc_encoder_column_add_bitfield(encoder, 7, "flag_a", 1)); CHECK_RESULT(odc_encoder_column_add_bitfield(encoder, 7, "flag_b", 2)); CHECK_RESULT(odc_encoder_column_add_bitfield(encoder, 7, "flag_c", 3)); CHECK_RESULT(odc_encoder_column_add_bitfield(encoder, 7, "flag_d", 1)); // Set input data array from which data will be encoded // Zero value for last `columnMajorWidth` argument indicates the row-major layout CHECK_RESULT(odc_encoder_set_data_array(encoder, data, ncols * sizeof(double), nrows, 0)); const char* property_key = "encoded_by"; const char* property_value = "odc_example"; // Add some key/value metadata to the frame CHECK_RESULT(odc_encoder_add_property(encoder, property_key, property_value)); int file_descriptor = open(path, O_CREAT | O_TRUNC | O_WRONLY, 0666); long size; // Encode ODB-2 into an already open file descriptor CHECK_RESULT(odc_encode_to_file_descriptor(encoder, file_descriptor, &size)); close(file_descriptor); // Deallocate memory used up by the encoder CHECK_RESULT(odc_free_encoder(encoder)); fprintf(stdout, "Written %ld rows to %s\n", nrows, path); return 0; } odc-1.6.3/tests/c_api/read.cc0000664000175000017500000006647015146027420016127 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include "eckit/io/FileHandle.h" #include "eckit/testing/Test.h" #include "odc/api/Odb.h" #include "odc/api/odc.h" using namespace eckit::testing; // Specialise custom deletion for odb_t #define CHECK_RETURN(x) EXPECT((x) == ODC_SUCCESS) namespace std { template <> struct default_delete { void operator()(const odc_reader_t* reader) { CHECK_RETURN(odc_close(reader)); } }; template <> struct default_delete { void operator()(const odc_frame_t* frame) { CHECK_RETURN(odc_free_frame(frame)); } }; template <> struct default_delete { void operator()(odc_decoder_t* t) { CHECK_RETURN(odc_free_decoder(t)); } }; } // namespace std // ------------------------------------------------------------------------------------------------------ bool test_check_file_exists(std::string file_path) { std::ifstream file_stream(file_path); return file_stream.good(); } void test_generate_odb(const std::string& path, int propertiesMode) { odc::api::Settings::treatIntegersAsDoubles(false); // Define row count const size_t nrows = 10; // Allocate data array for each column char data0[nrows][8]; int64_t data1[nrows]; double data2[nrows]; size_t i; // Set up the allocated arrays with scratch data for (i = 0; i < nrows; i++) { snprintf(data0[i], 8, "xxxx"); // expver data1[i] = 20210527; // date@hdr data2[i] = 12.3456 * i; // obsvalue@body } // Define all column names, their types and sizes std::vector columns = { {std::string("expver"), odc::api::ColumnType(odc::api::STRING), 8}, {std::string("date@hdr"), odc::api::ColumnType(odc::api::INTEGER), sizeof(int64_t)}, {std::string("obsvalue@body"), odc::api::ColumnType(odc::api::REAL), sizeof(double)}, }; // Set a custom data layout and data array for each column std::vector strides{ // ptr, nrows, element_size, stride {data0, nrows, 8, 8}, {data1, nrows, sizeof(int64_t), sizeof(int64_t)}, {data2, nrows, sizeof(double), sizeof(double)}, }; std::map properties = {}; const eckit::Length length; eckit::FileHandle fh(path); fh.openForWrite(length); eckit::AutoClose closer(fh); // Encode two ODB-2 frames with the same data for (i = 0; i < 2; i++) { // Encode additional properties depending on the current mode switch (propertiesMode) { // Where the properties in the two frames are distinct case 1: { if (i == 0) properties["foo"] = "bar"; else { properties = {}; // reset properties["baz"] = "qux"; } break; } // Where the properties in the two frames overlap with entries that are the same case 2: { properties["foo"] = "bar"; properties["baz"] = "qux"; break; } // Where the properties overlap with entries whose keys are the same, but the values different case 3: { if (i == 0) properties["foo"] = "bar"; else properties["foo"] = "baz"; break; } } // Encode the ODB-2 frame into a data handle encode(fh, columns, strides, properties); } ASSERT(test_check_file_exists(path)); } // ------------------------------------------------------------------------------------------------------ CASE("Count lines in an existing ODB file") { odc_reader_t* reader = nullptr; CHECK_RETURN(odc_open_path(&reader, "../2000010106-reduced.odb")); std::unique_ptr reader_deleter(reader); odc_frame_t* frame = nullptr; CHECK_RETURN(odc_new_frame(&frame, reader)); std::unique_ptr frame_deleter(frame); size_t ntables = 0; size_t totalRows = 0; int ierr; while ((ierr = odc_next_frame(frame)) == ODC_SUCCESS) { long nrows; CHECK_RETURN(odc_frame_row_count(frame, &nrows)); totalRows += nrows; int ncols; CHECK_RETURN(odc_frame_column_count(frame, &ncols)); EXPECT(ncols == 51); ++ntables; } EXPECT(ierr == ODC_ITERATION_COMPLETE); EXPECT(ntables == 5); EXPECT(totalRows == 50000); } CASE("Check column details in an existing ODB file") { char example_column_names[][24] = { "expver@desc", "andate@desc", "antime@desc", "seqno@hdr", "obstype@hdr", "obschar@hdr", "subtype@hdr", "date@hdr", "time@hdr", "rdbflag@hdr", "status@hdr", "event1@hdr", "blacklist@hdr", "sortbox@hdr", "sitedep@hdr", "statid@hdr", "ident@hdr", "lat@hdr", "lon@hdr", "stalt@hdr", "modoro@hdr", "trlat@hdr", "trlon@hdr", "instspec@hdr", "event2@hdr", "anemoht@hdr", "baroht@hdr", "sensor@hdr", "numlev@hdr", "varno_presence@hdr", "varno@body", "vertco_type@body", "rdbflag@body", "anflag@body", "status@body", "event1@body", "blacklist@body", "entryno@body", "press@body", "press_rl@body", "obsvalue@body", "aux1@body", "event2@body", "ppcode@body", "level@body", "biascorr@body", "final_obs_error@errstat", "obs_error@errstat", "repres_error@errstat", "pers_error@errstat", "fg_error@errstat", }; int example_column_types[] = { ODC_STRING, ODC_INTEGER, ODC_INTEGER, ODC_INTEGER, ODC_INTEGER, ODC_BITFIELD, ODC_INTEGER, ODC_INTEGER, ODC_INTEGER, ODC_BITFIELD, ODC_BITFIELD, ODC_BITFIELD, ODC_BITFIELD, ODC_INTEGER, ODC_INTEGER, ODC_STRING, ODC_INTEGER, ODC_REAL, ODC_REAL, ODC_REAL, ODC_REAL, ODC_REAL, ODC_REAL, ODC_INTEGER, ODC_INTEGER, ODC_REAL, ODC_REAL, ODC_INTEGER, ODC_INTEGER, ODC_BITFIELD, ODC_INTEGER, ODC_INTEGER, ODC_BITFIELD, ODC_BITFIELD, ODC_BITFIELD, ODC_BITFIELD, ODC_BITFIELD, ODC_INTEGER, ODC_REAL, ODC_REAL, ODC_REAL, ODC_REAL, ODC_INTEGER, ODC_INTEGER, ODC_BITFIELD, ODC_REAL, ODC_REAL, ODC_REAL, ODC_REAL, ODC_REAL, ODC_REAL, }; char column_10_bitfield_names[][15] = { "lat_humon", "lat_qcsub", "lat_override", "lat_flag", "lat_hqc_flag", "lon_humon", "lon_qcsub", "lon_override", "lon_flag", "lon_hqc_flag", "date_humon", "date_qcsub", "date_override", "date_flag", "date_hqc_flag", "time_humon", "time_qcsub", "time_override", "time_flag", "time_hqc_flag", "stalt_humon", "stalt_qcsub", "stalt_override", "stalt_flag", "stalt_hqc_flag", }; int column_10_bitfield_sizes[] = { 1, 1, 1, 2, 1, 1, 1, 1, 2, 1, 1, 1, 1, 2, 1, 1, 1, 1, 2, 1, 1, 1, 1, 2, 1, }; odc_reader_t* reader = nullptr; CHECK_RETURN(odc_open_path(&reader, "../2000010106-reduced.odb")); std::unique_ptr reader_deleter(reader); odc_frame_t* frame = nullptr; CHECK_RETURN(odc_new_frame(&frame, reader)); std::unique_ptr frame_deleter(frame); // Get the first frame CHECK_RETURN(odc_next_frame(frame)); int ncols; CHECK_RETURN(odc_frame_column_count(frame, &ncols)); EXPECT(ncols == 51); int col; for (col = 0; col < ncols; ++col) { const char* name; int type; int element_size; int bitfield_count; CHECK_RETURN(odc_frame_column_attributes(frame, col, &name, &type, &element_size, &bitfield_count)); EXPECT(strcmp(name, example_column_names[col]) == 0); EXPECT(type == example_column_types[col]); EXPECT(element_size == 8); if (type == ODC_BITFIELD) { EXPECT(bitfield_count > 0); } else { EXPECT(bitfield_count == 0); } // Test bitfields for column 10 if (col == 9) { EXPECT(bitfield_count == 25); int bf; int expected_offset = 0; for (bf = 0; bf < bitfield_count; ++bf) { const char* bf_name; int bf_offset; int bf_size; CHECK_RETURN(odc_frame_bitfield_attributes(frame, col, bf, &bf_name, &bf_offset, &bf_size)); EXPECT(strcmp(bf_name, column_10_bitfield_names[bf]) == 0); EXPECT(bf_size == column_10_bitfield_sizes[bf]); EXPECT(bf_offset == expected_offset); expected_offset = expected_offset + bf_size; } } } } CASE("Decode data in an existing ODB file") { CHECK_RETURN(odc_integer_behaviour(ODC_INTEGERS_AS_LONGS)); odc_reader_t* reader = nullptr; CHECK_RETURN(odc_open_path(&reader, "../2000010106-reduced.odb")); std::unique_ptr reader_deleter(reader); odc_frame_t* frame = nullptr; CHECK_RETURN(odc_new_frame(&frame, reader)); std::unique_ptr frame_deleter(frame); size_t ntables = 0; size_t totalRows = 0; // Get the first frame CHECK_RETURN(odc_next_frame(frame)); // Read the second frame, because why not CHECK_RETURN(odc_next_frame(frame)); odc_decoder_t* decoder; CHECK_RETURN(odc_new_decoder(&decoder)); EXPECT(decoder); std::unique_ptr decoder_deleter(decoder); CHECK_RETURN(odc_decoder_defaults_from_frame(decoder, frame)); long nrows; CHECK_RETURN(odc_decode(decoder, frame, &nrows)); EXPECT(nrows == 10000); long nrows2; CHECK_RETURN(odc_decoder_row_count(decoder, &nrows2)); EXPECT(nrows2 == 10000); int ncols; CHECK_RETURN(odc_decoder_column_count(decoder, &ncols)); EXPECT(ncols == 51); const void* data; long row_stride; bool column_major; CHECK_RETURN(odc_decoder_data_array(decoder, &data, &row_stride, 0, &column_major)); EXPECT(!column_major); EXPECT(data != nullptr); EXPECT(row_stride == 51 * sizeof(double)); const int64_t expected_seqno[] = {(int64_t)6106691, (int64_t)6002665, (int64_t)6162889, (int64_t)6162885}; const int64_t expected_obschar[] = { (int64_t)537918674, (int64_t)135265490, (int64_t)605027538, (int64_t)605027538, }; const double expected_lat[] = { (double)0.370279, (double)0.369519, (double)0.367451, (double)0.360161, }; const int width = 8; long missing_integer; double missing_double; CHECK_RETURN(odc_missing_integer(&missing_integer)); CHECK_RETURN(odc_missing_double(&missing_double)); int i; int row; for (i = 0; i < 4; ++i) { row = i * 765; // expver@desc (ODC_STRING, col 1) char* expver = &((char*)data)[width * 0 + width * row]; expver[4] = '\0'; EXPECT(strcmp(expver, "0018") == 0); // seqno@hdr (ODC_INTEGER, col 4) EXPECT(*(const int64_t*)&((const char*)data)[width * 3 + width * row] == expected_seqno[i]); // obschar@hdr (ODC_BITFIELD, col 6) EXPECT(*(const int64_t*)&((const char*)data)[width * 5 + width * row] == expected_obschar[i]); // sortbox@hdr (ODC_INTEGER, col 14, missing value!) EXPECT(*(const int64_t*)&((const char*)data)[width * 13 + 8 * row] == missing_integer); // lat@hdr (ODC_REAL, col 18) EXPECT(eckit::types::is_approximately_equal(*(const double*)&((const char*)data)[width * 17 + 8 * row], expected_lat[i], 0.000001)); // repres_error@errstat (ODC_REAL, col 49, missing value!) EXPECT(*(const double*)&((const char*)data)[width * 48 + 8 * row] == missing_double); } } // ------------------------------------------------------------------------------------------------------ CASE("Where the properties in the two frames are distinct (non-aggregated)") { test_generate_odb("properties-1.odb", 1); odc_reader_t* reader = nullptr; CHECK_RETURN(odc_open_path(&reader, "properties-1.odb")); std::unique_ptr reader_deleter(reader); odc_frame_t* frame = nullptr; CHECK_RETURN(odc_new_frame(&frame, reader)); std::unique_ptr frame_deleter(frame); int nproperties; const char* key; const char* value; const char* version; CHECK_RETURN(odc_version(&version)); char odc_version_str[255] = "odc version "; strcat(odc_version_str, version); int nframes = 0; int rc; while ((rc = odc_next_frame(frame)) == ODC_SUCCESS) { CHECK_RETURN(odc_frame_properties_count(frame, &nproperties)); EXPECT(nproperties == 2); int i; for (i = 0; i < 2; i++) { CHECK_RETURN(odc_frame_property_idx(frame, i, &key, &value)); if ((i == 0 && nframes == 0) || (i == 1 && nframes == 1)) { EXPECT(strcmp(key, "encoder") == 0); EXPECT(strcmp(value, odc_version_str) == 0); } else if (nframes == 0) { EXPECT(strcmp(key, "foo") == 0); EXPECT(strcmp(value, "bar") == 0); } else { EXPECT(strcmp(key, "baz") == 0); EXPECT(strcmp(value, "qux") == 0); } } if (nframes == 0) { CHECK_RETURN(odc_frame_property(frame, "encoder", &value)); EXPECT(strcmp(value, odc_version_str) == 0); CHECK_RETURN(odc_frame_property(frame, "foo", &value)); EXPECT(strcmp(value, "bar") == 0); } else { CHECK_RETURN(odc_frame_property(frame, "encoder", &value)); EXPECT(strcmp(value, odc_version_str) == 0); CHECK_RETURN(odc_frame_property(frame, "baz", &value)); EXPECT(strcmp(value, "qux") == 0); } // Check for reading of non-existent properties CHECK_RETURN(odc_frame_property(frame, "non-existent", &value)); EXPECT(value == NULL); ++nframes; } EXPECT(rc == ODC_ITERATION_COMPLETE); EXPECT(nframes == 2); } // ------------------------------------------------------------------------------------------------------ CASE("Where the properties in the two frames are distinct (aggregated)") { ASSERT(test_check_file_exists("properties-1.odb")); odc_reader_t* reader = nullptr; CHECK_RETURN(odc_open_path(&reader, "properties-1.odb")); std::unique_ptr reader_deleter(reader); odc_frame_t* frame = nullptr; CHECK_RETURN(odc_new_frame(&frame, reader)); std::unique_ptr frame_deleter(frame); int nproperties; const char* key; const char* value; const char* version; CHECK_RETURN(odc_version(&version)); char odc_version_str[255] = "odc version "; strcat(odc_version_str, version); long max_aggregated_rows = 1000000; int nframes = 0; int rc; while ((rc = odc_next_frame_aggregated(frame, max_aggregated_rows)) == ODC_SUCCESS) { CHECK_RETURN(odc_frame_properties_count(frame, &nproperties)); EXPECT(nproperties == 3); int i; for (i = 0; i < 3; i++) { CHECK_RETURN(odc_frame_property_idx(frame, i, &key, &value)); if (i == 0) { EXPECT(strcmp(key, "baz") == 0); EXPECT(strcmp(value, "qux") == 0); } else if (i == 1) { EXPECT(strcmp(key, "encoder") == 0); EXPECT(strcmp(value, odc_version_str) == 0); } else { EXPECT(strcmp(key, "foo") == 0); EXPECT(strcmp(value, "bar") == 0); } } CHECK_RETURN(odc_frame_property(frame, "encoder", &value)); EXPECT(strcmp(value, odc_version_str) == 0); CHECK_RETURN(odc_frame_property(frame, "foo", &value)); EXPECT(strcmp(value, "bar") == 0); CHECK_RETURN(odc_frame_property(frame, "baz", &value)); EXPECT(strcmp(value, "qux") == 0); // Check for reading of non-existent properties CHECK_RETURN(odc_frame_property(frame, "non-existent", &value)); EXPECT(value == NULL); ++nframes; } EXPECT(rc == ODC_ITERATION_COMPLETE); EXPECT(nframes == 1); } // ------------------------------------------------------------------------------------------------------ CASE("Where the properties in the two frames overlap with entries that are the same (non-aggregated)") { test_generate_odb("properties-2.odb", 2); odc_reader_t* reader = nullptr; CHECK_RETURN(odc_open_path(&reader, "properties-2.odb")); std::unique_ptr reader_deleter(reader); odc_frame_t* frame = nullptr; CHECK_RETURN(odc_new_frame(&frame, reader)); std::unique_ptr frame_deleter(frame); int nproperties; const char* key; const char* value; const char* version; CHECK_RETURN(odc_version(&version)); char odc_version_str[255] = "odc version "; strcat(odc_version_str, version); int nframes = 0; int rc; while ((rc = odc_next_frame(frame)) == ODC_SUCCESS) { CHECK_RETURN(odc_frame_properties_count(frame, &nproperties)); EXPECT(nproperties == 3); int i; for (i = 0; i < 3; i++) { CHECK_RETURN(odc_frame_property_idx(frame, i, &key, &value)); if (i == 0) { EXPECT(strcmp(key, "baz") == 0); EXPECT(strcmp(value, "qux") == 0); } else if (i == 1) { EXPECT(strcmp(key, "encoder") == 0); EXPECT(strcmp(value, odc_version_str) == 0); } else { EXPECT(strcmp(key, "foo") == 0); EXPECT(strcmp(value, "bar") == 0); } } CHECK_RETURN(odc_frame_property(frame, "encoder", &value)); EXPECT(strcmp(value, odc_version_str) == 0); CHECK_RETURN(odc_frame_property(frame, "foo", &value)); EXPECT(strcmp(value, "bar") == 0); CHECK_RETURN(odc_frame_property(frame, "baz", &value)); EXPECT(strcmp(value, "qux") == 0); // Check for reading of non-existent properties CHECK_RETURN(odc_frame_property(frame, "non-existent", &value)); EXPECT(value == NULL); ++nframes; } EXPECT(rc == ODC_ITERATION_COMPLETE); EXPECT(nframes == 2); } // ------------------------------------------------------------------------------------------------------ CASE("Where the properties in the two frames overlap with entries that are the same (aggregated)") { ASSERT(test_check_file_exists("properties-2.odb")); odc_reader_t* reader = nullptr; CHECK_RETURN(odc_open_path(&reader, "properties-2.odb")); std::unique_ptr reader_deleter(reader); odc_frame_t* frame = nullptr; CHECK_RETURN(odc_new_frame(&frame, reader)); std::unique_ptr frame_deleter(frame); int nproperties; const char* key; const char* value; const char* version; CHECK_RETURN(odc_version(&version)); char odc_version_str[255] = "odc version "; strcat(odc_version_str, version); long max_aggregated_rows = 1000000; int nframes = 0; int rc; while ((rc = odc_next_frame_aggregated(frame, max_aggregated_rows)) == ODC_SUCCESS) { CHECK_RETURN(odc_frame_properties_count(frame, &nproperties)); EXPECT(nproperties == 3); int i; for (i = 0; i < 3; i++) { CHECK_RETURN(odc_frame_property_idx(frame, i, &key, &value)); if (i == 0) { EXPECT(strcmp(key, "baz") == 0); EXPECT(strcmp(value, "qux") == 0); } else if (i == 1) { EXPECT(strcmp(key, "encoder") == 0); EXPECT(strcmp(value, odc_version_str) == 0); } else { EXPECT(strcmp(key, "foo") == 0); EXPECT(strcmp(value, "bar") == 0); } } CHECK_RETURN(odc_frame_property(frame, "encoder", &value)); EXPECT(strcmp(value, odc_version_str) == 0); CHECK_RETURN(odc_frame_property(frame, "foo", &value)); EXPECT(strcmp(value, "bar") == 0); CHECK_RETURN(odc_frame_property(frame, "baz", &value)); EXPECT(strcmp(value, "qux") == 0); // Check for reading of non-existent properties CHECK_RETURN(odc_frame_property(frame, "non-existent", &value)); EXPECT(value == NULL); ++nframes; } EXPECT(rc == ODC_ITERATION_COMPLETE); EXPECT(nframes == 1); } // ------------------------------------------------------------------------------------------------------ CASE("Where the properties overlap with entries whose keys are the same, but the values different (non-aggregated)") { test_generate_odb("properties-3.odb", 3); odc_reader_t* reader = nullptr; CHECK_RETURN(odc_open_path(&reader, "properties-3.odb")); std::unique_ptr reader_deleter(reader); odc_frame_t* frame = nullptr; CHECK_RETURN(odc_new_frame(&frame, reader)); std::unique_ptr frame_deleter(frame); int nproperties; const char* key; const char* value; const char* version; CHECK_RETURN(odc_version(&version)); char odc_version_str[255] = "odc version "; strcat(odc_version_str, version); int nframes = 0; int rc; while ((rc = odc_next_frame(frame)) == ODC_SUCCESS) { CHECK_RETURN(odc_frame_properties_count(frame, &nproperties)); EXPECT(nproperties == 2); int i; for (i = 0; i < 2; i++) { CHECK_RETURN(odc_frame_property_idx(frame, i, &key, &value)); if (i == 0) { EXPECT(strcmp(key, "encoder") == 0); EXPECT(strcmp(value, odc_version_str) == 0); } else if (nframes == 0) { EXPECT(strcmp(key, "foo") == 0); EXPECT(strcmp(value, "bar") == 0); } else { EXPECT(strcmp(key, "foo") == 0); EXPECT(strcmp(value, "baz") == 0); } } if (nframes == 0) { CHECK_RETURN(odc_frame_property(frame, "encoder", &value)); EXPECT(strcmp(value, odc_version_str) == 0); CHECK_RETURN(odc_frame_property(frame, "foo", &value)); EXPECT(strcmp(value, "bar") == 0); } else { CHECK_RETURN(odc_frame_property(frame, "encoder", &value)); EXPECT(strcmp(value, odc_version_str) == 0); CHECK_RETURN(odc_frame_property(frame, "foo", &value)); EXPECT(strcmp(value, "baz") == 0); } // Check for reading of non-existent properties CHECK_RETURN(odc_frame_property(frame, "non-existent", &value)); EXPECT(value == NULL); ++nframes; } EXPECT(rc == ODC_ITERATION_COMPLETE); EXPECT(nframes == 2); } // ------------------------------------------------------------------------------------------------------ CASE("Where the properties overlap with entries whose keys are the same, but the values different (aggregated)") { ASSERT(test_check_file_exists("properties-3.odb")); odc_reader_t* reader = nullptr; CHECK_RETURN(odc_open_path(&reader, "properties-3.odb")); std::unique_ptr reader_deleter(reader); odc_frame_t* frame = nullptr; CHECK_RETURN(odc_new_frame(&frame, reader)); std::unique_ptr frame_deleter(frame); int nproperties; const char* key; const char* value; const char* version; CHECK_RETURN(odc_version(&version)); char odc_version_str[255] = "odc version "; strcat(odc_version_str, version); long max_aggregated_rows = 1000000; int nframes = 0; int rc; while ((rc = odc_next_frame_aggregated(frame, max_aggregated_rows)) == ODC_SUCCESS) { CHECK_RETURN(odc_frame_properties_count(frame, &nproperties)); EXPECT(nproperties == 2); int i; for (i = 0; i < 2; i++) { CHECK_RETURN(odc_frame_property_idx(frame, i, &key, &value)); if (i == 0) { EXPECT(strcmp(key, "encoder") == 0); EXPECT(strcmp(value, odc_version_str) == 0); } else { EXPECT(strcmp(key, "foo") == 0); EXPECT(strcmp(value, "bar") == 0); // value from the first frame will win! } } CHECK_RETURN(odc_frame_property(frame, "encoder", &value)); EXPECT(strcmp(value, odc_version_str) == 0); CHECK_RETURN(odc_frame_property(frame, "foo", &value)); EXPECT(strcmp(value, "bar") == 0); // value from the first frame will win! // Check for reading of non-existent properties CHECK_RETURN(odc_frame_property(frame, "non-existent", &value)); EXPECT(value == NULL); ++nframes; } EXPECT(rc == ODC_ITERATION_COMPLETE); EXPECT(nframes == 1); } // ------------------------------------------------------------------------------------------------------ // CASE("Decode an entire ODB file") { // // // std::unique_ptr o(odc_open_path("../2000010106-reduced.odb")); // // size_t ntables = 0; // // std::unique_ptr table; // while (table.reset(odc_alloc_next_frame(o.get())), table) { // // std::unique_ptr decoded(odc_frame_decode_all(table.get())); // EXPECT(decoded->nrows == odc_frame_row_count(table.get())); // EXPECT(decoded->ncolumns == 51); // // ++ntables; // } // } // //// ------------------------------------------------------------------------------------------------------ // // CASE("Decode an entire ODB file preallocated data structures") { // // std::unique_ptr o(odc_open_path("../2000010106-reduced.odb")); // // int ntables = odc_num_frames(o.get()); // EXPECT(ntables == 5); // // odb_decoded_t decoded; // odb_strided_data_t strided_data[51]; // // for (int i = 0; i < ntables; i++) { // // std::unique_ptr table(odc_get_frame(o.get(), i)); // // ASSERT(odc_frame_column_count(table.get()) == 51); // // decoded.ncolumns = 51; // decoded.nrows = 10000; // decoded.columnData = strided_data; // // /// odc_frame_decode(table.get(), &decoded); // // ///EXPECT(decoded.nrows == odc_frame_row_count(table.get())); // ///EXPECT(decoded.ncolumns == 51); // // ///eckit::Log::info() << "Decoded: ncolumns = " << decoded.ncolumns << std::endl; // ///eckit::Log::info() << "Decoded: nrows = " << decoded.nrows << std::endl; // ///eckit::Log::info() << "Decoded: data = " << decoded.columnData << std::endl; // } // } // //// ------------------------------------------------------------------------------------------------------ int main(int argc, char* argv[]) { return run_tests(argc, argv); } odc-1.6.3/tests/c_api/CMakeLists.txt0000664000175000017500000000242615146027420017434 0ustar alastairalastair list( APPEND _capi_odc_tests read encode ) foreach( _test ${_capi_odc_tests} ) ecbuild_add_test( TARGET odc_capi_${_test} SOURCES ${_test}.cc ENVIRONMENT ${test_environment} TEST_DEPENDS odc_get_test_data LIBS eckit odccore ) endforeach() list( APPEND _capi_odc_example_sources odc_header odc_encode_custom odc_encode_row_major odc_ls ) list( APPEND _capi_odc_example_targets odc-c-header odc-c-encode-custom odc-c-encode-row-major odc-c-ls ) list( LENGTH _capi_odc_example_sources _count ) math( EXPR _count "${_count}-1" ) foreach( _i RANGE ${_count} ) list( GET _capi_odc_example_sources ${_i} _sources ) list( GET _capi_odc_example_targets ${_i} _target ) ecbuild_add_executable( TARGET ${_target} SOURCES ${_sources}.c LIBS odccore NOINSTALL ) endforeach() list( APPEND _capi_odc_tests_scripts usage_examples.sh ) foreach( _script ${_capi_odc_tests_scripts} ) ecbuild_add_test( TARGET odc_capi_${_script} TYPE script COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/${_script} ENVIRONMENT PATH=${CMAKE_BINARY_DIR}/bin:$ENV{PATH} ) endforeach() odc-1.6.3/tests/c_api/odc_encode_custom.c0000664000175000017500000001671615146027420020523 0ustar alastairalastair/** * To build this program, please make sure to reference linked library: * * gcc -lodccore -o odc-c-encode-custom odc_encode_custom.c */ #include #include #include #include #include #include #include "odc/api/odc.h" // Bitfield constants #define Ob00000001 1 #define Ob00001011 11 #define Ob01101011 107 #define CHECK_RESULT(x) \ do { \ int rc = (x); \ if (rc != ODC_SUCCESS) { \ fprintf(stderr, "Error calling odc function \"%s\": %s\n", #x, odc_error_string(rc)); \ exit(1); \ } \ } while (false); void usage() { fprintf(stderr, "Usage:\n"); fprintf(stderr, " odc-c-encode-custom \n\n"); } void cycle_longs(long* list, int size, long* pool, int pool_size) { int index = 0; int i; for (i = 0; i < size; i++) { if (index == pool_size) index = 0; list[i] = pool[index]; index++; } } void cycle_doubles(double* list, int size, double* pool, int pool_size) { int index = 0; int i; for (i = 0; i < size; i++) { if (index == pool_size) index = 0; list[i] = pool[index]; index++; } } void create_scratch_data(long nrows, char data0[][8], int64_t data1[], char data2[][8], char data3[][16], double data4[], int64_t data5[], double data6[], int64_t data7[]) { // Prepare the current date as an integer time_t rawtime; time(&rawtime); struct tm* timeinfo; timeinfo = localtime(&rawtime); int64_t date = 10000 * (timeinfo->tm_year + 1900) + 100 * (timeinfo->tm_mon + 1) + timeinfo->tm_mday; // Prepare the list of integer values, including the missing value long missing_integer; CHECK_RESULT(odc_missing_integer(&missing_integer)); long integer_pool[] = {1234, 4321, missing_integer}; int integer_pool_size = sizeof(integer_pool) / sizeof(integer_pool[0]); long missing_integers[nrows]; cycle_longs(missing_integers, nrows, integer_pool, integer_pool_size); // Prepare the list of double values, including the missing value double missing_double; CHECK_RESULT(odc_missing_double(&missing_double)); double double_pool[] = {12.34, 43.21, missing_double}; int double_pool_size = sizeof(double_pool) / sizeof(double_pool[0]); double missing_doubles[nrows]; cycle_doubles(missing_doubles, nrows, double_pool, double_pool_size); // Prepare the list of bitfield values long bitfield_pool[] = {Ob00000001, Ob00001011, Ob01101011}; int bitfield_pool_size = sizeof(bitfield_pool) / sizeof(bitfield_pool[0]); long bitfield_values[nrows]; cycle_longs(bitfield_values, nrows, bitfield_pool, bitfield_pool_size); int i; // Fill in the passed data arrays with scratch values for (i = 0; i < nrows; i++) { snprintf(data0[i], 8, "xxxx"); // expver data1[i] = date; // date@hdr snprintf(data2[i], 7, "stat%02d", i); // statid@hdr snprintf(data3[i], 16, "0-12345-0-678%02d", i); // wigos@hdr data4[i] = 12.3456 * i; // obsvalue@body data5[i] = missing_integers[i]; // integer_missing data6[i] = missing_doubles[i]; // double_missing data7[i] = bitfield_values[i]; // bitfield_column } } int main(int argc, char* argv[]) { if (argc != 2) { usage(); return 1; } // Get output path from command argument char* path = argv[1]; // Initialise API and set treatment of integers as longs CHECK_RESULT(odc_initialise_api()); CHECK_RESULT(odc_integer_behaviour(ODC_INTEGERS_AS_LONGS)); // Define row count const long nrows = 20; // Allocate data array for each column char data0[nrows][8]; int64_t data1[nrows]; char data2[nrows][8]; char data3[nrows][16]; double data4[nrows]; int64_t data5[nrows]; double data6[nrows]; int64_t data7[nrows]; // Set up the allocated array with scratch data create_scratch_data(nrows, data0, data1, data2, data3, data4, data5, data6, data7); // Initialise encoder odc_encoder_t* encoder = NULL; CHECK_RESULT(odc_new_encoder(&encoder)); // Set number of rows to allocate in the encoder CHECK_RESULT(odc_encoder_set_row_count(encoder, nrows)); // Define all column names and their types CHECK_RESULT(odc_encoder_add_column(encoder, "expver", ODC_STRING)); CHECK_RESULT(odc_encoder_add_column(encoder, "date@hdr", ODC_INTEGER)); CHECK_RESULT(odc_encoder_add_column(encoder, "statid@hdr", ODC_STRING)); CHECK_RESULT(odc_encoder_add_column(encoder, "wigos@hdr", ODC_STRING)); CHECK_RESULT(odc_encoder_add_column(encoder, "obsvalue@body", ODC_REAL)); CHECK_RESULT(odc_encoder_add_column(encoder, "integer_missing", ODC_INTEGER)); CHECK_RESULT(odc_encoder_add_column(encoder, "double_missing", ODC_REAL)); CHECK_RESULT(odc_encoder_add_column(encoder, "bitfield_column", ODC_BITFIELD)); // Column `wigos@hdr` is a 16-byte string column CHECK_RESULT(odc_encoder_column_set_data_size(encoder, 3, 16)); // Column `bitfield_column` is an integer with 4 bitfield values in it CHECK_RESULT(odc_encoder_column_add_bitfield(encoder, 7, "flag_a", 1)); CHECK_RESULT(odc_encoder_column_add_bitfield(encoder, 7, "flag_b", 2)); CHECK_RESULT(odc_encoder_column_add_bitfield(encoder, 7, "flag_c", 3)); CHECK_RESULT(odc_encoder_column_add_bitfield(encoder, 7, "flag_d", 1)); // Set a custom data layout and data array for each column CHECK_RESULT(odc_encoder_column_set_data_array(encoder, 0, 8, 8, data0)); CHECK_RESULT(odc_encoder_column_set_data_array(encoder, 1, sizeof(int64_t), sizeof(int64_t), data1)); CHECK_RESULT(odc_encoder_column_set_data_array(encoder, 2, 8, 8, data2)); CHECK_RESULT(odc_encoder_column_set_data_array(encoder, 3, 16, 16, data3)); CHECK_RESULT(odc_encoder_column_set_data_array(encoder, 4, sizeof(double), sizeof(double), data4)); CHECK_RESULT(odc_encoder_column_set_data_array(encoder, 5, sizeof(int64_t), sizeof(int64_t), data5)); CHECK_RESULT(odc_encoder_column_set_data_array(encoder, 6, sizeof(double), sizeof(double), data6)); CHECK_RESULT(odc_encoder_column_set_data_array(encoder, 7, sizeof(int64_t), sizeof(int64_t), data7)); const char* property_key = "encoded_by"; const char* property_value = "odc_example"; // Add some key/value metadata to the frame CHECK_RESULT(odc_encoder_add_property(encoder, property_key, property_value)); int file_descriptor = open(path, O_CREAT | O_TRUNC | O_WRONLY, 0666); long size; // Encode ODB-2 into an already open file descriptor CHECK_RESULT(odc_encode_to_file_descriptor(encoder, file_descriptor, &size)); close(file_descriptor); // Deallocate memory used up by the encoder CHECK_RESULT(odc_free_encoder(encoder)); fprintf(stdout, "Written %ld rows to %s\n", nrows, path); return 0; } odc-1.6.3/tests/c_api/odc_header.c0000664000175000017500000001057615146027420017122 0ustar alastairalastair/** * To build this program, please make sure to reference linked library: * * gcc -lodccore -o odc-c-header odc_header.c */ #include #include #include "odc/api/odc.h" #define CHECK_RESULT(x) \ do { \ int rc = (x); \ if (rc != ODC_SUCCESS) { \ fprintf(stderr, "Error calling odc function \"%s\": %s\n", #x, odc_error_string(rc)); \ exit(1); \ } \ } while (false); void usage() { fprintf(stderr, "Usage:\n"); fprintf(stderr, " odc-c-header [ ...]\n\n"); } int main(int argc, char* argv[]) { if (argc < 2) { usage(); return 1; } // Initialise API CHECK_RESULT(odc_initialise_api()); int argi; // Iterate over all supplied path arguments for (argi = 1; argi < argc; argi++) { char* path = argv[argi]; odc_reader_t* reader = NULL; odc_frame_t* frame = NULL; // Open current path and initialise frame CHECK_RESULT(odc_open_path(&reader, path)); CHECK_RESULT(odc_new_frame(&frame, reader)); fprintf(stdout, "File: %s\n", path); int i = 0; int rc; // Iterate over all frames in the stream in non-aggregated mode, without decoding them while ((rc = odc_next_frame(frame)) == ODC_SUCCESS) { long row_count; int column_count; // Get row and column counts CHECK_RESULT(odc_frame_row_count(frame, &row_count)); CHECK_RESULT(odc_frame_column_count(frame, &column_count)); fprintf(stdout, " Frame: %d, Row count: %ld, Column count: %d\n", ++i, row_count, column_count); int nproperties; // Get number of properties encoded in the frame CHECK_RESULT(odc_frame_properties_count(frame, &nproperties)); const char* key; const char* value; int i; for (i = 0; i < nproperties; i++) { // Get property key and value by its index CHECK_RESULT(odc_frame_property_idx(frame, i, &key, &value)); fprintf(stdout, " Property: %s => %s\n", key, value); } int col; // Iterate over frame columns for (col = 0; col < column_count; ++col) { const char* name; int type; int element_size; int bitfield_count; // Get column information CHECK_RESULT(odc_frame_column_attributes(frame, col, &name, &type, &element_size, &bitfield_count)); const char* type_name; // Lookup column type name CHECK_RESULT(odc_column_type_name(type, &type_name)); fprintf(stdout, " Column: %d, Name: %s, Type: %s, Size: %d\n", col + 1, name, type_name, element_size); // Process bitfields only if (type == ODC_BITFIELD) { int bf; for (bf = 0; bf < bitfield_count; ++bf) { const char* bf_name; int bf_offset; int bf_size; // Get bitfield information CHECK_RESULT(odc_frame_bitfield_attributes(frame, col, bf, &bf_name, &bf_offset, &bf_size)); fprintf(stdout, " Bitfield: %d, Name: %s, Offset: %d, Nbits: %d\n", bf + 1, bf_name, bf_offset, bf_size); } } } fprintf(stdout, "\n"); } // Unsuccessful end of frame iteration if (rc != ODC_ITERATION_COMPLETE) { fprintf(stderr, "Error: %s\n", odc_error_string(rc)); return 1; } CHECK_RESULT(odc_free_frame(frame)); // Deallocate reader instance CHECK_RESULT(odc_close(reader)); } return 0; } odc-1.6.3/tests/c_api/usage_examples.sh0000775000175000017500000001340315146027420020232 0ustar alastairalastair#!/bin/bash set -uex # A unique working directory wd=$(pwd) test_wd=$(pwd)/usage_examples mkdir -p ${test_wd} cd ${test_wd} # In case we are resuming from a previous failed run, which has left output in the directory rm *.odb *.txt || true # Helper functions expect_error () { exit_code="${exit_code-0}" [[ "$exit_code" -eq 1 ]] || (echo "Unexpected exit code: $exit_code != 1" ; false) unset exit_code } expect_success () { exit_code="${exit_code-1}" [[ "$exit_code" -eq 0 ]] || (echo "Unexpected exit code: $exit_code != 0" ; false) unset exit_code } # -- # Test C encode row-major example ${wd}/odc-c-encode-row-major 2> encode-row-major-usage.txt || exit_code=$? ; expect_error cat > encode-row-major-usage-expected.txt < EOF cmp encode-row-major-usage.txt encode-row-major-usage-expected.txt ${wd}/odc-c-encode-row-major ${test_wd}/test-1.odb > encode-row-major.txt && exit_code=$? ; expect_success cat > encode-row-major-expected.txt < encode-custom-usage.txt || exit_code=$? ; expect_error cat > encode-custom-usage-expected.txt < EOF cmp encode-custom-usage.txt encode-custom-usage-expected.txt ${wd}/odc-c-encode-custom ${test_wd}/test-2.odb > encode-custom.txt && exit_code=$? ; expect_success cat > encode-custom-expected.txt < ls-usage.txt || exit_code=$? ; expect_error cat > ls-usage-expected.txt < EOF cmp ls-usage.txt ls-usage-expected.txt ${wd}/odc-c-ls ${test_wd}/test-1.odb > ls.txt && exit_code=$? ; expect_success current_date=$(date +"%Y%m%d") cat > ls-expected.txt < header-usage.txt || exit_code=$? ; expect_error cat > header-usage-expected.txt < [ ...] EOF cmp header-usage.txt header-usage-expected.txt ${wd}/odc-c-header ${test_wd}/test-1.odb ${test_wd}/test-2.odb > header.txt && exit_code=$? ; expect_success odc_version=$(odc --version | head -n 1 | sed "s/^ODBAPI Version: \([0-9.]\)/\1/") cat > header-expected.txt < odc_example Property: encoder => odc version $odc_version Column: 1, Name: expver, Type: string, Size: 8 Column: 2, Name: date@hdr, Type: integer, Size: 8 Column: 3, Name: statid@hdr, Type: string, Size: 8 Column: 4, Name: wigos@hdr, Type: string, Size: 16 Column: 5, Name: obsvalue@body, Type: real, Size: 8 Column: 6, Name: integer_missing, Type: integer, Size: 8 Column: 7, Name: double_missing, Type: real, Size: 8 Column: 8, Name: bitfield_column, Type: bitfield, Size: 8 Bitfield: 1, Name: flag_a, Offset: 0, Nbits: 1 Bitfield: 2, Name: flag_b, Offset: 1, Nbits: 2 Bitfield: 3, Name: flag_c, Offset: 3, Nbits: 3 Bitfield: 4, Name: flag_d, Offset: 6, Nbits: 1 File: $test_wd/test-2.odb Frame: 1, Row count: 20, Column count: 8 Property: encoded_by => odc_example Property: encoder => odc version $odc_version Column: 1, Name: expver, Type: string, Size: 8 Column: 2, Name: date@hdr, Type: integer, Size: 8 Column: 3, Name: statid@hdr, Type: string, Size: 8 Column: 4, Name: wigos@hdr, Type: string, Size: 16 Column: 5, Name: obsvalue@body, Type: real, Size: 8 Column: 6, Name: integer_missing, Type: integer, Size: 8 Column: 7, Name: double_missing, Type: real, Size: 8 Column: 8, Name: bitfield_column, Type: bitfield, Size: 8 Bitfield: 1, Name: flag_a, Offset: 0, Nbits: 1 Bitfield: 2, Name: flag_b, Offset: 1, Nbits: 2 Bitfield: 3, Name: flag_c, Offset: 3, Nbits: 3 Bitfield: 4, Name: flag_d, Offset: 6, Nbits: 1 EOF cmp header.txt header-expected.txt # Clean up cd ${wd} rm -rf ${test_wd} odc-1.6.3/tests/c_api/odc_ls.c0000664000175000017500000001423115146027420016300 0ustar alastairalastair/** * To build this program, please make sure to reference linked library: * * gcc -lodccore -o odc-c-ls odc_ls.c */ #include #include #include #include "odc/api/odc.h" #define CHECK_RESULT(x) \ do { \ int rc = (x); \ if (rc != ODC_SUCCESS) { \ fprintf(stderr, "Error calling odc function \"%s\": %s\n", #x, odc_error_string(rc)); \ exit(1); \ } \ } while (false); void usage() { fprintf(stderr, "Usage:\n"); fprintf(stderr, " odc-c-ls \n\n"); } void write_header(odc_frame_t* frame, int ncols) { int col; for (col = 0; col < ncols; ++col) { const char* name; int type; int element_size; int bitfield_count; odc_frame_column_attributes(frame, col, &name, &type, &element_size, &bitfield_count); int padding = element_size - floor(log10(abs(col + 1))) + 2; fprintf(stdout, "%d. %-*s\t", col + 1, padding, name); } fprintf(stdout, "\n"); } void write_bitfield(int64_t val, int nbits) { int bit; for (bit = nbits - 1; bit >= 0; --bit) { fprintf(stdout, "%s", (val & (1 << bit)) ? "1" : "0"); } } void write_data(odc_decoder_t* decoder, odc_frame_t* frame, long nrows, int ncols) { const void* data; long width; long height; bool columnMajor; CHECK_RESULT(odc_decoder_data_array(decoder, &data, &width, &height, &columnMajor)); const char* name; int bitfield_count; int column_types[ncols]; int column_offsets[ncols]; int column_sizes[ncols]; int column_bf_sizes[ncols]; long missing_integer; double missing_double; int current_offset = 0; int col; for (col = 0; col < ncols; ++col) { CHECK_RESULT( odc_frame_column_attributes(frame, col, &name, &column_types[col], &column_sizes[col], &bitfield_count)); column_offsets[col] = current_offset; current_offset += column_sizes[col]; if (column_types[col] == ODC_BITFIELD) { int bf; for (bf = 0; bf < bitfield_count; ++bf) { const char* bf_name; int bf_offset; int bf_size; CHECK_RESULT(odc_frame_bitfield_attributes(frame, col, bf, &bf_name, &bf_offset, &bf_size)); if (bf == 0) column_bf_sizes[col] = bf_size; else column_bf_sizes[col] += bf_size; } } } CHECK_RESULT(odc_missing_integer(&missing_integer)); CHECK_RESULT(odc_missing_double(&missing_double)); int row; for (row = 0; row < nrows; row++) { int col; for (col = 0; col < ncols; col++) { const void* value_p = &((const char*)data)[column_offsets[col] + (width * row)]; switch (column_types[col]) { case ODC_INTEGER: if (*(const int64_t*)value_p == missing_integer) { fprintf(stdout, "."); } else { fprintf(stdout, "%-*lld", column_sizes[col], *(const int64_t*)value_p); } break; case ODC_BITFIELD: if (*(const int64_t*)value_p == 0) { fprintf(stdout, "."); } else { write_bitfield(*(const int64_t*)value_p, column_bf_sizes[col]); } break; case ODC_REAL: case ODC_DOUBLE: if (*(const double*)value_p == missing_double) { fprintf(stdout, "."); } else { fprintf(stdout, "%-*f", column_sizes[col], *(const double*)value_p); } break; case ODC_STRING: fprintf(stdout, "%-.*s", column_sizes[col], (const char*)value_p); break; default: fprintf(stdout, ""); break; } fprintf(stdout, "\t"); } fprintf(stdout, "\n"); } } int main(int argc, char* argv[]) { if (argc != 2) { usage(); return 1; } char* path = argv[1]; CHECK_RESULT(odc_initialise_api()); // initialising api CHECK_RESULT(odc_integer_behaviour(ODC_INTEGERS_AS_LONGS)); // change from default doubles odc_reader_t* reader = NULL; odc_frame_t* frame = NULL; CHECK_RESULT(odc_open_path(&reader, path)); // opening path CHECK_RESULT(odc_new_frame(&frame, reader)); // initialising frame int rc; long max_aggregated_rows = 1000000; while ((rc = odc_next_frame_aggregated(frame, max_aggregated_rows)) == ODC_SUCCESS) { int ncols; CHECK_RESULT(odc_frame_column_count(frame, &ncols)); // getting column count write_header(frame, ncols); long nrows; odc_decoder_t* decoder = NULL; CHECK_RESULT(odc_new_decoder(&decoder)); // initialising decoder CHECK_RESULT(odc_decoder_defaults_from_frame(decoder, frame)); // setting decoder structure CHECK_RESULT(odc_decode(decoder, frame, &nrows)); // decoding data write_data(decoder, frame, nrows, ncols); CHECK_RESULT(odc_free_decoder(decoder)); // cleaning up decoder } if (rc != ODC_ITERATION_COMPLETE) { fprintf(stderr, "Error: %s\n", odc_error_string(rc)); // unsuccessful end of frame iteration return 1; } CHECK_RESULT(odc_free_frame(frame)); CHECK_RESULT(odc_close(reader)); // closing reader return 0; } odc-1.6.3/tests/TemporaryFiles.h0000664000175000017500000000251215146027420016733 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef odc_tests_TemporaryODB_H #define odc_tests_TemporaryODB_H #include "eckit/filesystem/PathName.h" #include "odc/Writer.h" //---------------------------------------------------------------------------------------------------------------------- /// A class to be used as a fixture, which writes a temporary ODB file using the supplied /// initialisation function, and then ensures that it is correctly cleaned up at the /// end of the test. class TemporaryFile { public: // methods TemporaryFile() : path_(eckit::PathName::unique("_temporary_testing_file")) {} virtual ~TemporaryFile() { if (path_.exists()) { path_.unlink(); } } const eckit::PathName& path() const { return path_; } private: // members eckit::PathName path_; }; //---------------------------------------------------------------------------------------------------------------------- #endif // odc_tests_TemporaryODB_H odc-1.6.3/tests/api/0000775000175000017500000000000015146027420014366 5ustar alastairalastairodc-1.6.3/tests/api/read.cc0000664000175000017500000012413615146027420015617 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include "eckit/io/FileHandle.h" #include "eckit/testing/Test.h" #include "odc/api/Odb.h" #include "odc/api/odc.h" using namespace eckit::testing; // ------------------------------------------------------------------------------------------------------ bool test_check_file_exists(std::string file_path) { std::ifstream file_stream(file_path); return file_stream.good(); } void test_generate_odb_properties(const std::string& path, int propertiesMode = 0) { odc::api::Settings::treatIntegersAsDoubles(false); // Define row count const size_t nrows = 10; // Allocate data array for each column char data0[nrows][8]; int64_t data1[nrows]; double data2[nrows]; size_t i; // Set up the allocated arrays with scratch data for (i = 0; i < nrows; i++) { snprintf(data0[i], 8, "xxxx"); // expver data1[i] = 20210527; // date@hdr data2[i] = 12.3456 * i; // obsvalue@body } // Define all column names, their types and sizes std::vector columns = { {std::string("expver"), odc::api::ColumnType(odc::api::STRING), 8, {}}, {std::string("date@hdr"), odc::api::ColumnType(odc::api::INTEGER), sizeof(int64_t), {}}, {std::string("obsvalue@body"), odc::api::ColumnType(odc::api::REAL), sizeof(double), {}}, }; // Set a custom data layout and data array for each column std::vector strides{ // ptr, nrows, element_size, stride {data0, nrows, 8, 8}, {data1, nrows, sizeof(int64_t), sizeof(int64_t)}, {data2, nrows, sizeof(double), sizeof(double)}, }; std::map properties = {}; const eckit::Length length; eckit::FileHandle fh(path); fh.openForWrite(length); eckit::AutoClose closer(fh); // Encode two ODB-2 frames with the same data for (i = 0; i < 2; i++) { // Encode additional properties depending on the current mode switch (propertiesMode) { // Where the properties in the two frames are distinct case 1: { if (i == 0) properties["foo"] = "bar"; else { properties = {}; // reset properties["baz"] = "qux"; } break; } // Where the properties in the two frames overlap with entries that are the same case 2: { properties["foo"] = "bar"; properties["baz"] = "qux"; break; } // Where the properties overlap with entries whose keys are the same, but the values different case 3: { if (i == 0) properties["foo"] = "bar"; else properties["foo"] = "baz"; break; } default: break; } // Encode the ODB-2 frame into a data handle encode(fh, columns, strides, properties); } ASSERT(test_check_file_exists(path)); } void test_generate_odb_span(const std::string& path) { odc::api::Settings::treatIntegersAsDoubles(false); // Define row count const size_t nrows = 10; // Allocate data array for each column in all three of the frames char data0_0[nrows][8]; int64_t data0_1[nrows]; double data0_2[nrows]; double data0_3[nrows]; char data1_0[nrows][8]; int64_t data1_1[nrows]; double data1_2[nrows]; double data1_3[nrows]; char data2_0[nrows][8]; int64_t data2_1[nrows]; double data2_2[nrows]; double data2_3[nrows]; int i; // Set up the allocated arrays with scratch data for (i = 0; i < nrows; i++) { snprintf(data0_0[i], 8, "xxxx"); // expver data0_1[i] = 20210527; // date@hdr data0_2[i] = 12.3456 * i; // obsvalue@body data0_3[i] = odc::api::Settings::doubleMissingValue(); // missing_value snprintf(data1_0[i], 8, "xxxx"); // expver data1_1[i] = 20210528; // date@hdr data1_2[i] = 12.3456 * i; // obsvalue@body data1_3[i] = odc::api::Settings::doubleMissingValue(); // missing_value snprintf(data2_0[i], 8, "xxxx"); // expver data2_1[i] = 20210529; // date@hdr data2_2[i] = 12.3456 * i; // obsvalue@body data2_3[i] = odc::api::Settings::doubleMissingValue(); // missing_value } // Define all column names, their types and sizes std::vector columns = { {std::string("expver"), odc::api::ColumnType(odc::api::STRING), 8, {}}, {std::string("date@hdr"), odc::api::ColumnType(odc::api::INTEGER), sizeof(int64_t), {}}, {std::string("obsvalue@body"), odc::api::ColumnType(odc::api::REAL), sizeof(double), {}}, {std::string("missing_value"), odc::api::ColumnType(odc::api::REAL), sizeof(double), {}}, }; // Set a custom data layout and data array for each column and frame std::vector strides0{ // ptr, nrows, element_size, stride {data0_0, nrows, 8, 8}, {data0_1, nrows, sizeof(int64_t), sizeof(int64_t)}, {data0_2, nrows, sizeof(double), sizeof(double)}, {data0_3, nrows, sizeof(double), sizeof(double)}, }; std::vector strides1{ // ptr, nrows, element_size, stride {data1_0, nrows, 8, 8}, {data1_1, nrows, sizeof(int64_t), sizeof(int64_t)}, {data1_2, nrows, sizeof(double), sizeof(double)}, {data1_3, nrows, sizeof(double), sizeof(double)}, }; std::vector strides2{ // ptr, nrows, element_size, stride {data2_0, nrows, 8, 8}, {data2_1, nrows, sizeof(int64_t), sizeof(int64_t)}, {data2_2, nrows, sizeof(double), sizeof(double)}, {data2_3, nrows, sizeof(double), sizeof(double)}, }; const eckit::Length length; eckit::FileHandle fh(path); fh.openForWrite(length); eckit::AutoClose closer(fh); // Encode each ODB-2 frame into the same data handle encode(fh, columns, strides0); encode(fh, columns, strides1); encode(fh, columns, strides2); ASSERT(test_check_file_exists(path)); } // ------------------------------------------------------------------------------------------------------ CASE("Count lines in an existing ODB file") { bool aggregated = false; odc::api::Reader reader("../2000010106-reduced.odb", aggregated); size_t nframes = 0; size_t totalRows = 0; odc::api::Frame frame; while ((frame = reader.next())) { totalRows += frame.rowCount(); EXPECT(frame.columnCount() == 51); ++nframes; } EXPECT(nframes == 5); EXPECT(totalRows == 50000); } CASE("Check column details in an existing ODB file") { std::vector cols{ "expver@desc", "andate@desc", "antime@desc", "seqno@hdr", "obstype@hdr", "obschar@hdr", "subtype@hdr", "date@hdr", "time@hdr", "rdbflag@hdr", "status@hdr", "event1@hdr", "blacklist@hdr", "sortbox@hdr", "sitedep@hdr", "statid@hdr", "ident@hdr", "lat@hdr", "lon@hdr", "stalt@hdr", "modoro@hdr", "trlat@hdr", "trlon@hdr", "instspec@hdr", "event2@hdr", "anemoht@hdr", "baroht@hdr", "sensor@hdr", "numlev@hdr", "varno_presence@hdr", "varno@body", "vertco_type@body", "rdbflag@body", "anflag@body", "status@body", "event1@body", "blacklist@body", "entryno@body", "press@body", "press_rl@body", "obsvalue@body", "aux1@body", "event2@body", "ppcode@body", "level@body", "biascorr@body", "final_obs_error@errstat", "obs_error@errstat", "repres_error@errstat", "pers_error@errstat", "fg_error@errstat", }; std::vector types{ odc::api::STRING, odc::api::INTEGER, odc::api::INTEGER, odc::api::INTEGER, odc::api::INTEGER, odc::api::BITFIELD, odc::api::INTEGER, odc::api::INTEGER, odc::api::INTEGER, odc::api::BITFIELD, odc::api::BITFIELD, odc::api::BITFIELD, odc::api::BITFIELD, odc::api::INTEGER, odc::api::INTEGER, odc::api::STRING, odc::api::INTEGER, odc::api::REAL, odc::api::REAL, odc::api::REAL, odc::api::REAL, odc::api::REAL, odc::api::REAL, odc::api::INTEGER, odc::api::INTEGER, odc::api::REAL, odc::api::REAL, odc::api::INTEGER, odc::api::INTEGER, odc::api::BITFIELD, odc::api::INTEGER, odc::api::INTEGER, odc::api::BITFIELD, odc::api::BITFIELD, odc::api::BITFIELD, odc::api::BITFIELD, odc::api::BITFIELD, odc::api::INTEGER, odc::api::REAL, odc::api::REAL, odc::api::REAL, odc::api::REAL, odc::api::INTEGER, odc::api::INTEGER, odc::api::BITFIELD, odc::api::REAL, odc::api::REAL, odc::api::REAL, odc::api::REAL, odc::api::REAL, odc::api::REAL, }; std::vector bitfields{ "lat_humon", "lat_qcsub", "lat_override", "lat_flag", "lat_hqc_flag", "lon_humon", "lon_qcsub", "lon_override", "lon_flag", "lon_hqc_flag", "date_humon", "date_qcsub", "date_override", "date_flag", "date_hqc_flag", "time_humon", "time_qcsub", "time_override", "time_flag", "time_hqc_flag", "stalt_humon", "stalt_qcsub", "stalt_override", "stalt_flag", "stalt_hqc_flag", }; std::vector bitfield_sizes{ 1, 1, 1, 2, 1, 1, 1, 1, 2, 1, 1, 1, 1, 2, 1, 1, 1, 1, 2, 1, 1, 1, 1, 2, 1, }; bool aggregated = false; odc::api::Reader reader("../2000010106-reduced.odb", aggregated); // Get the first frame odc::api::Frame frame = reader.next(); EXPECT(frame.columnCount() == 51); int i = 0; // Iterate over frame columns for (const auto& column : frame.columnInfo()) { const int col = i++; EXPECT(column.name == cols.at(col)); EXPECT(column.type == types.at(col)); EXPECT(column.decodedSize == 8); if (column.type == odc::api::BITFIELD) { EXPECT(column.bitfield.size() > 0); } else { EXPECT(column.bitfield.size() == 0); } // Test bitfields for column 10 if (col == 9) { EXPECT(column.bitfield.size() == 25); int j = 0; int expected_offset = 0; for (auto const& bitfield : column.bitfield) { const int bf = j++; EXPECT(bitfield.name == bitfields.at(bf)); EXPECT(bitfield.size == bitfield_sizes.at(bf)); EXPECT(bitfield.offset == expected_offset); expected_offset = expected_offset + bitfield.size; } } } } CASE("Decode data in an existing ODB file") { odc::api::Settings::treatIntegersAsDoubles(false); bool aggregated = false; odc::api::Reader reader("../2000010106-reduced.odb", aggregated); // Get the first frame odc::api::Frame frame = reader.next(); // Read the second frame, because why not frame = reader.next(); // Properties of this frame size_t ncols = frame.columnCount(); size_t nrows = frame.rowCount(); const auto& columnInfo = frame.columnInfo(); EXPECT(nrows == 10000); EXPECT(ncols == 51); // Determine storage requirements size_t row_size = 0; size_t i; for (i = 0; i < frame.columnCount(); ++i) { const auto& col(columnInfo[i]); row_size += col.decodedSize; } // Allocate storage required size_t storage_size = row_size * nrows; std::vector buffer(storage_size); // Decoder prerequisites std::vector columns; std::vector strides; char* ptr = &buffer[0]; for (const auto& col : columnInfo) { columns.push_back(col.name); strides.emplace_back(odc::api::StridedData{ptr, nrows, col.decodedSize, col.decodedSize}); ptr += nrows * col.decodedSize; } EXPECT(ptr == (&buffer[0] + storage_size)); // Decode the data int nthreads = 4; odc::api::Decoder decoder(columns, strides); decoder.decode(frame, nthreads); std::vector expected_seqno{(int64_t)6106691, (int64_t)6002945, (int64_t)6003233, (int64_t)6105819}; std::vector expected_obschar{ (int64_t)537918674, (int64_t)135265490, (int64_t)135265490, (int64_t)537918674, }; std::vector expected_lat{ (double)0.370279, (double)0.226484, (double)0.105947, (double)-0.0300668, }; const int width = 8; long integer_missing = odc::api::Settings::integerMissingValue(); double double_missing = odc::api::Settings::doubleMissingValue(); size_t row; for (i = 0; i < 4; ++i) { row = i * 765; // expver@desc (ODC_STRING, col 1) EXPECT(std::string(strides[0][row], ::strnlen(strides[0][row], columnInfo[0].decodedSize)).substr(0, 4) == "0018"); // seqno@hdr (ODC_INTEGER, col 4) EXPECT(*reinterpret_cast(strides[3][row]) == expected_seqno.at(i)); // obschar@hdr (ODC_BITFIELD, col 6) EXPECT(*reinterpret_cast(strides[5][row]) == expected_obschar.at(i)); // sortbox@hdr (ODC_INTEGER, col 14, missing value!) EXPECT(*reinterpret_cast(strides[13][row]) == integer_missing); // lat@hdr (ODC_REAL, col 18) EXPECT(eckit::types::is_approximately_equal(*reinterpret_cast(strides[17][row]), expected_lat.at(i), 0.000001)); // repres_error@errstat (ODC_REAL, col 49, missing value!) EXPECT(*reinterpret_cast(strides[48][row]) == double_missing); } } // ------------------------------------------------------------------------------------------------------ CASE("Where the properties in the two frames are distinct (non-aggregated)") { test_generate_odb_properties("properties-1.odb", 1); bool aggregated = false; odc::api::Reader reader("properties-1.odb", aggregated); odc::api::Frame frame; size_t nframes = 0; std::map seenProperties; while ((frame = reader.next())) { for (const auto& property : frame.properties()) { if (nframes == 0) { EXPECT(property.first == "foo" || property.first == "encoder"); EXPECT(property.second == "bar" || property.second == std::string("odc version ") + odc::api::Settings::version()); } else { EXPECT(property.first == "baz" || property.first == "encoder"); EXPECT(property.second == "qux" || property.second == std::string("odc version ") + odc::api::Settings::version()); } seenProperties[property.first] = property.second; } ++nframes; } EXPECT(nframes == 2); EXPECT(seenProperties.size() == 3); } CASE("Where the properties in the two frames are distinct (aggregated)") { ASSERT(test_check_file_exists("properties-1.odb")); bool aggregated = true; odc::api::Reader reader("properties-1.odb", aggregated); odc::api::Frame frame; size_t nframes = 0; std::map seenProperties; while ((frame = reader.next())) { for (const auto& property : frame.properties()) { seenProperties[property.first] = property.second; } ++nframes; } EXPECT(nframes == 1); EXPECT(seenProperties.size() == 3); for (const auto& property : seenProperties) { EXPECT(property.first == "foo" || property.first == "baz" || property.first == "encoder"); EXPECT(property.second == "bar" || property.second == "qux" || property.second == std::string("odc version ") + odc::api::Settings::version()); } } CASE("Where the properties in the two frames overlap with entries that are the same (non-aggregated)") { test_generate_odb_properties("properties-2.odb", 2); bool aggregated = false; odc::api::Reader reader("properties-2.odb", aggregated); odc::api::Frame frame; size_t nframes = 0; std::map seenProperties; while ((frame = reader.next())) { for (const auto& property : frame.properties()) { if (nframes == 0) { EXPECT(property.first == "foo" || property.first == "baz" || property.first == "encoder"); EXPECT(property.second == "bar" || property.second == "qux" || property.second == std::string("odc version ") + odc::api::Settings::version()); } else { EXPECT(property.first == "foo" || property.first == "baz" || property.first == "encoder"); EXPECT(property.second == "bar" || property.second == "qux" || property.second == std::string("odc version ") + odc::api::Settings::version()); } seenProperties[property.first] = property.second; } ++nframes; } EXPECT(nframes == 2); EXPECT(seenProperties.size() == 3); } CASE("Where the properties in the two frames overlap with entries that are the same (aggregated)") { ASSERT(test_check_file_exists("properties-2.odb")); bool aggregated = true; odc::api::Reader reader("properties-2.odb", aggregated); odc::api::Frame frame; size_t nframes = 0; std::map seenProperties; while ((frame = reader.next())) { for (const auto& property : frame.properties()) { seenProperties[property.first] = property.second; } ++nframes; } EXPECT(nframes == 1); EXPECT(seenProperties.size() == 3); for (const auto& property : seenProperties) { EXPECT(property.first == "foo" || property.first == "baz" || property.first == "encoder"); EXPECT(property.second == "bar" || property.second == "qux" || property.second == std::string("odc version ") + odc::api::Settings::version()); } } CASE("Where the properties overlap with entries whose keys are the same, but the values different (non-aggregated)") { test_generate_odb_properties("properties-3.odb", 3); bool aggregated = false; odc::api::Reader reader("properties-3.odb", aggregated); odc::api::Frame frame; size_t nframes = 0; std::map seenProperties; while ((frame = reader.next())) { for (const auto& property : frame.properties()) { if (nframes == 0) { EXPECT(property.first == "foo" || property.first == "encoder"); EXPECT(property.second == "bar" || property.second == std::string("odc version ") + odc::api::Settings::version()); } else { EXPECT(property.first == "foo" || property.first == "encoder"); EXPECT(property.second == "baz" || property.second == std::string("odc version ") + odc::api::Settings::version()); } seenProperties[property.first] = property.second; } ++nframes; } EXPECT(nframes == 2); EXPECT(seenProperties.size() == 2); } CASE("Where the properties overlap with entries whose keys are the same, but the values different (aggregated)") { ASSERT(test_check_file_exists("properties-3.odb")); bool aggregated = true; odc::api::Reader reader("properties-3.odb", aggregated); odc::api::Frame frame; size_t nframes = 0; std::map seenProperties; while ((frame = reader.next())) { for (const auto& property : frame.properties()) { seenProperties[property.first] = property.second; } ++nframes; } EXPECT(nframes == 1); EXPECT(seenProperties.size() == 2); for (const auto& property : seenProperties) { EXPECT(property.first == "foo" || property.first == "encoder"); EXPECT(property.second == "bar" // value from the first frame will win! || property.second == std::string("odc version ") + odc::api::Settings::version()); } } // ------------------------------------------------------------------------------------------------------ class TestVisitor : public odc::api::SpanVisitor { public: TestVisitor(int frame, bool mustBeConstant) : frame_(frame), mustBeConstant_(mustBeConstant) {}; private: template void test(const std::string& columnName, const std::set& vals) { if (mustBeConstant_) { ASSERT(vals.size() == 1); } else { if (columnName == "obsvalue@body") ASSERT(vals.size() == 10); else ASSERT(vals.size() == 1); } std::stringstream val; val << *vals.begin(); switch (frame_) { case 0: if (columnName == "expver") { ASSERT(val.str() == "xxxx"); } else if (columnName == "date@hdr") { ASSERT(val.str() == "20210527"); } else if (columnName == "obsvalue@body") { int i = 0; for (auto obsvalue : vals) { std::stringstream obsvalue_stream; obsvalue_stream << obsvalue; std::stringstream expvalue_stream; expvalue_stream << 12.3456 * i++; ASSERT(obsvalue_stream.str() == expvalue_stream.str()); } } else if (columnName == "missing_value") { std::stringstream expvalue_stream; expvalue_stream << odc::api::Settings::doubleMissingValue(); ASSERT(val.str() == expvalue_stream.str()); } break; case 1: if (columnName == "expver") { ASSERT(val.str() == "xxxx"); } else if (columnName == "date@hdr") { ASSERT(val.str() == "20210528"); } else if (columnName == "obsvalue@body") { int i = 0; for (auto obsvalue : vals) { std::stringstream obsvalue_stream; obsvalue_stream << obsvalue; std::stringstream expvalue_stream; expvalue_stream << 12.3456 * i++; ASSERT(obsvalue_stream.str() == expvalue_stream.str()); } } break; case 2: if (columnName == "expver") { ASSERT(val.str() == "xxxx"); } else if (columnName == "date@hdr") { ASSERT(val.str() == "20210529"); } else if (columnName == "obsvalue@body") { int i = 0; for (auto obsvalue : vals) { std::stringstream obsvalue_stream; obsvalue_stream << obsvalue; std::stringstream expvalue_stream; expvalue_stream << 12.3456 * i++; ASSERT(obsvalue_stream.str() == expvalue_stream.str()); } } break; } } void operator()(const std::string& columnName, const std::set& vals) override { test(columnName, vals); } void operator()(const std::string& columnName, const std::set& vals) override { test(columnName, vals); } void operator()(const std::string& columnName, const std::set& vals) override { test(columnName, vals); } int frame_; bool mustBeConstant_; }; CASE("Where Span interface is used with constant value constraint") { test_generate_odb_span("span-1.odb"); // Define columns with constant values std::vector cols{"expver", "date@hdr"}; // Parse frames in non-aggregated mode bool aggregated = false; // Enforce the constant values constraint bool mustBeConstant = true; { odc::api::Reader reader("span-1.odb", aggregated); odc::api::Frame frame; odc::api::Span lastSpan; long offset = 0; long length = 0; bool first = true; int i = 0; // Iterate over frames while ((frame = reader.next())) { // Get values for the frame odc::api::Span span = frame.span(cols, mustBeConstant); // If the values are the same, just increase the length if (span == lastSpan || first) { length += span.length(); // Remember the first set of values if (first) std::swap(lastSpan, span); } // If the values differ, output the last set else { TestVisitor v(i, mustBeConstant); lastSpan.visit(v); EXPECT(offset == i++ * 453); EXPECT(length == 453); // Reset offset and length counters offset = span.offset(); length = span.length(); // Remember the current set of values std::swap(lastSpan, span); } first = false; } TestVisitor v(i, mustBeConstant); lastSpan.visit(v); EXPECT(offset == 906); EXPECT(length == 453); } // Add a non-constant value column to the list cols.emplace_back("obsvalue@body"); { odc::api::Reader reader("span-1.odb", aggregated); odc::api::Frame frame; // Iterate over frames while ((frame = reader.next())) { // Try to get values for the frame try { odc::api::Span span = frame.span(cols, mustBeConstant); } // A user error is raised when constant value constraint is not met catch (eckit::Exception& e) { EXPECT(std::string(e.what()) == "UserError: Non-constant columns required in span: [obsvalue@body]"); } } } } CASE("Where Span interface is used without constant value constraint") { ASSERT(test_check_file_exists("span-1.odb")); // Define columns std::vector cols{"expver", "date@hdr", "obsvalue@body"}; // Parse frames in non-aggregated mode bool aggregated = false; // Do not enforce the constant values constraint bool mustBeConstant = false; odc::api::Reader reader("span-1.odb", aggregated); odc::api::Frame frame; odc::api::Span lastSpan; long offset = 0; long length = 0; bool first = true; int i = 0; // Iterate over frames while ((frame = reader.next())) { // Get values for the frame odc::api::Span span = frame.span(cols, mustBeConstant); // If the values are the same, just increase the length if (span == lastSpan || first) { length += span.length(); // Remember the first set of values if (first) std::swap(lastSpan, span); } // If the values differ, output the last set else { TestVisitor v(i, mustBeConstant); lastSpan.visit(v); EXPECT(offset == i++ * 453); EXPECT(length == 453); // Reset offset and length counters offset = span.offset(); length = span.length(); // Remember the current set of values std::swap(lastSpan, span); } first = false; } TestVisitor v(i, mustBeConstant); lastSpan.visit(v); EXPECT(offset == 906); EXPECT(length == 453); } CASE("Where Span interface is used with a missing column") { ASSERT(test_check_file_exists("span-1.odb")); // Define columns and include a missing one std::vector cols{"expver", "date@hdr", "foo@bar"}; // Parse frames in non-aggregated mode bool aggregated = false; // Do not enforce the constant values constraint bool mustBeConstant = false; odc::api::Reader reader("span-1.odb", aggregated); odc::api::Frame frame; // Iterate over frames while ((frame = reader.next())) { // Try to get values for the frame try { odc::api::Span span = frame.span(cols, mustBeConstant); } // A user error is raised when specified column cannot be found catch (eckit::Exception& e) { EXPECT(std::string(e.what()) == "UserError: Column 'foo@bar' not found."); } } } CASE("Where Span interface is used with no columns specified") { ASSERT(test_check_file_exists("span-1.odb")); // Define empty list of columns std::vector cols{}; // Parse frames in non-aggregated mode bool aggregated = false; // Do not enforce the constant values constraint bool mustBeConstant = false; odc::api::Reader reader("span-1.odb", aggregated); odc::api::Frame frame; odc::api::Span lastSpan; long offset = 0; long length = 0; bool first = true; int i = 0; // Iterate over frames while ((frame = reader.next())) { // Get values for the frame odc::api::Span span = frame.span(cols, mustBeConstant); // If the values are the same, just increase the length if (span == lastSpan || first) { length += span.length(); // Remember the first set of values if (first) std::swap(lastSpan, span); } // If the values differ, output the last set else { TestVisitor v(i++, mustBeConstant); lastSpan.visit(v); EXPECT(offset == 0); EXPECT(length == 0); // Reset offset and length counters offset = span.offset(); length = span.length(); // Remember the current set of values std::swap(lastSpan, span); } first = false; } TestVisitor v(i, mustBeConstant); lastSpan.visit(v); EXPECT(offset == 0); EXPECT(length == 1359); } CASE("Where Span interface is used with all columns specified") { ASSERT(test_check_file_exists("span-1.odb")); // Define list of columns with all of them std::vector cols{"expver", "date@hdr", "obsvalue@body", "missing_value"}; // Parse frames in non-aggregated mode bool aggregated = false; // Do not enforce the constant values constraint bool mustBeConstant = false; odc::api::Reader reader("span-1.odb", aggregated); odc::api::Frame frame; odc::api::Span lastSpan; long offset = 0; long length = 0; bool first = true; int i = 0; // Iterate over frames while ((frame = reader.next())) { // Get values for the frame odc::api::Span span = frame.span(cols, mustBeConstant); // If the values are the same, just increase the length if (span == lastSpan || first) { length += span.length(); // Remember the first set of values if (first) std::swap(lastSpan, span); } // If the values differ, output the last set else { TestVisitor v(i, mustBeConstant); lastSpan.visit(v); EXPECT(offset == 453 * i++); EXPECT(length == 453); // Reset offset and length counters offset = span.offset(); length = span.length(); // Remember the current set of values std::swap(lastSpan, span); } first = false; } TestVisitor v(i, mustBeConstant); lastSpan.visit(v); EXPECT(offset == 906); EXPECT(length == 453); } CASE("Where Span interface is used with missing values") { ASSERT(test_check_file_exists("span-1.odb")); // Add missing value column to the list std::vector cols{"missing_value"}; // Parse frames in non-aggregated mode bool aggregated = false; // Enforce the constant values constraint bool mustBeConstant = false; odc::api::Reader reader("span-1.odb", aggregated); odc::api::Frame frame; odc::api::Span lastSpan; long offset = 0; long length = 0; bool first = true; int i = 0; // Iterate over frames while ((frame = reader.next())) { // Get values for the frame odc::api::Span span = frame.span(cols, mustBeConstant); // If the values are the same, just increase the length if (span == lastSpan || first) { length += span.length(); // Remember the first set of values if (first) std::swap(lastSpan, span); } // If the values differ, output the last set else { TestVisitor v(i++, mustBeConstant); lastSpan.visit(v); EXPECT(offset == 0); EXPECT(length == 0); // Reset offset and length counters offset = span.offset(); length = span.length(); // Remember the current set of values std::swap(lastSpan, span); } first = false; } TestVisitor v(i, mustBeConstant); lastSpan.visit(v); EXPECT(offset == 0); EXPECT(length == 1359); } CASE("Where Span interface is used to read values without decoding") { ASSERT(test_check_file_exists("span-1.odb")); // Add columns of all three types to the list std::vector cols{"expver", "date@hdr", "obsvalue@body"}; // Parse frames in aggregated mode bool aggregated = true; // Do not enforce the constant values constraint bool mustBeConstant = false; odc::api::Reader reader("span-1.odb", aggregated); odc::api::Frame frame; std::set expver_vals; std::set date_vals; std::set obsvalue_vals; // Iterate over frames while ((frame = reader.next())) { // Get values for the frame odc::api::Span span = frame.span(cols, mustBeConstant); expver_vals = span.getStringValues("expver"); date_vals = span.getIntegerValues("date@hdr"); obsvalue_vals = span.getRealValues("obsvalue@body"); } // Check string values for (const auto& val : expver_vals) { EXPECT(val == "xxxx"); } std::vector expdate_vals = {20210527, 20210528, 20210529}; int i = 0; // Check integer values for (const long val : date_vals) { EXPECT(val == expdate_vals[i++]); } i = 0; // Check real values for (const double val : obsvalue_vals) { EXPECT(eckit::types::is_approximately_equal(val, 12.3456 * i++, 0.0001)); } } // ------------------------------------------------------------------------------------------------------ CASE("Filter a subset of ODB-2 data") { eckit::FileHandle in("../2000010106-reduced.odb"); in.openForRead(); eckit::AutoClose close_in(in); const eckit::Length length; eckit::FileHandle out("2000010106-filtered.odb"); out.openForWrite(length); eckit::AutoClose close_out(out); // Create a new file with a subset of data odc::api::filter("select expver, date, lat, lon, obsvalue where rownumber() <= 10", in, out); EXPECT(test_check_file_exists("2000010106-filtered.odb")); eckit::FileHandle fh("2000010106-filtered.odb"); fh.openForRead(); eckit::AutoClose close_fh(fh); // Access file via data handle reference, in order to test alternate constructor eckit::DataHandle* dh = fh.clone(); bool aggregated = true; odc::api::Reader reader(dh, aggregated); size_t nframes = 0; size_t totalRows = 0; odc::api::Frame frame = reader.next(); // Test second type of frame constructor via an existing copy odc::api::Frame test_frame(frame); EXPECT(test_frame.rowCount() == 10); // Access encoded data directly from a frame EXPECT(test_frame.encodedData().size() == 487); // Check if frame has expected columns. EXPECT(test_frame.columnCount() == 5); EXPECT(test_frame.hasColumn("expver")); EXPECT(test_frame.hasColumn("date@hdr")); EXPECT(test_frame.hasColumn("lat@hdr")); EXPECT(test_frame.hasColumn("lon@hdr")); EXPECT(test_frame.hasColumn("obsvalue@body")); // Check expected frame bounds EXPECT(int(test_frame.offset()) == 0); EXPECT(long(test_frame.length()) == 487); // Duplicate current frame via the SQL filter function odc::api::Frame sub_frame(test_frame.filter("select *")); // Check that frames are indeed identical EXPECT(sub_frame.rowCount() == 10); EXPECT(sub_frame.columnCount() == 5); EXPECT(sub_frame.hasColumn("expver")); EXPECT(sub_frame.hasColumn("date@hdr")); EXPECT(sub_frame.hasColumn("lat@hdr")); EXPECT(sub_frame.hasColumn("lon@hdr")); EXPECT(sub_frame.hasColumn("obsvalue@body")); EXPECT(int(sub_frame.offset()) == 0); EXPECT(long(sub_frame.length()) == 487); // Test creation of a frame via the assignment operator odc::api::Frame dummy_frame = frame; EXPECT(dummy_frame.rowCount() == 10); } CASE("Test Span interface with and without treatIntegersAsDoubles") { // Construct some test data std::vector RESTRICTED{1, 1, 1, 1, 0, 1, 0, 0, 1, 0, 0, 0}; std::vector VALUES{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}; eckit::PathName path = "span-test-1"; { odc::api::Settings::treatIntegersAsDoubles(false); std::unique_ptr dh(path.fileHandle()); dh->openForWrite(0); eckit::AutoClose closer(*dh); std::vector columns{ {"values@body", odc::api::ColumnType::INTEGER, sizeof(int64_t), {}}, {"restricted@hdr", odc::api::ColumnType::INTEGER, sizeof(int64_t), {}}, }; std::vector strides{ {&VALUES[0], VALUES.size(), sizeof(int64_t), sizeof(int64_t)}, {&RESTRICTED[0], RESTRICTED.size(), sizeof(int64_t), sizeof(int64_t)}, }; odc::api::encode(*dh, columns, strides); } // Get the span of this data for (bool asDoubles : {false, true}) { odc::api::Settings::treatIntegersAsDoubles(asDoubles); std::unique_ptr dh(path.fileHandle()); dh->openForRead(); eckit::AutoClose closer(*dh); odc::api::Reader reader(*dh, /* aggregated */ true); odc::api::Frame frame = reader.next(); odc::api::Span s = frame.span(std::vector({"values", "restricted"}), /* onlyConst */ false); EXPECT(s.getIntegerValues("values") == std::set({1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12})); EXPECT(s.getIntegerValues("restricted") == std::set({0, 1})); EXPECT(!reader.next()); } } // ------------------------------------------------------------------------------------------------------ // CASE("Decode an entire ODB file") { // // odc::api::Odb o("../2000010106-reduced.odb"); // // size_t ntables = 0; // // while (const auto& table = o.next()) { // // DecodeTarget decoded; // table.get().decode(); // // EXPECT(decoded.rows() == table.get().rowCount()); // EXPECT(decoded.columns() == 51); // // ++ntables; // } // } // //// ------------------------------------------------------------------------------------------------------ // // CASE("Decode only some columns") { // // odc::api::Odb o("../2000010106-reduced.odb"); // // size_t ntables = 0; // // while (const auto& table = o.next()) { // // DecodeTarget decoded; // decoded.addColumn("statid"); // decoded.addColumn("expver"); // decoded.addColumn("andate"); // decoded.addColumn("obsvalue"); // // DecodeTarget decoded = table.get().decode(); // // EXPECT(decoded.rows() == table.get().rowCount()); // EXPECT(decoded.columns() == 51); // // ++ntables; // } //} // ------------------------------------------------------------------------------------------------------ // // CASE("Decode an entire ODB file preallocated data structures") { // // std::unique_ptr o(odc_open_for_read("../2000010106-reduced.odb")); // // int ntables = odc_num_tables(o.get()); // EXPECT(ntables == 5); // // odb_decoded_t decoded; // odb_strided_data_t strided_data[51]; // // for (int i = 0; i < ntables; i++) { // // std::unique_ptr table(odc_get_table(o.get(), i)); // // ASSERT(odc_table_num_columns(table.get()) == 51); // // decoded.ncolumns = 51; // decoded.nrows = 10000; // decoded.columnData = strided_data; // // /// odc_table_decode(table.get(), &decoded); // // ///EXPECT(decoded.nrows == odc_table_num_rows(table.get())); // ///EXPECT(decoded.ncolumns == 51); // // ///eckit::Log::info() << "Decoded: ncolumns = " << decoded.ncolumns << std::endl; // ///eckit::Log::info() << "Decoded: nrows = " << decoded.nrows << std::endl; // ///eckit::Log::info() << "Decoded: data = " << decoded.columnData << std::endl; // } // } // //// ------------------------------------------------------------------------------------------------------ int main(int argc, char* argv[]) { return run_tests(argc, argv); } odc-1.6.3/tests/api/CMakeLists.txt0000664000175000017500000000241515146027420017130 0ustar alastairalastair list( APPEND _api_odc_tests read import ) foreach( _test ${_api_odc_tests} ) ecbuild_add_test( TARGET odc_api_${_test} SOURCES ${_test}.cc ENVIRONMENT ${test_environment} TEST_DEPENDS odc_get_test_data LIBS eckit odccore odctools ) endforeach() list( APPEND _api_odc_example_sources odc_encode_custom odc_header odc_index odc_ls ) list( APPEND _api_odc_example_targets odc-cpp-encode-custom odc-cpp-header odc-cpp-index odc-cpp-ls ) list( LENGTH _api_odc_example_sources _count ) math( EXPR _count "${_count}-1" ) foreach( _i RANGE ${_count} ) list( GET _api_odc_example_sources ${_i} _sources ) list( GET _api_odc_example_targets ${_i} _target ) ecbuild_add_executable( TARGET ${_target} SOURCES ${_sources}.cc LIBS eckit odccore NOINSTALL ) endforeach() list( APPEND _api_odc_tests_scripts usage_examples.sh ) foreach( _script ${_api_odc_tests_scripts} ) ecbuild_add_test( TARGET odc_api_${_script} TYPE script COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/${_script} ENVIRONMENT PATH=${CMAKE_BINARY_DIR}/bin:$ENV{PATH} ) endforeach() odc-1.6.3/tests/api/odc_index.cc0000664000175000017500000000557715146027420016647 0ustar alastairalastair/** * To build this program, please make sure to reference linked libraries: * * g++ -std=c++11 -leckit -lodccore -o odc-cpp-index odc_index.cc */ #include "eckit/runtime/Main.h" #include "odc/api/Odb.h" #include #include #include #include using namespace odc::api; void usage() { std::cerr << "Usage:\n odc-cpp-index " << std::endl << std::endl; } class Printer : public SpanVisitor { template void prnt(const std::string& columnName, const std::set& vals) { ASSERT(vals.size() == 1); std::cout << columnName << "=" << *vals.begin() << " "; } void operator()(const std::string& columnName, const std::set& vals) override { prnt(columnName, vals); } void operator()(const std::string& columnName, const std::set& vals) override { prnt(columnName, vals); } void operator()(const std::string& columnName, const std::set& vals) override { prnt(columnName, vals); } }; void write_index(Span& span, long offset, long length) { std::cout << "Archival unit: offset=" << offset << " length=" << length << std::endl; std::cout << " Key: "; // Dump the index values without decoding the frame data Printer p; span.visit(p); std::cout << std::endl; } int main(int argc, char** argv) { if (argc != 2) { usage(); return 1; } char* path = argv[1]; eckit::Main::initialise(argc, argv); // Initialise library odc::api::Settings::treatIntegersAsDoubles(false); bool aggregated = false; // Open supplied path in non-aggregated mode Reader reader(path, aggregated); // Define which columns will be used as index keys std::vector index_keys{"key1", "key2", "key3"}; Frame frame; Span lastSpan; long offset = 0; long length = 0; bool first = true; // Iterate over frames while ((frame = reader.next())) { // Enforce the constant values constraint bool mustBeConstant = true; // Get index values for the frame Span span = frame.span(index_keys, mustBeConstant); // If the index values are the same, just increase the length if (span == lastSpan || first) { length += span.length(); // Remember the first set of index values if (first) std::swap(lastSpan, span); } // If the index values differ, output the last set else { write_index(lastSpan, offset, length); // Reset offset and length counters offset = span.offset(); length = span.length(); // Remember the current set of index values std::swap(lastSpan, span); } first = false; } // Output last set of index values write_index(lastSpan, offset, length); return 0; } odc-1.6.3/tests/api/odc_header.cc0000664000175000017500000000424615146027420016760 0ustar alastairalastair/** * To build this program, please make sure to reference linked libraries: * * g++ -std=c++11 -leckit -lodccore -o odc-cpp-header odc_header.cc */ #include #include "eckit/runtime/Main.h" #include "odc/api/Odb.h" void usage() { std::cerr << "Usage:\n odc-cpp-header [ ...]" << std::endl << std::endl; } using namespace eckit; using namespace odc::api; int main(int argc, char** argv) { if (argc < 2) { usage(); return 1; } // Initialise API Main::initialise(argc, argv); int argi; // Iterate over all supplied path arguments for (argi = 1; argi < argc; argi++) { char* path = argv[argi]; bool aggregated = false; // Open supplied path in non-aggregated mode Reader reader(path, aggregated); std::cout << "File: " << path << std::endl; Frame frame; int i = 0; // Iterate over all frames in the stream, without decoding them while ((frame = reader.next())) { std::cout << " Frame: " << ++i << ", Row count: " << frame.rowCount() << ", Column count: " << frame.columnCount() << std::endl; for (const auto& property : frame.properties()) { std::cout << " Property: " << property.first << " => " << property.second << std::endl; } int j = 0; // Iterate over frame columns for (const auto& column : frame.columnInfo()) { std::cout << " Column: " << ++j << ", Name: " << column.name << ", Type: " << columnTypeName(column.type) << ", Size: " << column.decodedSize << std::endl; // Process bitfields only if (column.type == BITFIELD) { int k = 0; for (auto const& bf : column.bitfield) { std::cout << " Bitfield: " << ++k << ", Name: " << bf.name << ", Offset: " << bf.offset << ", Nbits: " << bf.size << std::endl; } } } std::cout << std::endl; } } return 0; } odc-1.6.3/tests/api/import.cc0000664000175000017500000000327715146027420016220 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/io/MemoryHandle.h" #include "eckit/testing/Test.h" #include "odc/api/Odb.h" using namespace eckit::testing; // ------------------------------------------------------------------------------------------------------ CASE("We can import data") { const char* SOURCE_DATA = R"(col1:INTEGER,col2:REAL,col3:DOUBLE,col4:INTEGER,col5:BITFIELD[a:1;b:2;c:5] 1,1.23,4.56,7,999 123,0.0,0.0,321,888 321,0.0,0.0,123,777 0,3.25,0.0,0,666 0,0.0,3.25,0,555)"; eckit::MemoryHandle dh_out; size_t importedSize; { dh_out.openForWrite(0); eckit::AutoClose close(dh_out); ::odc::api::odbFromCSV(SOURCE_DATA, dh_out); importedSize = dh_out.position(); } eckit::Log::info() << "Imported length: " << importedSize << std::endl; eckit::MemoryHandle readAgain(dh_out.data(), importedSize); readAgain.openForRead(); bool aggregated = false; odc::api::Reader r(readAgain, aggregated); odc::api::Frame f; while ((f = r.next())) { eckit::Log::info() << "Frame: " << f.rowCount() << std::endl; } } // ------------------------------------------------------------------------------------------------------ int main(int argc, char* argv[]) { return run_tests(argc, argv); } odc-1.6.3/tests/api/odc_ls.cc0000664000175000017500000001207215146027420016142 0ustar alastairalastair/** * To build this program, please make sure to reference linked libraries: * * g++ -std=c++11 -leckit -lodccore -o odc-cpp-ls odc_ls.cc */ #include #include #include #include #include #include #include "eckit/runtime/Main.h" #include "odc/api/Odb.h" void usage() { std::cerr << "Usage:\n odc-cpp-ls " << std::endl << std::endl; } void write_header(int i, odc::api::ColumnInfo col) { int padding = col.decodedSize - floor(log10(abs(i + 1))) + 2; std::cout << (i + 1) << ". " << std::left << std::setw(padding) << col.name << "\t"; } void write_data(size_t nrows, size_t ncols, std::vector columnInfo, std::vector strides, std::vector nbits) { long integer_missing = odc::api::Settings::integerMissingValue(); double double_missing = odc::api::Settings::doubleMissingValue(); for (size_t row = 0; row < nrows; ++row) { for (size_t col = 0; col < ncols; ++col) { switch (columnInfo[col].type) { case odc::api::INTEGER: { int64_t val = *reinterpret_cast(strides[col][row]); if (val == integer_missing) { std::cout << "."; } else { std::cout << std::left << std::setw(columnInfo[col].decodedSize) << val; } break; } case odc::api::BITFIELD: { int64_t val = *reinterpret_cast(strides[col][row]); for (int bit = nbits[col] - 1; bit >= 0; --bit) { std::cout << ((val & (1 << bit)) ? "1" : "0"); } break; } case odc::api::REAL: case odc::api::DOUBLE: { double val = *reinterpret_cast(strides[col][row]); if (val == double_missing) { std::cout << "."; } else { std::cout << std::left << std::setw(columnInfo[col].decodedSize) << val; } break; } std::cout << *reinterpret_cast(strides[col][row]); break; case odc::api::STRING: std::cout << std::left << std::setw(columnInfo[col].decodedSize) << std::string(strides[col][row], ::strnlen(strides[col][row], columnInfo[col].decodedSize)); break; default: ASSERT(false); }; std::cout << "\t"; } std::cout << std::endl; } } int main(int argc, char** argv) { if (argc != 2) { usage(); return 1; } char* path = argv[1]; eckit::Main::initialise(argc, argv); // Initialise library odc::api::Settings::treatIntegersAsDoubles(false); // Open the specified ODB file odc::api::Reader reader(path); // Iterate through the frames odc::api::Frame frame; while ((frame = reader.next())) { // Properties of this frame size_t ncols = frame.columnCount(); size_t nrows = frame.rowCount(); const auto& columnInfo = frame.columnInfo(); // Print headers & determine storage requirements size_t row_size = 0; size_t i; for (i = 0; i < frame.columnCount(); ++i) { const auto& col(columnInfo[i]); write_header(i, col); row_size += col.decodedSize; } std::cout << std::endl; // Allocate storage required size_t storage_size = row_size * nrows; std::vector buffer(storage_size); // Decoder prerequisites std::vector columns; std::vector strides; std::vector nbits; char* ptr = &buffer[0]; for (const auto& col : columnInfo) { columns.push_back(col.name); strides.emplace_back(odc::api::StridedData{ptr, nrows, col.decodedSize, col.decodedSize}); ptr += nrows * col.decodedSize; // Bits for formatting bitfields if (col.type == odc::api::BITFIELD) { nbits.push_back(std::accumulate(col.bitfield.begin(), col.bitfield.end(), 0, [](int l, const odc::api::ColumnInfo::Bit& r) { return l + r.size; })); } else { nbits.push_back(0); } } ASSERT(ptr == (&buffer[0] + storage_size)); // Decode the data int nthreads = 4; odc::api::Decoder decoder(columns, strides); decoder.decode(frame, nthreads); // Iterate through the decoded data, and print it write_data(nrows, ncols, columnInfo, strides, nbits); } return 0; } odc-1.6.3/tests/api/odc_encode_custom.cc0000664000175000017500000001373715146027420020364 0ustar alastairalastair/** * To build this program, please make sure to reference linked libraries: * * g++ -std=c++11 -leckit -lodccore -o odc-cpp-encode-custom odc_encode_custom.cc */ #include #include #include #include #include #include "eckit/io/FileHandle.h" #include "eckit/runtime/Main.h" #include "odc/api/Odb.h" using namespace eckit; using namespace odc::api; // Bitfield constants #define Ob00000001 1 #define Ob00001011 11 #define Ob01101011 107 void usage() { std::cerr << "Usage:\n odc-cpp-encode-custom " << std::endl << std::endl; } void cycle_longs(long* list, int size, long* pool, int pool_size) { int index = 0; int i; for (i = 0; i < size; i++) { if (index == pool_size) index = 0; list[i] = pool[index]; index++; } } void cycle_doubles(double* list, int size, double* pool, int pool_size) { int index = 0; int i; for (i = 0; i < size; i++) { if (index == pool_size) index = 0; list[i] = pool[index]; index++; } } void create_scratch_data(size_t nrows, char data0[][8], int64_t data1[], char data2[][8], char data3[][16], double data4[], int64_t data5[], double data6[], int64_t data7[]) { // Prepare the current date as an integer time_t rawtime; time(&rawtime); struct tm* timeinfo; timeinfo = localtime(&rawtime); int64_t date = 10000 * (timeinfo->tm_year + 1900) + 100 * (timeinfo->tm_mon + 1) + timeinfo->tm_mday; // Prepare the list of integer values, including the missing value long integer_pool[] = {1234, 4321, Settings::integerMissingValue()}; int integer_pool_size = sizeof(integer_pool) / sizeof(integer_pool[0]); std::vector missing_integers(nrows); cycle_longs(missing_integers.data(), nrows, integer_pool, integer_pool_size); // Prepare the list of double values, including the missing value double double_pool[] = {12.34, 43.21, Settings::doubleMissingValue()}; int double_pool_size = sizeof(double_pool) / sizeof(double_pool[0]); std::vector missing_doubles(nrows); cycle_doubles(missing_doubles.data(), nrows, double_pool, double_pool_size); // Prepare the list of bitfield values long bitfield_pool[] = {Ob00000001, Ob00001011, Ob01101011}; int bitfield_pool_size = sizeof(bitfield_pool) / sizeof(bitfield_pool[0]); std::vector bitfield_values(nrows); cycle_longs(bitfield_values.data(), nrows, bitfield_pool, bitfield_pool_size); // Fill in the passed data arrays with scratch values for (size_t i = 0; i < nrows; i++) { ASSERT(snprintf(data0[i], 8, "xxxx") == 4); // expver data1[i] = date; // date@hdr ASSERT(snprintf(data2[i], 7, "stat%02ld", i) == 6); // statid@hdr ASSERT(snprintf(data3[i], 16, "0-12345-0-678%02ld", i) == 15); // wigos@hdr data4[i] = 12.3456 * i; // obsvalue@body data5[i] = missing_integers[i]; // integer_missing data6[i] = missing_doubles[i]; // double_missing data7[i] = bitfield_values[i]; // bitfield_column } } int main(int argc, char** argv) { if (argc != 2) { usage(); return 1; } // Get output path from command argument char* path = argv[1]; // Initialise API Main::initialise(argc, argv); Settings::treatIntegersAsDoubles(false); // Define row count const size_t nrows = 20; // Allocate data array for each column char data0[nrows][8]; int64_t data1[nrows]; char data2[nrows][8]; char data3[nrows][16]; double data4[nrows]; int64_t data5[nrows]; double data6[nrows]; int64_t data7[nrows]; // Set up the allocated arrays with scratch data create_scratch_data(nrows, data0, data1, data2, data3, data4, data5, data6, data7); // Column `bitfield_column` is an integer with 4 bitfield values in it std::vector bitfields = { // name, size, offset+=size(n-1) {"flag_a", 1, 0}, {"flag_b", 2, 1}, {"flag_c", 3, 3}, {"flag_d", 1, 6}, }; // Define all column names, their types and sizes std::vector columns = { ColumnInfo{std::string("expver"), ColumnType(STRING), 8}, ColumnInfo{std::string("date@hdr"), ColumnType(INTEGER), sizeof(int64_t)}, ColumnInfo{std::string("statid@hdr"), ColumnType(STRING), 8}, ColumnInfo{std::string("wigos@hdr"), ColumnType(STRING), 16}, ColumnInfo{std::string("obsvalue@body"), ColumnType(REAL), sizeof(double)}, ColumnInfo{std::string("integer_missing"), ColumnType(INTEGER), sizeof(int64_t)}, ColumnInfo{std::string("double_missing"), ColumnType(REAL), sizeof(double)}, ColumnInfo{std::string("bitfield_column"), ColumnType(BITFIELD), sizeof(int64_t), bitfields}, }; // Set a custom data layout and data array for each column std::vector strides{ // ptr, nrows, element_size, stride {data0, nrows, 8, 8}, {data1, nrows, sizeof(int64_t), sizeof(int64_t)}, {data2, nrows, 8, 8}, {data3, nrows, 16, 16}, {data4, nrows, sizeof(double), sizeof(double)}, {data5, nrows, sizeof(int64_t), sizeof(int64_t)}, {data6, nrows, sizeof(double), sizeof(double)}, {data7, nrows, sizeof(int64_t), sizeof(int64_t)}, }; // Add some key/value metadata to the frame std::map properties = { {"encoded_by", "odc_example"}, }; const Length length; FileHandle fh(path); fh.openForWrite(length); AutoClose closer(fh); // Encode ODB-2 into a data handle encode(fh, columns, strides, properties); std::cout << "Written " << nrows << " to " << path << std::endl; return 0; } odc-1.6.3/tests/api/usage_examples.sh0000775000175000017500000001330315146027420017727 0ustar alastairalastair#!/bin/bash set -uex # A unique working directory wd=$(pwd) test_wd=$(pwd)/usage_examples mkdir -p ${test_wd} cd ${test_wd} # In case we are resuming from a previous failed run, which has left output in the directory rm *.csv *.odb *.txt || true # Create some test data cat > span.csv < encode-custom-usage.txt || exit_code=$? ; expect_error cat > encode-custom-usage-expected.txt < EOF cmp encode-custom-usage.txt encode-custom-usage-expected.txt ${wd}/odc-cpp-encode-custom ${test_wd}/test.odb > encode-custom.txt && exit_code=$? ; expect_success cat > encode-custom-expected.txt < index-usage.txt || exit_code=$? ; expect_error cat > index-usage-expected.txt < EOF cmp index-usage.txt index-usage-expected.txt ${wd}/odc-cpp-index ${test_wd}/span.odb > index.txt && exit_code=$? ; expect_success cat > index-expected.txt < ls-usage.txt || exit_code=$? ; expect_error cat > ls-usage-expected.txt < EOF cmp ls-usage.txt ls-usage-expected.txt ${wd}/odc-cpp-ls ${test_wd}/test.odb > ls.txt && exit_code=$? ; expect_success current_date=$(date +"%Y%m%d") cat > ls-expected.txt < header-usage.txt || exit_code=$? ; expect_error cat > header-usage-expected.txt < [ ...] EOF cmp header-usage.txt header-usage-expected.txt ${wd}/odc-cpp-header ${test_wd}/test.odb ${test_wd}/span.odb > header.txt && exit_code=$? ; expect_success odc_version=$(odc --version | head -n 1 | sed "s/^ODBAPI Version: \([0-9.]\)/\1/") cat > header-expected.txt < odc_example Property: encoder => odc version $odc_version Column: 1, Name: expver, Type: string, Size: 8 Column: 2, Name: date@hdr, Type: integer, Size: 8 Column: 3, Name: statid@hdr, Type: string, Size: 8 Column: 4, Name: wigos@hdr, Type: string, Size: 16 Column: 5, Name: obsvalue@body, Type: real, Size: 8 Column: 6, Name: integer_missing, Type: integer, Size: 8 Column: 7, Name: double_missing, Type: real, Size: 8 Column: 8, Name: bitfield_column, Type: bitfield, Size: 8 Bitfield: 1, Name: flag_a, Offset: 0, Nbits: 1 Bitfield: 2, Name: flag_b, Offset: 1, Nbits: 2 Bitfield: 3, Name: flag_c, Offset: 3, Nbits: 3 Bitfield: 4, Name: flag_d, Offset: 6, Nbits: 1 File: $test_wd/span.odb Frame: 1, Row count: 9, Column count: 5 Column: 1, Name: key1, Type: integer, Size: 8 Column: 2, Name: key2, Type: integer, Size: 8 Column: 3, Name: key3, Type: string, Size: 8 Column: 4, Name: val1, Type: integer, Size: 8 Column: 5, Name: val2, Type: real, Size: 8 EOF cmp header.txt header-expected.txt # Clean up cd ${wd} rm -rf ${test_wd} odc-1.6.3/regressions/0000775000175000017500000000000015146027420015016 5ustar alastairalastairodc-1.6.3/regressions/SD-45479.sh0000775000175000017500000000065515146027420016363 0ustar alastairalastair#!/bin/bash set -uex # A unique working directory wd=$(pwd) test_wd=$(pwd)/test_sd_45479 mkdir -p ${test_wd} cd ${test_wd} odc sql "select statid, datum_status.active@body, datum_status.passive@body, datum_status.blacklisted@body" -i ../sd_45479_source.odb -o temporary.odb -f odb # And check that we made a change, and reverted it odc compare ../sd_45479_compare.odb temporary.odb # Clean up cd ${wd} rm -rf ${test_wd} odc-1.6.3/regressions/ODB-387-and-388.sh0000775000175000017500000000302515146027420017360 0ustar alastairalastair#!/bin/bash set -uex # A unique working directory wd=$(pwd) test_wd=$(pwd)/test_odb_387_388 mkdir -p ${test_wd} cd ${test_wd} # Test doing a replace odc mdset "expver=' 9876'" ../odb_387_mdset.odb temporary1.odb # Demonstrate that we have changed a column odc compare ../odb_387_mdset.odb temporary1.odb && exit -1 odc compare -excludeColumns expver ../odb_387_mdset.odb temporary1.odb # The file sizes are the same if [[ $(stat -c%s ../odb_387_mdset.odb) -ne $(stat -c%s temporary1.odb) ]]; then if [[ $(stat -c%s temporary1.odb) -ne 59246 ]]; then # n.b. A change. We no longer encode 10x unused flags into ODB files (which are then not used on reading). exit -1 fi fi # Change another column odc mdset "stream=123456" temporary1.odb temporary2.odb # Demonstrate that we can exclude only one, or a range of columns odc compare ../odb_387_mdset.odb temporary1.odb && exit -1 odc compare temporary1.odb temporary2.odb && exit -1 odc compare -excludeColumns stream temporary1.odb temporary2.odb odc compare -excludeColumns stream ../odb_387_mdset.odb temporary2.odb && exit -1 odc compare -excludeColumns stream,expver ../odb_387_mdset.odb temporary2.odb # The file sizes are still the same if [[ $(stat -c%s ../odb_387_mdset.odb) -ne $(stat -c%s temporary2.odb) ]]; then if [[ $(stat -c%s temporary1.odb) -ne 59246 ]]; then # n.b. A change. We no longer encode 10x unused flags into ODB files (which are then not used on reading). exit -1 fi fi # Clean up cd ${wd} rm -rf ${test_wd} odc-1.6.3/regressions/ODB-529.sh0000775000175000017500000000201315146027420016272 0ustar alastairalastair#!/bin/bash set -uex # A unique working directory wd=$(pwd) test_wd=$(pwd)/test_odb_529 mkdir -p ${test_wd} cd ${test_wd} # Select a bitfield from an existing odb odc sql "select status.active@body" -i ../../tests/2000010106-reduced.odb -f odb -o first.odb # Check that appropriate columns are there odc header first.odb | grep INTEGER odc header first.odb | grep "status.active@body" set +e odc header first.odb | grep "status.active.active@body" rc=$? set -e [[ $rc = 0 ]] && echo "status.active.active should not be found in odb file first.odb" && exit 1 # Attempt to select the output of this select odc sql "select status.active@body" -i first.odb -f odb -o second.odb # Check that appropriate columns are there odc header second.odb | grep INTEGER odc header second.odb | grep "status.active@body" set +e odc header second.odb | grep "status.active.active@body" rc=$? set -e [[ $rc = 0 ]] && echo "status.active.active should not be found in odb file second.odb" && exit 1 # Clean up cd ${wd} rm -rf ${test_wd} odc-1.6.3/regressions/ODB-374.sh0000775000175000017500000000105115146027420016271 0ustar alastairalastair#!/bin/bash set -uex # A unique working directory wd=$(pwd) test_wd=$(pwd)/test_odb_374 mkdir -p ${test_wd} cd ${test_wd} # Test doing a replace odc mdset "expver=' 0001'" ../../tests/2000010106-reduced.odb temporary1.odb # And go back to where we were odc mdset "expver='0018 '" temporary1.odb temporary2.odb # And check that we made a change, and reverted it odc compare ../../tests/2000010106-reduced.odb temporary2.odb odc compare ../../tests/2000010106-reduced.odb temporary1.odb && exit -1 # Clean up cd ${wd} rm -rf ${test_wd} odc-1.6.3/regressions/CMakeLists.txt0000664000175000017500000000163115146027420017557 0ustar alastairalastair list( APPEND regressions_data_files odb_387_mdset.odb odb_463.odb sd_45315-reduced.odb sd_45479_source.odb sd_45479_compare.odb ) ecbuild_get_test_multidata( TARGET odc_get_regressions_data #DIRNAME odc/tests NAMES ${regressions_data_files} NOCHECK ) set(regressions_scripts ODB-374.sh ODB-387-and-388.sh ODB-463.sh SD-45315.sh SD-45479.sh ODB-529.sh ) foreach( _script ${regressions_scripts}) ecbuild_add_test( TYPE script COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/${_script} ENVIRONMENT PATH=${CMAKE_BINARY_DIR}/bin:$ENV{PATH} TEST_DEPENDS odc_get_test_data odc_get_regressions_data ) endforeach() # Add a custom target to make these files visible in qtcreator add_custom_target( regressions_file_visibility SOURCES ${regressions_scripts} ) odc-1.6.3/regressions/SD-45315.sh0000775000175000017500000000110315146027420016335 0ustar alastairalastair#!/bin/bash set -uex # A unique working directory wd=$(pwd) test_wd=$(pwd)/test_sd_45315 mkdir -p ${test_wd} cd ${test_wd} # Test running an SQL command that # i) Selects fields # ii) Implicitly and explicitly uses tables odc sql "select anflag.final, anflag.varqc, anflag,status.active@hdr, status.passive@hdr, status.active@body, status.passive@body where statid=' 00247'" -i ../../tests/2000010106-reduced.odb -o temporary.odb # And check that we made a change, and reverted it odc compare ../sd_45315-reduced.odb temporary.odb # Clean up cd ${wd} rm -rf ${test_wd} odc-1.6.3/regressions/ODB-463.sh0000775000175000017500000000443115146027420016275 0ustar alastairalastair#!/bin/bash set -uex # A unique working directory wd=$(pwd) test_wd=$(pwd)/test_odb_463 mkdir -p ${test_wd} cd ${test_wd} # Test doing a split operation odc split ../odb_463.odb ./out.{reportype}.odb # Check that all of the expected files are created [[ "$(ls out.*.odb | wc -l)" -ne "21" ]] && echo "Incorrect number of files created by split" && exit -1 for rt in 16001 16002 16004 16005 16006 16008 16009 16012 16017 16022 16025 16026 16029 16045 16065 16068 16074 16076 16082 16083 16084; do [[ ! -f "out.${rt}.odb" ]] && "expected split output 'out.${rt}.odb' not found" && exit -1 done cat > expected_wigosid < reduced_sql for f in out.*.odb; do odc sql 'select distinct wigosid' -i $f; done | sed 's/ *$//g' | sort -u > split_sql diff reduced_sql expected_wigosid diff split_sql expected_wigosid # Clean up cd ${wd} rm -rf ${test_wd} odc-1.6.3/src/0000775000175000017500000000000015146027420013242 5ustar alastairalastairodc-1.6.3/src/odc_config.h.in0000664000175000017500000000101215146027420016104 0ustar alastairalastair#ifndef odc_config_h #define odc_config_h #include "odc_ecbuild_config.h" // generated by ecbuild_generate_config_headers() #define odc_VERSION_STR "@odc_VERSION_STR@" #define odc_VERSION "@odc_VERSION@" #define odc_VERSION_MAJOR @odc_VERSION_MAJOR@ #define odc_VERSION_MINOR @odc_VERSION_MINOR@ #define odc_VERSION_PATCH @odc_VERSION_PATCH@ #define odc_INSTALL_PREFIX "@CMAKE_INSTALL_PREFIX@" #define odc_BINARY_DIR "@CMAKE_BINARY_DIR@" #define odc_GIT_SHA1 "@odc_GIT_SHA1@" #endif // odc_config_h odc-1.6.3/src/CMakeLists.txt0000664000175000017500000000073415146027420016006 0ustar alastairalastair## check if all sources are used ecbuild_find_project_files() ### config header ecbuild_generate_config_headers( DESTINATION ${INSTALL_INCLUDE_DIR}/odc ) configure_file( odc_config.h.in odc_config.h ) install(FILES ${CMAKE_CURRENT_BINARY_DIR}/odc_config.h DESTINATION ${INSTALL_INCLUDE_DIR}/odc ) ### libs add_subdirectory( odc ) add_subdirectory( odc/tools ) ### fortran bindings -- OFF for the moment due to errors with gfortran 6.3 / 7.3 add_subdirectory( fortran ) odc-1.6.3/src/fortran/0000775000175000017500000000000015146027420014715 5ustar alastairalastairodc-1.6.3/src/fortran/legacy_test_fortran_api_open_non_existing_file.f900000664000175000017500000000322315146027420026730 0ustar alastairalastair! (C) Copyright 1996-2012 ECMWF. ! ! This software is licensed under the terms of the Apache Licence Version 2.0 ! which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. ! In applying this licence, ECMWF does not waive the privileges and immunities ! granted to it by virtue of its status as an intergovernmental organisation nor ! does it submit to any jurisdiction. ! program test_fortran_api use, intrinsic :: iso_c_binding use odc_c_binding implicit none integer, parameter :: max_varlen = 128 integer(kind=4) :: ncolumns = 4 write(0,*) "Calling odb_start..." call odb_start() call test_fortran_api_non_existing_file() contains subroutine test_fortran_api_non_existing_file implicit none type(C_PTR) :: odb_handle, odb_it, odb_select_handle, odb_select_it integer(kind=C_INT) :: cerr character(kind=C_CHAR, len=64) :: config = C_NULL_CHAR character(kind=C_CHAR, len=max_varlen) :: inputfile = "test_fortran_api_non_existing_file.odb"//achar(0) character(kind=C_CHAR, len=128) :: sql='select * from "test_fortran_api_non_existing_file.odb";'//achar(0) write(0,*) 'test_fortran_api_non_existing_file' odb_handle = odb_read_new(config, cerr) odb_it = odb_read_iterator_new(odb_handle, inputfile, cerr); ! This should fail write(0,*) 'test_fortran_api_non_existing_file: odb_read_iterator_new => ', cerr if (cerr == 0) STOP 1 odb_handle = odb_select_new(config, cerr) odb_it = odb_select_iterator_new(odb_handle, sql, cerr); if (cerr == 0) STOP 1 end subroutine test_fortran_api_non_existing_file end program test_fortran_api odc-1.6.3/src/fortran/legacy_fortran_api_examples.f900000664000175000017500000004760415146027420022776 0ustar alastairalastair! (C) Copyright 1996-2012 ECMWF. ! ! This software is licensed under the terms of the Apache Licence Version 2.0 ! which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. ! In applying this licence, ECMWF does not waive the privileges and immunities ! granted to it by virtue of its status as an intergovernmental organisation nor ! does it submit to any jurisdiction. ! program example_fortran_api use, intrinsic :: iso_c_binding use odc_c_binding implicit none integer, parameter :: max_varlen = 128 integer(kind=4), parameter :: ncolumns = 6 ! Correct values for checking integer(c_int), dimension(ncolumns), parameter:: column_types = (/ODB_INTEGER, ODB_REAL, ODB_BITFIELD, & ODB_STRING, ODB_STRING, ODB_DOUBLE/) character(8), dimension(ncolumns), parameter :: column_names = (/'ifoo ', 'nbar ', 'status ', 'wigos ', & 'expver ', 'dbar '/) integer(c_int), dimension(ncolumns), parameter:: column_offsets = (/1, 2, 3, 4, 7, 8/) integer(c_int), dimension(ncolumns), parameter:: column_sizes = (/1, 1, 1, 3, 1, 1/) integer, parameter :: row_size_doubles = 8 interface function strlen(s) result(l) bind(c, name='strlen') use, intrinsic :: iso_c_binding character(c_char) :: s integer(c_long) :: l end function end interface write(0,*) "Calling odb_start..." call odb_start() call example_fortran_api_setup() call example_fortran_api1() call example_fortran_api2() call example_fortran_api_append() contains subroutine example_fortran_api_append implicit none type(C_PTR) :: odb_handle, odb_it integer(kind=C_INT) :: cerr character(kind=C_CHAR, len=max_varlen) :: config = C_NULL_CHAR character(kind=C_CHAR, len=max_varlen) :: outputfile="example_fortran_api_append.odb"//achar(0) integer(kind=4) :: i integer(kind=C_INT) :: itype, c_ncolumns real(kind=C_DOUBLE), dimension(:), allocatable:: one_row integer(kind=c_int) :: offsets(ncolumns) integer(kind=c_int) :: row_length_doubles character(len=100) :: expver="fihn"//achar(0) character(len=100) :: wigos="this-is-a-long-string"//achar(0) write(0,*) 'example_fortran_api_append' c_ncolumns = ncolumns odb_handle = odb_write_new(config, cerr) odb_it = odb_write_iterator_new(odb_handle, outputfile, cerr); cerr = odb_write_set_no_of_columns(odb_it, ncolumns) if (cerr == 0) cerr = odb_write_set_column(odb_it, 0, ODB_INTEGER, "ifoo"//achar(0)) if (cerr == 0) cerr = odb_write_set_column(odb_it, 1, ODB_REAL, "nbar"//achar(0)) if (cerr == 0) cerr = odb_write_set_bitfield(odb_it, 2, ODB_BITFIELD, "status"//achar(0), & "active:passive:blacklisted:"//achar(0), & "1:1:4:"//achar(0)) if (cerr == 0) cerr = odb_write_set_column(odb_it, 3, ODB_STRING, "wigos"//achar(0)) if (cerr == 0) cerr = odb_write_set_column_size_doubles(odb_it, 3, 4); if (cerr == 0) cerr = odb_write_set_column(odb_it, 4, ODB_STRING, "expver"//achar(0)) if (cerr == 0) cerr = odb_write_set_column(odb_it, 5, ODB_DOUBLE, "dbar"//achar(0)) if (cerr == 0) cerr = odb_write_set_missing_value(odb_it, 0, 1.0_8) if (cerr == 0) cerr = odb_write_header(odb_it) if (cerr == 0) cerr = odb_write_get_row_buffer_size_doubles(odb_it, row_length_doubles) do i = 1, ncolumns if (cerr == 0) cerr = odb_write_get_column_offset(odb_it, i-1, offsets(i)) enddo if (cerr /= 0) stop 1 ! Sanity check! if (row_length_doubles /= 9) stop 1 if (any(offsets /= (/1, 2, 3, 4, 8, 9/))) stop 1 allocate(one_row(row_length_doubles)) do i=1,10 one_row(offsets(1)) = i one_row(offsets(2)) = i one_row(offsets(3)) = 5 one_row(offsets(4):offsets(4)+3) = transfer(wigos(1:32), one_row(offsets(4):offsets(4)+3)) one_row(offsets(5)) = transfer(expver, one_row(5)) one_row(offsets(6)) = 5 cerr = odb_write_set_next_row(odb_it, one_row, c_ncolumns) if (cerr /= 0) stop 1 enddo deallocate(one_row) cerr = odb_write_iterator_delete(odb_it) if (cerr == 0) odb_it = odb_append_iterator_new(odb_handle, outputfile, cerr); if (cerr == 0) cerr = odb_write_set_no_of_columns(odb_it, ncolumns) if (cerr == 0) cerr = odb_write_set_column(odb_it, 0, ODB_INTEGER, "ifoo"//achar(0)) if (cerr == 0) cerr = odb_write_set_column(odb_it, 1, ODB_REAL, "nbar"//achar(0)) if (cerr == 0) cerr = odb_write_set_bitfield(odb_it, 2, ODB_BITFIELD, "status"//achar(0), & "active:passive:blacklisted:"//achar(0), & "1:1:4:"//achar(0)) if (cerr == 0) cerr = odb_write_set_column(odb_it, 3, ODB_STRING, "wigos"//achar(0)) if (cerr == 0) cerr = odb_write_set_column_size_doubles(odb_it, 3, 4); if (cerr == 0) cerr = odb_write_set_column(odb_it, 4, ODB_STRING, "expver"//achar(0)) if (cerr == 0) cerr = odb_write_set_column(odb_it, 5, ODB_DOUBLE, "dbar"//achar(0)) if (cerr == 0) cerr = odb_write_set_missing_value(odb_it, 0, 1.0_8) if (cerr == 0) cerr = odb_write_header(odb_it) if (cerr == 0) cerr = odb_write_get_row_buffer_size_doubles(odb_it, row_length_doubles) do i = 1, ncolumns if (cerr == 0) cerr = odb_write_get_column_offset(odb_it, i-1, offsets(i)) enddo if (cerr /= 0) stop 1 ! Sanity check! if (row_length_doubles /= 9) stop 1 if (any(offsets /= (/1, 2, 3, 4, 8, 9/))) stop 1 allocate(one_row(row_length_doubles)) do i=1,10 one_row(offsets(1)) = i one_row(offsets(2)) = i one_row(offsets(3)) = 5 one_row(offsets(4):offsets(4)+3) = transfer(wigos(1:32), one_row(offsets(4):offsets(4)+3)) one_row(offsets(5)) = transfer(expver, one_row(5)) one_row(offsets(6)) = 5 cerr = odb_write_set_next_row(odb_it, one_row, c_ncolumns) if (cerr /= 0) stop 1 enddo deallocate(one_row) cerr = odb_write_iterator_delete(odb_it) if (cerr == 0) cerr = odb_write_delete(odb_handle) if (cerr /= 0) stop 1 end subroutine example_fortran_api_append subroutine example_fortran_api_setup implicit none type(C_PTR) :: odb_handle, odb_it integer(kind=C_INT) :: cerr character(kind=C_CHAR, len=max_varlen) :: config = C_NULL_CHAR character(kind=C_CHAR, len=max_varlen) :: outputfile="test.odb"//achar(0) integer(kind=4) :: i integer(kind=C_INT) :: itype, c_ncolumns real(kind=C_DOUBLE), dimension(:), allocatable:: one_row integer(kind=c_int) :: offsets(ncolumns) integer(kind=c_int) :: row_length_doubles character(len=100) :: expver="fihn"//achar(0) character(len=100) :: wigos="this-is-a-long-string"//achar(0) write(0,*) 'example_fortran_api_setup' c_ncolumns = ncolumns odb_handle = odb_write_new(config, cerr) odb_it = odb_write_iterator_new(odb_handle, outputfile, cerr); cerr = odb_write_set_no_of_columns(odb_it, ncolumns) if (cerr == 0) cerr = odb_write_set_column(odb_it, 0, ODB_INTEGER, "ifoo"//achar(0)) if (cerr == 0) cerr = odb_write_set_column(odb_it, 1, ODB_REAL, "nbar"//achar(0)) if (cerr == 0) cerr = odb_write_set_bitfield(odb_it, 2, ODB_BITFIELD, "status"//achar(0), & "active:passive:blacklisted:"//achar(0), & "1:1:4:"//achar(0)) if (cerr == 0) cerr = odb_write_set_column(odb_it, 3, ODB_STRING, "wigos"//achar(0)) if (cerr == 0) cerr = odb_write_set_column_size_doubles(odb_it, 3, 4); if (cerr == 0) cerr = odb_write_set_column(odb_it, 4, ODB_STRING, "expver"//achar(0)) if (cerr == 0) cerr = odb_write_set_column(odb_it, 5, ODB_DOUBLE, "dbar"//achar(0)) if (cerr == 0) cerr = odb_write_set_missing_value(odb_it, 0, 1.0_8) if (cerr == 0) cerr = odb_write_header(odb_it) if (cerr == 0) cerr = odb_write_get_row_buffer_size_doubles(odb_it, row_length_doubles) do i = 1, ncolumns if (cerr == 0) cerr = odb_write_get_column_offset(odb_it, i-1, offsets(i)) enddo if (cerr /= 0) stop 1 ! Sanity check! if (row_length_doubles /= 9) stop 1 if (any(offsets /= (/1, 2, 3, 4, 8, 9/))) stop 1 allocate(one_row(row_length_doubles)) do i=1,10 one_row(offsets(1)) = i one_row(offsets(2)) = i one_row(offsets(3)) = 5 one_row(offsets(4):offsets(4)+3) = transfer(wigos(1:32), one_row(offsets(4):offsets(4)+3)) one_row(offsets(5)) = transfer(expver, one_row(5)) one_row(offsets(6)) = 5 cerr = odb_write_set_next_row(odb_it, one_row, c_ncolumns) if (cerr /= 0) stop 1 enddo deallocate(one_row) cerr = odb_write_iterator_delete(odb_it) if (cerr == 0) cerr = odb_write_delete(odb_handle) if (cerr /= 0) stop 1 end subroutine example_fortran_api_setup subroutine example_fortran_api1 ! This function tests the reader api implicit none type(C_PTR) :: odb_handle, odb_it integer(kind=C_INT) :: cerr character(kind=C_CHAR, len=max_varlen) :: config = C_NULL_CHAR character(kind=C_CHAR, len=max_varlen) :: inputfile = "test.odb"//achar(0) type(C_PTR) :: ptr_colname type(C_PTR) :: ptr_bitfield_names type(C_PTR) :: ptr_bitfield_sizes character(kind=C_CHAR,len=1), dimension(:), pointer :: f_ptr_colname character(kind=C_CHAR,len=1), dimension(:), pointer :: f_ptr_bitfield_names character(kind=C_CHAR,len=1), dimension(:), pointer :: f_ptr_bitfield_sizes character(len=max_varlen) :: colname character(len=max_varlen) :: bitfield_names character(len=max_varlen) :: bitfield_sizes integer(kind=4) :: i integer(kind=C_INT) :: itype, newdataset, c_ncolumns=2, size_name integer(kind=C_INT) :: bitfield_names_size, bitfield_sizes_size real(kind=C_DOUBLE), dimension(:), allocatable:: one_row real(kind=C_DOUBLE) :: val integer(c_int) :: isize, ioffset character(len=24) :: tmp_str1 character(len=8) :: tmp_str2 integer :: col write(0,*) 'example_fortran_api1' odb_handle = odb_read_new(config, cerr) odb_it = odb_read_iterator_new(odb_handle, inputfile, cerr); if (cerr /= 0) STOP 1 ! Total number of columns cerr = odb_read_get_no_of_columns(odb_it, c_ncolumns) if (cerr /=0) stop 2 if (c_ncolumns /= ncolumns) stop 3 ! Total amount of data cerr = odb_read_get_row_buffer_size_doubles(odb_it, isize) if (cerr /= 0) stop 4 if (isize /= row_size_doubles) stop 5 ! n.b. 9 in writer, but only needed 3 columns to encode strings! ! Check the values reported per column ... do col = 1, ncolumns cerr = odb_read_get_column_name(odb_it, col-1, ptr_colname,size_name) if (cerr == 0) cerr = odb_read_get_column_type(odb_it, col-1, itype) if (cerr == 0) cerr = odb_read_get_column_offset(odb_it, col-1, ioffset) if (cerr == 0) cerr = odb_read_get_column_size_doubles(odb_it, col-1, isize) if (cerr /= 0) stop 6 ! Check the column names call c_f_pointer(CPTR=ptr_colname, FPTR=f_ptr_colname, shape=(/size_name/)); do i=1, size_name colname(i:i) = f_ptr_colname(i) end do write(0,'(a,i1,3a,i1,a,i1,a)') 'column name ', col, ' : ', colname(1:size_name), & ' [', ioffset, ', ', isize, ']' ! Do we get the expected values? if (colname(1:size_name) /= trim(column_names(col))) stop 7 if (itype /= column_types(col)) stop 9 if (ioffset /= column_offsets(col)) stop 11 if (isize /= column_sizes(col)) stop 12 end do ! Test modified missing value val = -13 cerr = odb_read_get_missing_value(odb_it, 0, val) if (cerr /=0) STOP 13 write(0,*) 'odb_read_get_missing_value: missing value of column 0 => ', val if (val /= 1.0) stop 14 ! Test the contents of the bitfields cerr = odb_read_get_bitfield(odb_it, 2, ptr_bitfield_names, ptr_bitfield_sizes, bitfield_names_size, bitfield_sizes_size) write(0,*) 'odb_read_get_bitfield column 2 => ', cerr if (cerr /=0) STOP 15 write(0,*) 'column 2 bitfield_names_size: ', bitfield_names_size call C_F_POINTER(CPTR=ptr_bitfield_names, FPTR=f_ptr_bitfield_names, shape=(/bitfield_names_size/)); do i=1, bitfield_names_size bitfield_names(i:i) = f_ptr_bitfield_names(i) end do write(0,*) 'column 2 bitfield_names: ', bitfield_names(1:bitfield_names_size) if (bitfield_names(1:bitfield_names_size) /= 'active:passive:blacklisted:') STOP 16 write(0,*) 'column 2 bitfield_sizes_size: ', bitfield_sizes_size call C_F_POINTER(CPTR=ptr_bitfield_sizes, FPTR=f_ptr_bitfield_sizes, shape=(/bitfield_sizes_size/)); do i=1, bitfield_sizes_size bitfield_sizes(i:i) = f_ptr_bitfield_sizes(i) end do write(0,*) 'column 2 bitfield_sizes: ', bitfield_sizes(1:bitfield_sizes_size) if (bitfield_sizes(1:bitfield_sizes_size) /= '1:1:4:') STOP 17 ! Test the contents of the data section! allocate(one_row(row_size_doubles)) do i = 1, 100 cerr = odb_read_get_next_row(odb_it, c_ncolumns, one_row, newdataset) if (cerr /= 0) exit ! Extract the strings tmp_str1(1:24) = transfer(one_row(column_offsets(4):column_offsets(5)-1), tmp_str1(1:24)) tmp_str2(1:8) = transfer(one_row(column_offsets(5)), tmp_str2(1:8)) write(0,*) i, ":", one_row(column_offsets(1)), & one_row(column_offsets(2)), & one_row(column_offsets(3)), & tmp_str1(1:24), " ", & tmp_str2(1:8), & one_row(column_offsets(6)) if (one_row(column_offsets(1)) /= i) stop 18 if (one_row(column_offsets(2)) /= i) stop 19 if (one_row(column_offsets(3)) /= 5) stop 20 if (trim(tmp_str1(1:strlen(tmp_str1))) /= 'this-is-a-long-string') stop 21 if (trim(tmp_str2(1:strlen(tmp_str2))) /= 'fihn') stop 22 if (one_row(column_offsets(6)) /= 5) stop 23 enddo deallocate(one_row) ! Did we get the correct number of rows? if (i /= 11) stop 24 ! Clean up cerr = odb_read_iterator_delete(odb_it) cerr = odb_read_delete(odb_handle) end subroutine example_fortran_api1 subroutine example_fortran_api2 implicit none type(C_PTR) :: odb_handle, odb_it integer(kind=C_INT) :: cerr character(kind=C_CHAR, len=64) :: config = C_NULL_CHAR type(C_PTR) :: ptr_colname type(C_PTR) :: ptr_bitfield_names type(C_PTR) :: ptr_bitfield_sizes character(kind=C_CHAR), dimension(:), pointer :: f_ptr_colname character(kind=C_CHAR,len=1), dimension(:), pointer :: f_ptr_bitfield_names character(kind=C_CHAR,len=1), dimension(:), pointer :: f_ptr_bitfield_sizes character(len=max_varlen) :: colname character(len=max_varlen) :: bitfield_names character(len=max_varlen) :: bitfield_sizes integer(kind=4) :: i character(kind=C_CHAR, len=128) :: sql='select * from "test.odb"'//achar(0) integer(kind=C_INT) :: itype, newdataset, c_ncolumns=3, size_name integer(kind=C_INT) :: bitfield_names_size, bitfield_sizes_size, ioffset, isize real(kind=C_DOUBLE), dimension(:), allocatable:: one_row real(kind=c_double) :: bitfield_missing character(len=64) :: tmp_str1, tmp_str2 integer :: col write(0,*) 'example_fortran_api2' odb_handle = odb_select_new(config, cerr) if (cerr == 0) odb_it = odb_select_iterator_new(odb_handle, sql, cerr); if (cerr /= 0) STOP 25 ! Total number of columns cerr = odb_select_get_no_of_columns(odb_it, c_ncolumns) if (cerr /=0) STOP 26 if (c_ncolumns /= ncolumns) stop 27 ! Total amount of data cerr = odb_select_get_row_buffer_size_doubles(odb_it, isize) if (cerr /= 0) stop 28 if (isize /= row_size_doubles) stop 29 ! n.b. 9 in writer, but only needed 3 columns to encode strings! ! Check the values reported per column ... do col = 1, ncolumns cerr = odb_select_get_column_name(odb_it, col-1, ptr_colname,size_name) if (cerr == 0) cerr = odb_select_get_column_type(odb_it, col-1, itype) if (cerr == 0) cerr = odb_select_get_column_offset(odb_it, col-1, ioffset) if (cerr == 0) cerr = odb_select_get_column_size_doubles(odb_it, col-1, isize) if (cerr /= 0) stop 30 ! Check the column names call c_f_pointer(CPTR=ptr_colname, FPTR=f_ptr_colname, shape=(/size_name/)); do i=1, size_name colname(i:i) = f_ptr_colname(i) end do write(0,'(a,i1,3a,i1,a,i1,a)') 'column name ', col, ' : ', colname(1:size_name), & ' [', ioffset, ', ', isize, ']' ! Do we get the expected values write(9,*) itype, column_types(col) if (colname(1:size_name) /= trim(column_names(col))) stop 31 if (itype /= column_types(col)) stop 32 if (ioffset /= column_offsets(col)) stop 33 if (isize /= column_sizes(col)) stop 34 end do ! Test the contents of the bitfields cerr = odb_select_get_bitfield(odb_it, 2, ptr_bitfield_names, ptr_bitfield_sizes, bitfield_names_size, bitfield_sizes_size) write(0,*) 'odb_select_get_bitfield column 2 => ', cerr if (cerr /=0) STOP 35 write(0,*) 'column 2 bitfield_names_size: ', bitfield_names_size call C_F_POINTER(CPTR=ptr_bitfield_names, FPTR=f_ptr_bitfield_names, shape=(/bitfield_names_size/)); do i=1, bitfield_names_size bitfield_names(i:i) = f_ptr_bitfield_names(i) end do write(0,*) 'column 2 bitfield_names: ', bitfield_names(1:bitfield_names_size) if (bitfield_names(1:bitfield_names_size) /= 'active:passive:blacklisted:') STOP 36 write(0,*) 'column 2 bitfield_sizes_size: ', bitfield_sizes_size call C_F_POINTER(CPTR=ptr_bitfield_sizes, FPTR=f_ptr_bitfield_sizes, shape=(/bitfield_sizes_size/)); do i=1, bitfield_sizes_size bitfield_sizes(i:i) = f_ptr_bitfield_sizes(i) end do write(0,*) 'column 2 bitfield_sizes: ', bitfield_sizes(1:bitfield_sizes_size) if (bitfield_sizes(1:bitfield_sizes_size) /= '1:1:4:') STOP 37 ! Test the contents of the data section! cerr = odb_select_get_missing_value(odb_it, 0, bitfield_missing) if (cerr /= 0) stop 45 if (bitfield_missing /= 2147483647) stop 46 allocate(one_row(row_size_doubles)) do i = 1, 100 cerr = odb_select_get_next_row(odb_it, c_ncolumns, one_row, newdataset) if ( cerr /= 0) exit ! Extract the strings tmp_str1(1:24) = transfer(one_row(column_offsets(4):column_offsets(5)-1), tmp_str1(1:24)) tmp_str2(1:8) = transfer(one_row(column_offsets(5)), tmp_str2(1:8)) write(0, *) column_offsets write(0,*) i, ":", one_row(column_offsets(1)), & one_row(column_offsets(2)), & one_row(column_offsets(3)), & tmp_str1(1:24), " ", & tmp_str2(1:8), & one_row(column_offsets(6)) if (i == 1) then if (one_row(column_offsets(1)) /= bitfield_missing) stop 47 else if (one_row(column_offsets(1)) /= i) stop 39 end if if (one_row(column_offsets(2)) /= i) stop 40 if (one_row(column_offsets(3)) /= 5) stop 41 if (trim(tmp_str1(1:strlen(tmp_str1))) /= 'this-is-a-long-string') stop 42 if (trim(tmp_str2(1:strlen(tmp_str2))) /= 'fihn') stop 43 if (one_row(column_offsets(6)) /= 5) stop 44 enddo deallocate(one_row) ! Did we get the correct number of rows? if (i /= 11) stop 24 ! Clean up cerr = odb_select_iterator_delete(odb_it) cerr = odb_select_delete(odb_handle) end subroutine example_fortran_api2 end program example_fortran_api odc-1.6.3/src/fortran/CMakeLists.txt0000664000175000017500000000274215146027420017462 0ustar alastairalastair ecbuild_add_library(TARGET odc_fortran CONDITION HAVE_FORTRAN SOURCES # Legacy: odc_c_binding.f90 PUBLIC_LIBS odccore PUBLIC_INCLUDES $ $ ) set( test_environment_fortran TEST_DHSHOME=${PROJECT_SOURCE_DIR}/tests/dhshome/ ) if( HAVE_FORTRAN ) list( APPEND fortran_tests # Legacy API examples legacy_fortran_api_examples legacy_test_fortran_api_open_non_existing_file ) foreach( _test ${fortran_tests} ) ecbuild_add_test( TARGET odc_${_test} CONDITION HAVE_FORTRAN SOURCES ${_test}.f90 LIBS odc_fortran ENVIRONMENT ${test_environment_fortran} LABELS odc odc_fortran TEST_DEPENDS odc_get_test_data odc_get_mars_client_test_data_mo odc_get_mars_client_test_data_ec LINKER_LANGUAGE Fortran ) endforeach() # Legacy Fortran bindings: install( FILES ${CMAKE_Fortran_MODULE_DIRECTORY}/${CMAKE_CFG_INTDIR}/odc_c_binding.mod DESTINATION module/odc COMPONENT modules ) endif() odc-1.6.3/src/fortran/legacy_test_client_lib_fortran_local.ecml0000664000175000017500000000031415146027420025151 0ustar alastairalastairtest, label = Execute test_client_lib_fortran_local, do = ( sh, _ = (getenv, values = MARS_INSTALLATION_DIRECTORY)/"/bin/legacy_test_client_lib_fortran_local" sequence, values = OK ), expect = OK odc-1.6.3/src/fortran/odc_c_binding.f900000664000175000017500000005137415146027420020010 0ustar alastairalastair! (C) Copyright 1996-2012 ECMWF. ! ! This software is licensed under the terms of the Apache Licence Version 2.0 ! which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. ! In applying this licence, ECMWF does not waive the privileges and immunities ! granted to it by virtue of its status as an intergovernmental organisation nor ! does it submit to any jurisdiction. ! !------------------------------------------------------------------------------ ! ECMWF !------------------------------------------------------------------------------ ! ! MODULE: odc_c_binding ! !> @author Anne Fouilloux, ECMWF ! ! DESCRIPTION: !> Provides Fortran bindings for ODB API ! ! REVISION HISTORY: ! DD Mmm YYYY - Initial Version ! TODO_dd_mmm_yyyy - TODO_describe_appropriate_changes - TODO_name !------------------------------------------------------------------------------ module odc_c_binding use iso_c_binding use, intrinsic :: iso_c_binding implicit none integer, parameter :: ODB_IGNORE=0 integer, parameter :: ODB_INTEGER=1 integer, parameter :: ODB_REAL=2 integer, parameter :: ODB_STRING=3 integer, parameter :: ODB_BITFIELD=4 integer, parameter :: ODB_DOUBLE=5 interface !> Initialize ODB API. This function must be called before any other function from the ODB API. subroutine odb_start() bind(C, name = "odb_start") end subroutine !> Counts number of rows in an ODB file. !> !> @param[in] filename name of the file !> @return number of rows on file function odb_count(filename) bind(C, name="odb_count") use, intrinsic :: iso_c_binding character(kind=C_CHAR),dimension(*) :: filename real(kind=C_DOUBLE) :: odb_count end function odb_count function odb_select_new(config, err) bind(C, name = "odb_select_create") use, intrinsic :: iso_c_binding character(kind=C_CHAR),dimension(*) :: config integer(kind=C_INT) :: err type(C_PTR) :: odb_select_new end function odb_select_new function odb_select_delete(odb) bind(C, name = "odb_select_destroy") use, intrinsic :: iso_c_binding type(C_PTR),VALUE :: odb integer(kind=C_INT) :: odb_select_delete end function odb_select_delete function odb_read_new(config, err) bind(C, name = "odb_read_create") use, intrinsic :: iso_c_binding character(kind=C_CHAR),dimension(*) :: config integer(kind=C_INT) :: err type(C_PTR) :: odb_read_new end function odb_read_new function odb_read_delete(odb) bind(C, name = "odb_read_destroy") use, intrinsic :: iso_c_binding type(C_PTR),VALUE :: odb integer(kind=C_INT) :: odb_read_delete end function odb_read_delete !> Create new read iterator. function odb_read_iterator_new(odb, filename, err) bind(C, name="odb_create_read_iterator") use, intrinsic :: iso_c_binding type(C_PTR), VALUE :: odb character(kind=C_CHAR),dimension(*) :: filename integer(kind=C_INT) :: err type(C_PTR) :: odb_read_iterator_new end function odb_read_iterator_new function odb_read_iterator_delete(odb_iterator) bind(C, name="odb_read_iterator_destroy") use, intrinsic :: iso_c_binding type(C_PTR), VALUE :: odb_iterator integer(kind=C_INT) :: odb_read_iterator_delete end function odb_read_iterator_delete function odb_read_get_no_of_columns(odb_iterator, ncols) bind(C, name="odb_read_iterator_get_no_of_columns") use, intrinsic :: iso_c_binding type(C_PTR),VALUE :: odb_iterator integer(kind=C_INT) :: ncols integer(kind=C_INT) :: odb_read_get_no_of_columns ! return an error code end function odb_read_get_no_of_columns function odb_read_get_column_type(odb_iterator, n, type) bind(C, name="odb_read_iterator_get_column_type") use, intrinsic :: iso_c_binding type(C_PTR), VALUE :: odb_iterator integer(kind=C_INT),VALUE :: n integer(kind=C_INT) :: type integer(kind=C_INT) :: odb_read_get_column_type end function odb_read_get_column_type function odb_read_get_column_name(odb_iterator, n, colname, nchar) bind(C, name="odb_read_iterator_get_column_name") use, intrinsic :: iso_c_binding type(C_PTR), VALUE :: odb_iterator integer(kind=C_INT), VALUE :: n integer(kind=C_INT) :: nchar type(C_PTR) :: colname integer(kind=C_INT) :: odb_read_get_column_name end function odb_read_get_column_name function odb_read_get_bitfield(odb_iterator, n, bitfield_names, bitfield_sizes, bitfield_names_size, bitfield_sizes_size) & bind(C, name="odb_read_iterator_get_bitfield") use, intrinsic :: iso_c_binding type(C_PTR), VALUE :: odb_iterator integer(kind=C_INT), VALUE :: n type(C_PTR) :: bitfield_names type(C_PTR) :: bitfield_sizes integer(kind=C_INT) :: bitfield_names_size integer(kind=C_INT) :: bitfield_sizes_size integer(kind=C_INT) :: odb_read_get_bitfield end function odb_read_get_bitfield function odb_read_get_next_row(odb_iterator, count, data, new_dataset) bind(C, name="odb_read_iterator_get_next_row") use, intrinsic :: iso_c_binding type(C_PTR), VALUE :: odb_iterator integer(kind=C_INT), VALUE :: count integer(kind=C_INT) :: new_dataset real(kind=C_DOUBLE),dimension(*) :: data integer(kind=C_INT) :: odb_read_get_next_row end function odb_read_get_next_row ! odb_read_iterator_get_missing_value(oda_read_iterator_ptr ri, int index, double* value) function odb_read_get_missing_value(odb_iterator, n, v) & bind(C, name="odb_read_iterator_get_missing_value") use, intrinsic :: iso_c_binding type(C_PTR), VALUE :: odb_iterator integer(kind=C_INT),VALUE :: n real(kind=C_DOUBLE) :: v integer(kind=C_INT) :: odb_read_get_missing_value end function odb_read_get_missing_value function odb_read_get_row_buffer_size_doubles(odb_iterator, sz) & result(cerr) & bind(C, name="odb_read_iterator_get_row_buffer_size_doubles") use, intrinsic :: iso_c_binding type(c_ptr), value :: odb_iterator integer(kind=c_int) :: sz integer(kind=c_int) :: cerr end function function odb_read_get_column_offset_internal(odb_iterator, n, offset) & result(cerr) & bind(C, name="odb_read_iterator_get_column_offset") use, intrinsic :: iso_c_binding type(c_ptr), value :: odb_iterator integer(kind=c_int), value :: n integer(kind=c_int) :: offset integer(kind=c_int) :: cerr end function function odb_read_get_column_size_doubles(odb_iterator, n, sz) & result(cerr) & bind(C, name="odb_read_iterator_get_column_size_doubles") use, intrinsic :: iso_c_binding type(c_ptr), value :: odb_iterator integer(kind=c_int), value :: n integer(kind=c_int) :: sz integer(kind=c_int) :: cerr end function ! SELECT ITERATOR function odb_select_iterator_new(odb, sql, err) bind(C, name="odb_create_select_iterator") use, intrinsic :: iso_c_binding type(C_PTR), VALUE :: odb character(kind=C_CHAR),dimension(*) :: sql integer(kind=C_INT) :: err type(C_PTR) :: odb_select_iterator_new end function odb_select_iterator_new function odb_select_iterator_new_from_file(odb, sql, filename, err) bind(C, name="odb_create_select_iterator_from_file") use, intrinsic :: iso_c_binding type(C_PTR), VALUE :: odb character(kind=C_CHAR),dimension(*) :: sql, filename integer(kind=C_INT) :: err type(C_PTR) :: odb_select_iterator_new_from_file end function odb_select_iterator_new_from_file function odb_select_iterator_delete(odb_iterator) bind(C, name="odb_select_iterator_destroy") use, intrinsic :: iso_c_binding type(C_PTR), VALUE :: odb_iterator integer(kind=C_INT) :: odb_select_iterator_delete end function odb_select_iterator_delete function odb_select_get_no_of_columns(odb_iterator, ncols) bind(C, name="odb_select_iterator_get_no_of_columns") use, intrinsic :: iso_c_binding type(C_PTR),VALUE :: odb_iterator integer(kind=C_INT) :: ncols integer(kind=C_INT) :: odb_select_get_no_of_columns ! return an error code end function odb_select_get_no_of_columns function odb_select_get_row_buffer_size_doubles(odb_iterator, sz) & result(cerr) & bind(C, name="odb_select_iterator_get_row_buffer_size_doubles") use, intrinsic :: iso_c_binding type(c_ptr), value :: odb_iterator integer(kind=c_int) :: sz integer(kind=c_int) :: cerr end function function odb_select_get_column_offset_internal(odb_iterator, n, offset) & result(cerr) & bind(C, name="odb_select_iterator_get_column_offset") use, intrinsic :: iso_c_binding type(c_ptr), value :: odb_iterator integer(kind=c_int), value :: n integer(kind=c_int) :: offset integer(kind=c_int) :: cerr end function function odb_select_get_column_size_doubles(odb_iterator, n, sz) & result(cerr) & bind(C, name="odb_select_iterator_get_column_size_doubles") use, intrinsic :: iso_c_binding type(c_ptr), value :: odb_iterator integer(kind=c_int), value :: n integer(kind=c_int) :: sz integer(kind=c_int) :: cerr end function function odb_select_get_missing_value(odb_iterator, n, v) & bind(C, name="odb_select_iterator_get_missing_value") use, intrinsic :: iso_c_binding type(C_PTR), VALUE :: odb_iterator integer(kind=C_INT),VALUE :: n real(kind=C_DOUBLE) :: v integer(kind=C_INT) :: odb_select_get_missing_value end function odb_select_get_missing_value function odb_select_get_column_type(odb_iterator, n, type) bind(C, name="odb_select_iterator_get_column_type") use, intrinsic :: iso_c_binding type(C_PTR), VALUE :: odb_iterator integer(kind=C_INT),VALUE :: n integer(kind=C_INT) :: type integer(kind=C_INT) :: odb_select_get_column_type end function odb_select_get_column_type function odb_select_get_column_name(odb_iterator, n, colname, nchar) bind(C, name="odb_select_iterator_get_column_name") use, intrinsic :: iso_c_binding type(C_PTR), VALUE :: odb_iterator integer(kind=C_INT), VALUE :: n type(C_PTR) :: colname integer(kind=C_INT) :: nchar integer(kind=C_INT) :: odb_select_get_column_name end function odb_select_get_column_name function odb_select_get_bitfield(odb_iterator, n, bitfield_names, bitfield_sizes, bitfield_names_size, bitfield_sizes_size) & bind(C, name="odb_select_iterator_get_bitfield") use, intrinsic :: iso_c_binding type(C_PTR), VALUE :: odb_iterator integer(kind=C_INT), VALUE :: n type(C_PTR) :: bitfield_names type(C_PTR) :: bitfield_sizes integer(kind=C_INT) :: bitfield_names_size integer(kind=C_INT) :: bitfield_sizes_size integer(kind=C_INT) :: odb_select_get_bitfield end function odb_select_get_bitfield function odb_select_get_next_row(odb_iterator, count, data, new_dataset) bind(C, name="odb_select_iterator_get_next_row") use, intrinsic :: iso_c_binding type(C_PTR), VALUE :: odb_iterator integer(kind=C_INT), VALUE :: count integer(kind=C_INT) :: new_dataset real(kind=C_DOUBLE),dimension(*) :: data integer(kind=C_INT) :: odb_select_get_next_row end function odb_select_get_next_row ! WRITE function odb_write_new(config, err) bind(C, name = "odb_writer_create") use, intrinsic :: iso_c_binding character(kind=C_CHAR),dimension(*) :: config integer(kind=C_INT) :: err type(C_PTR) :: odb_write_new end function odb_write_new function odb_write_delete(odb) bind(C, name = "odb_writer_destroy") use, intrinsic :: iso_c_binding type(C_PTR),VALUE :: odb integer(kind=C_INT) :: odb_write_delete end function odb_write_delete ! WRITE iterator function odb_write_iterator_new(odb, filename, err) bind(C, name="odb_create_write_iterator") use, intrinsic :: iso_c_binding type(C_PTR), VALUE :: odb character(kind=C_CHAR),dimension(*) :: filename integer(kind=C_INT) :: err type(C_PTR) :: odb_write_iterator_new end function odb_write_iterator_new function odb_append_iterator_new(odb, filename, err) bind(C, name="odb_create_append_iterator") use, intrinsic :: iso_c_binding type(C_PTR), VALUE :: odb character(kind=C_CHAR),dimension(*) :: filename integer(kind=C_INT) :: err type(C_PTR) :: odb_append_iterator_new end function odb_append_iterator_new function odb_write_iterator_delete(odb_iterator) bind(C, name="odb_write_iterator_destroy") use, intrinsic :: iso_c_binding type(C_PTR), VALUE :: odb_iterator integer(kind=C_INT) :: odb_write_iterator_delete end function odb_write_iterator_delete function odb_write_set_no_of_columns(odb_iterator, ncols) bind(C, name="odb_write_iterator_set_no_of_columns") use, intrinsic :: iso_c_binding type(C_PTR),VALUE :: odb_iterator integer(kind=C_INT), VALUE :: ncols integer(kind=C_INT) :: odb_write_set_no_of_columns ! return an error code end function odb_write_set_no_of_columns function odb_write_set_column(odb_iterator, n, type, colname) bind(C, name="odb_write_iterator_set_column") use, intrinsic :: iso_c_binding type(C_PTR), VALUE :: odb_iterator integer(kind=C_INT),VALUE :: n integer(kind=C_INT),VALUE :: type character(kind=C_CHAR),dimension(*) :: colname integer(kind=C_INT) :: odb_write_set_column end function odb_write_set_column function odb_write_set_bitfield(odb_iterator, n, type, colname, bitfield_names, bitfield_sizes) & bind(C, name="odb_write_iterator_set_bitfield") use, intrinsic :: iso_c_binding type(C_PTR), VALUE :: odb_iterator integer(kind=C_INT),VALUE :: n integer(kind=C_INT),VALUE :: type character(kind=C_CHAR),dimension(*) :: colname character(kind=C_CHAR),dimension(*) :: bitfield_names, bitfield_sizes integer(kind=C_INT) :: odb_write_set_bitfield end function odb_write_set_bitfield function odb_write_set_missing_value(odb_iterator, n, v) bind(C, name="odb_write_iterator_set_missing_value") use, intrinsic :: iso_c_binding type(C_PTR), VALUE :: odb_iterator integer(kind=C_INT),VALUE :: n real(kind=C_DOUBLE),VALUE :: v integer(kind=C_INT) :: odb_write_set_missing_value end function odb_write_set_missing_value function odb_write_set_column_size_doubles(odb_iterator, n, sz) & result(cerr) & bind(C, name="odb_write_iterator_set_column_size_doubles") use, intrinsic :: iso_c_binding type(c_ptr), value :: odb_iterator integer(kind=c_int), value :: n integer(kind=c_int), value :: sz integer(kind=c_int) :: cerr end function function odb_write_get_row_buffer_size_doubles(odb_iterator, sz) & result(cerr) & bind(C, name="odb_write_iterator_get_row_buffer_size_doubles") use, intrinsic :: iso_c_binding type(c_ptr), value :: odb_iterator integer(kind=c_int) :: sz integer(kind=c_int) :: cerr end function function odb_write_get_column_offset_internal(odb_iterator, n, offset) & result(cerr) & bind(C, name="odb_write_iterator_get_column_offset") use, intrinsic :: iso_c_binding type(c_ptr), value :: odb_iterator integer(kind=c_int), value :: n integer(kind=c_int) :: offset integer(kind=c_int) :: cerr end function function odb_write_set_next_row(odb_iterator, data, count) bind(C, name="odb_write_iterator_set_next_row") use, intrinsic :: iso_c_binding type(C_PTR), VALUE :: odb_iterator integer(kind=C_INT), VALUE :: count real(kind=C_DOUBLE),dimension(*) :: data integer(kind=C_INT) :: odb_write_set_next_row end function odb_write_set_next_row function odb_write_header(odb_iterator) bind(C, name="odb_write_iterator_write_header") use, intrinsic :: iso_c_binding type(C_PTR), VALUE :: odb_iterator integer(kind=C_INT) :: odb_write_header end function odb_write_header end interface contains function odb_read_get_column_offset(odb_iterator, n, offset) result(cerr) ! Map between C (0-index) and Fortran (1-index) offsets use, intrinsic :: iso_c_binding type(c_ptr), value :: odb_iterator integer(kind=c_int), value :: n integer(kind=c_int) :: offset integer(kind=c_int) :: cerr cerr = odb_read_get_column_offset_internal(odb_iterator, n, offset) offset = offset + 1 end function function odb_select_get_column_offset(odb_iterator, n, offset) result(cerr) ! Map between C (0-index) and Fortran (1-index) offsets use, intrinsic :: iso_c_binding type(c_ptr), value :: odb_iterator integer(kind=c_int), value :: n integer(kind=c_int) :: offset integer(kind=c_int) :: cerr cerr = odb_select_get_column_offset_internal(odb_iterator, n, offset) offset = offset + 1 end function function odb_write_get_column_offset(odb_iterator, n, offset) result(cerr) ! Map between C (0-index) and Fortran (1-index) offsets use, intrinsic :: iso_c_binding type(c_ptr), value :: odb_iterator integer(kind=c_int), value :: n integer(kind=c_int) :: offset integer(kind=c_int) :: cerr cerr = odb_write_get_column_offset_internal(odb_iterator, n, offset) offset = offset + 1 end function end module odc_c_binding odc-1.6.3/src/fortran/legacy_test_client_lib_fortran_local.f900000664000175000017500000001447315146027420024642 0ustar alastairalastair! (C) Copyright 1996-2012 ECMWF. ! ! This software is licensed under the terms of the Apache Licence Version 2.0 ! which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. ! In applying this licence, ECMWF does not waive the privileges and immunities ! granted to it by virtue of its status as an intergovernmental organisation nor ! does it submit to any jurisdiction. ! program test_client_lib_fortran_local use, intrinsic :: iso_c_binding use odc_c_binding implicit none integer, parameter :: max_varlen = 128 integer(kind=4) :: ncolumns write(0,*) "Calling odb_start..." call odb_start() call example_fortran_api_stage_local() call example_fortran_api_retrieve_part_local() call example_fortran_api_stage_local() call example_fortran_api_retrieve_part_local() contains subroutine example_fortran_api_stage_local implicit none type(C_PTR) :: odb_handle, odb_it integer(kind=C_INT) :: cerr character(kind=C_CHAR, len=64) :: config = C_NULL_CHAR type(C_PTR) :: ptr_colname type(C_PTR) :: ptr_bitfield_names type(C_PTR) :: ptr_bitfield_sizes character(kind=C_CHAR), dimension(:), pointer :: f_ptr_colname character(kind=C_CHAR,len=1), dimension(:), pointer :: f_ptr_bitfield_names character(kind=C_CHAR,len=1), dimension(:), pointer :: f_ptr_bitfield_sizes character(len=max_varlen) :: colname character(len=max_varlen) :: bitfield_names character(len=max_varlen) :: bitfield_sizes integer(kind=4) :: i, ci character(kind=C_CHAR, len=2048) :: sql='select partition_number,number_of_rows from & &"local://stage,class=OD,date=20151108,time=1200,type=OFB,& &obsgroup=conv,reportype=16030,stream=oper,expver=qu12,& &odbpathnameschema=''{date}/{time}/{reportype}.odb'',& &odbserverroots=''~/data/root'',& &partitionsinfo=''~/data/partitions_info.txt'',& &n_parts=13,& &database=localhost";'//achar(0) integer(kind=C_INT) :: itype, newdataset, c_ncolumns=2, size_name integer(kind=C_INT) :: bitfield_names_size, bitfield_sizes_size real(kind=C_DOUBLE), dimension(:), allocatable:: one_row character(len=8) :: tmp_str write(0,*) 'example_fortran_api_stage_local: ', sql odb_handle = odb_select_new(config, cerr) if (cerr /=0) STOP 1 odb_it = odb_select_iterator_new(odb_handle, sql, cerr); if (cerr /=0) STOP 2 cerr = odb_select_get_no_of_columns(odb_it, ncolumns) if (cerr /=0) STOP 1 write(0,*) '-=-=-=-=-= example_fortran_api_stage_local: number of columns: ', ncolumns if (ncolumns /= 2) STOP 2 do ci=0, ncolumns - 1 cerr = odb_select_get_column_name(odb_it, ci, ptr_colname, size_name) if (cerr /=0) STOP 1 call C_F_POINTER(CPTR=ptr_colname, FPTR=f_ptr_colname, shape=(/size_name/)); do i=1, size_name colname(i:i) = f_ptr_colname(i) end do write(0,*) ' : ', colname(1:i) end do allocate(one_row(c_ncolumns)) cerr=0 i = 0 do cerr = odb_select_get_next_row(odb_it, c_ncolumns, one_row, newdataset) if ( cerr /= 0) exit i = i + 1 enddo deallocate(one_row) write(0,*) '-=-=-=-=-= example_fortran_api_stage_local: number of rows: ', i if (i /= 13) STOP 22 cerr = odb_select_iterator_delete(odb_it) cerr = odb_read_delete(odb_handle) end subroutine example_fortran_api_stage_local subroutine example_fortran_api_retrieve_part_local implicit none type(C_PTR) :: odb_handle, odb_it integer(kind=C_INT) :: cerr character(kind=C_CHAR, len=64) :: config = C_NULL_CHAR type(C_PTR) :: ptr_colname type(C_PTR) :: ptr_bitfield_names type(C_PTR) :: ptr_bitfield_sizes character(kind=C_CHAR), dimension(:), pointer :: f_ptr_colname character(kind=C_CHAR,len=1), dimension(:), pointer :: f_ptr_bitfield_names character(kind=C_CHAR,len=1), dimension(:), pointer :: f_ptr_bitfield_sizes character(len=max_varlen) :: colname character(len=max_varlen) :: bitfield_names character(len=max_varlen) :: bitfield_sizes integer(kind=4) :: i, ci character(kind=C_CHAR, len=2048) :: sql='select * from & & "local://retrieve,class=OD,date=20151108,time=1200,type=OFB,& & obsgroup=conv,reportype=16030,stream=oper,expver=qu12,& & odbpathnameschema=''{date}/{time}/{reportype}.odb'',& & odbserverroots=''~/data/root'',& & partitionsinfo=''~/data/partitions_info.txt'',& & n_parts=13,& & part_number=0,& & database=localhost";'//achar(0) integer(kind=C_INT) :: itype, newdataset, c_ncolumns=52, size_name integer(kind=C_INT) :: bitfield_names_size, bitfield_sizes_size real(kind=C_DOUBLE), dimension(:), allocatable:: one_row character(len=8) :: tmp_str write(0,*) 'example_fortran_api_retrieve_part_local: ', sql odb_handle = odb_select_new(config, cerr) if (cerr /=0) STOP 1 odb_it = odb_select_iterator_new(odb_handle, sql, cerr); if (cerr /=0) STOP 1 cerr = odb_select_get_no_of_columns(odb_it, ncolumns) if (cerr /=0) STOP 1 write(0,*) '-=-=-=-=-= example_fortran_api_retrieve_part_local: number of columns: ', ncolumns if (c_ncolumns /= ncolumns) STOP 2 do ci=0, ncolumns - 1 cerr = odb_select_get_column_name(odb_it, ci, ptr_colname, size_name) if (cerr /=0) STOP 1 call C_F_POINTER(CPTR=ptr_colname, FPTR=f_ptr_colname, shape=(/size_name/)); do i=1, size_name colname(i:i) = f_ptr_colname(i) end do write(0,*) ' : ', colname(1:i) end do allocate(one_row(c_ncolumns)) cerr=0 i = 0 do cerr = odb_select_get_next_row(odb_it, c_ncolumns, one_row, newdataset) if ( cerr /= 0) exit i = i + 1 enddo deallocate(one_row) write(0,*) '-=-=-=-=-= example_fortran_api_retrieve_part_local: number of rows: ', i if (i /= 9938) STOP 22 cerr = odb_select_iterator_delete(odb_it) cerr = odb_read_delete(odb_handle) end subroutine example_fortran_api_retrieve_part_local end program test_client_lib_fortran_local odc-1.6.3/src/fortran/odbapi.fortran.doxyfile0000664000175000017500000015143215146027420021400 0ustar alastairalastair# Doxyfile 1.5.3 # This file describes the settings to be used by the documentation system # doxygen (www.doxygen.org) for a project # # All text after a hash (#) is considered a comment and will be ignored # The format is: # TAG = value [value, ...] # For lists items can also be appended using: # TAG += value [value, ...] # Values that contain spaces should be placed between quotes (" ") #--------------------------------------------------------------------------- # Project related configuration options #--------------------------------------------------------------------------- # This tag specifies the encoding used for all characters in the config file that # follow. The default is UTF-8 which is also the encoding used for all text before # the first occurrence of this tag. Doxygen uses libiconv (or the iconv built into # libc) for the transcoding. See http://www.gnu.org/software/libiconv for the list of # possible encodings. DOXYFILE_ENCODING = UTF-8 # The PROJECT_NAME tag is a single word (or a sequence of words surrounded # by quotes) that should identify the project. PROJECT_NAME = "ODB API Fortran bindings" # The PROJECT_NUMBER tag can be used to enter a project or revision number. # This could be handy for archiving the generated documentation or # if some version control system is used. PROJECT_NUMBER = # The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) # base path where the generated documentation will be put. # If a relative path is entered, it will be relative to the location # where doxygen was started. If left blank the current directory will be used. OUTPUT_DIRECTORY = ./dox # If the CREATE_SUBDIRS tag is set to YES, then doxygen will create # 4096 sub-directories (in 2 levels) under the output directory of each output # format and will distribute the generated files over these directories. # Enabling this option can be useful when feeding doxygen a huge amount of # source files, where putting all generated files in the same directory would # otherwise cause performance problems for the file system. CREATE_SUBDIRS = NO # The OUTPUT_LANGUAGE tag is used to specify the language in which all # documentation generated by doxygen is written. Doxygen will use this # information to generate all constant output in the proper language. # The default language is English, other supported languages are: # Afrikaans, Arabic, Brazilian, Catalan, Chinese, Chinese-Traditional, # Croatian, Czech, Danish, Dutch, Finnish, French, German, Greek, Hungarian, # Italian, Japanese, Japanese-en (Japanese with English messages), Korean, # Korean-en, Lithuanian, Norwegian, Polish, Portuguese, Romanian, Russian, # Serbian, Slovak, Slovene, Spanish, Swedish, and Ukrainian. OUTPUT_LANGUAGE = English # If the BRIEF_MEMBER_DESC tag is set to YES (the default) Doxygen will # include brief member descriptions after the members that are listed in # the file and class documentation (similar to JavaDoc). # Set to NO to disable this. BRIEF_MEMBER_DESC = YES # If the REPEAT_BRIEF tag is set to YES (the default) Doxygen will prepend # the brief description of a member or function before the detailed description. # Note: if both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the # brief descriptions will be completely suppressed. REPEAT_BRIEF = YES # This tag implements a quasi-intelligent brief description abbreviator # that is used to form the text in various listings. Each string # in this list, if found as the leading text of the brief description, will be # stripped from the text and the result after processing the whole list, is # used as the annotated text. Otherwise, the brief description is used as-is. # If left blank, the following values are used ("$name" is automatically # replaced with the name of the entity): "The $name class" "The $name widget" # "The $name file" "is" "provides" "specifies" "contains" # "represents" "a" "an" "the" ABBREVIATE_BRIEF = # If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then # Doxygen will generate a detailed section even if there is only a brief # description. ALWAYS_DETAILED_SEC = NO # If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all # inherited members of a class in the documentation of that class as if those # members were ordinary class members. Constructors, destructors and assignment # operators of the base classes will not be shown. INLINE_INHERITED_MEMB = NO # If the FULL_PATH_NAMES tag is set to YES then Doxygen will prepend the full # path before files name in the file list and in the header files. If set # to NO the shortest path that makes the file name unique will be used. FULL_PATH_NAMES = YES # If the FULL_PATH_NAMES tag is set to YES then the STRIP_FROM_PATH tag # can be used to strip a user-defined part of the path. Stripping is # only done if one of the specified strings matches the left-hand part of # the path. The tag can be used to show relative paths in the file list. # If left blank the directory from which doxygen is run is used as the # path to strip. STRIP_FROM_PATH = . # The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of # the path mentioned in the documentation of a class, which tells # the reader which header file to include in order to use a class. # If left blank only the name of the header file containing the class # definition is used. Otherwise one should specify the include paths that # are normally passed to the compiler using the -I flag. STRIP_FROM_INC_PATH = # If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter # (but less readable) file names. This can be useful is your file systems # doesn't support long names like on DOS, Mac, or CD-ROM. SHORT_NAMES = NO # If the JAVADOC_AUTOBRIEF tag is set to YES then Doxygen # will interpret the first line (until the first dot) of a JavaDoc-style # comment as the brief description. If set to NO, the JavaDoc # comments will behave just like regular Qt-style comments # (thus requiring an explicit @brief command for a brief description.) JAVADOC_AUTOBRIEF = NO # If the QT_AUTOBRIEF tag is set to YES then Doxygen will # interpret the first line (until the first dot) of a Qt-style # comment as the brief description. If set to NO, the comments # will behave just like regular Qt-style comments (thus requiring # an explicit \brief command for a brief description.) QT_AUTOBRIEF = NO # The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make Doxygen # treat a multi-line C++ special comment block (i.e. a block of //! or /// # comments) as a brief description. This used to be the default behaviour. # The new default is to treat a multi-line C++ comment block as a detailed # description. Set this tag to YES if you prefer the old behaviour instead. MULTILINE_CPP_IS_BRIEF = NO # If the DETAILS_AT_TOP tag is set to YES then Doxygen # will output the detailed description near the top, like JavaDoc. # If set to NO, the detailed description appears after the member # documentation. DETAILS_AT_TOP = NO # If the INHERIT_DOCS tag is set to YES (the default) then an undocumented # member inherits the documentation from any documented member that it # re-implements. INHERIT_DOCS = YES # If the SEPARATE_MEMBER_PAGES tag is set to YES, then doxygen will produce # a new page for each member. If set to NO, the documentation of a member will # be part of the file/class/namespace that contains it. SEPARATE_MEMBER_PAGES = NO # The TAB_SIZE tag can be used to set the number of spaces in a tab. # Doxygen uses this value to replace tabs by spaces in code fragments. TAB_SIZE = 4 # This tag can be used to specify a number of aliases that acts # as commands in the documentation. An alias has the form "name=value". # For example adding "sideeffect=\par Side Effects:\n" will allow you to # put the command \sideeffect (or @sideeffect) in the documentation, which # will result in a user-defined paragraph with heading "Side Effects:". # You can put \n's in the value part of an alias to insert newlines. ALIASES = # Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C # sources only. Doxygen will then generate output that is more tailored for C. # For instance, some of the names that are used will be different. The list # of all members will be omitted, etc. OPTIMIZE_OUTPUT_FOR_C = NO OPTIMIZE_FOR_FORTRAN = YES # Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java # sources only. Doxygen will then generate output that is more tailored for Java. # For instance, namespaces will be presented as packages, qualified scopes # will look different, etc. OPTIMIZE_OUTPUT_JAVA = NO # If you use STL classes (i.e. std::string, std::vector, etc.) but do not want to # include (a tag file for) the STL sources as input, then you should # set this tag to YES in order to let doxygen match functions declarations and # definitions whose arguments contain STL classes (e.g. func(std::string); v.s. # func(std::string) {}). This also make the inheritance and collaboration # diagrams that involve STL classes more complete and accurate. BUILTIN_STL_SUPPORT = NO # If you use Microsoft's C++/CLI language, you should set this option to YES to # enable parsing support. CPP_CLI_SUPPORT = NO # If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC # tag is set to YES, then doxygen will reuse the documentation of the first # member in the group (if any) for the other members of the group. By default # all members of a group must be documented explicitly. DISTRIBUTE_GROUP_DOC = NO # Set the SUBGROUPING tag to YES (the default) to allow class member groups of # the same type (for instance a group of public functions) to be put as a # subgroup of that type (e.g. under the Public Functions section). Set it to # NO to prevent subgrouping. Alternatively, this can be done per class using # the \nosubgrouping command. SUBGROUPING = YES #--------------------------------------------------------------------------- # Build related configuration options #--------------------------------------------------------------------------- # If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in # documentation are documented, even if no documentation was available. # Private class members and static file members will be hidden unless # the EXTRACT_PRIVATE and EXTRACT_STATIC tags are set to YES EXTRACT_ALL = YES # If the EXTRACT_PRIVATE tag is set to YES all private members of a class # will be included in the documentation. EXTRACT_PRIVATE = YES # If the EXTRACT_STATIC tag is set to YES all static members of a file # will be included in the documentation. EXTRACT_STATIC = YES # If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) # defined locally in source files will be included in the documentation. # If set to NO only classes defined in header files are included. EXTRACT_LOCAL_CLASSES = YES # This flag is only useful for Objective-C code. When set to YES local # methods, which are defined in the implementation section but not in # the interface are included in the documentation. # If set to NO (the default) only methods in the interface are included. EXTRACT_LOCAL_METHODS = NO # If this flag is set to YES, the members of anonymous namespaces will be extracted # and appear in the documentation as a namespace called 'anonymous_namespace{file}', # where file will be replaced with the base name of the file that contains the anonymous # namespace. By default anonymous namespace are hidden. EXTRACT_ANON_NSPACES = YES # If the HIDE_UNDOC_MEMBERS tag is set to YES, Doxygen will hide all # undocumented members of documented classes, files or namespaces. # If set to NO (the default) these members will be included in the # various overviews, but no documentation section is generated. # This option has no effect if EXTRACT_ALL is enabled. HIDE_UNDOC_MEMBERS = NO # If the HIDE_UNDOC_CLASSES tag is set to YES, Doxygen will hide all # undocumented classes that are normally visible in the class hierarchy. # If set to NO (the default) these classes will be included in the various # overviews. This option has no effect if EXTRACT_ALL is enabled. HIDE_UNDOC_CLASSES = NO # If the HIDE_FRIEND_COMPOUNDS tag is set to YES, Doxygen will hide all # friend (class|struct|union) declarations. # If set to NO (the default) these declarations will be included in the # documentation. HIDE_FRIEND_COMPOUNDS = NO # If the HIDE_IN_BODY_DOCS tag is set to YES, Doxygen will hide any # documentation blocks found inside the body of a function. # If set to NO (the default) these blocks will be appended to the # function's detailed documentation block. HIDE_IN_BODY_DOCS = NO # The INTERNAL_DOCS tag determines if documentation # that is typed after a \internal command is included. If the tag is set # to NO (the default) then the documentation will be excluded. # Set it to YES to include the internal documentation. INTERNAL_DOCS = NO # If the CASE_SENSE_NAMES tag is set to NO then Doxygen will only generate # file names in lower-case letters. If set to YES upper-case letters are also # allowed. This is useful if you have classes or files whose names only differ # in case and if your file system supports case sensitive file names. Windows # and Mac users are advised to set this option to NO. CASE_SENSE_NAMES = YES # If the HIDE_SCOPE_NAMES tag is set to NO (the default) then Doxygen # will show members with their full class and namespace scopes in the # documentation. If set to YES the scope will be hidden. HIDE_SCOPE_NAMES = NO # If the SHOW_INCLUDE_FILES tag is set to YES (the default) then Doxygen # will put a list of the files that are included by a file in the documentation # of that file. SHOW_INCLUDE_FILES = YES # If the INLINE_INFO tag is set to YES (the default) then a tag [inline] # is inserted in the documentation for inline members. INLINE_INFO = YES # If the SORT_MEMBER_DOCS tag is set to YES (the default) then doxygen # will sort the (detailed) documentation of file and class members # alphabetically by member name. If set to NO the members will appear in # declaration order. SORT_MEMBER_DOCS = NO # If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the # brief documentation of file, namespace and class members alphabetically # by member name. If set to NO (the default) the members will appear in # declaration order. SORT_BRIEF_DOCS = NO # If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be # sorted by fully-qualified names, including namespaces. If set to # NO (the default), the class list will be sorted only by class name, # not including the namespace part. # Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. # Note: This option applies only to the class list, not to the # alphabetical list. SORT_BY_SCOPE_NAME = NO # The GENERATE_TODOLIST tag can be used to enable (YES) or # disable (NO) the todo list. This list is created by putting \todo # commands in the documentation. GENERATE_TODOLIST = YES # The GENERATE_TESTLIST tag can be used to enable (YES) or # disable (NO) the test list. This list is created by putting \test # commands in the documentation. GENERATE_TESTLIST = YES # The GENERATE_BUGLIST tag can be used to enable (YES) or # disable (NO) the bug list. This list is created by putting \bug # commands in the documentation. GENERATE_BUGLIST = YES # The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or # disable (NO) the deprecated list. This list is created by putting # \deprecated commands in the documentation. GENERATE_DEPRECATEDLIST= YES # The ENABLED_SECTIONS tag can be used to enable conditional # documentation sections, marked by \if sectionname ... \endif. ENABLED_SECTIONS = # The MAX_INITIALIZER_LINES tag determines the maximum number of lines # the initial value of a variable or define consists of for it to appear in # the documentation. If the initializer consists of more lines than specified # here it will be hidden. Use a value of 0 to hide initializers completely. # The appearance of the initializer of individual variables and defines in the # documentation can be controlled using \showinitializer or \hideinitializer # command in the documentation regardless of this setting. MAX_INITIALIZER_LINES = 30 # Set the SHOW_USED_FILES tag to NO to disable the list of files generated # at the bottom of the documentation of classes and structs. If set to YES the # list will mention the files that were used to generate the documentation. SHOW_USED_FILES = YES # If the sources in your project are distributed over multiple directories # then setting the SHOW_DIRECTORIES tag to YES will show the directory hierarchy # in the documentation. The default is NO. SHOW_DIRECTORIES = YES # The FILE_VERSION_FILTER tag can be used to specify a program or script that # doxygen should invoke to get the current version for each file (typically from the # version control system). Doxygen will invoke the program by executing (via # popen()) the command , where is the value of # the FILE_VERSION_FILTER tag, and is the name of an input file # provided by doxygen. Whatever the program writes to standard output # is used as the file version. See the manual for examples. FILE_VERSION_FILTER = #--------------------------------------------------------------------------- # configuration options related to warning and progress messages #--------------------------------------------------------------------------- # The QUIET tag can be used to turn on/off the messages that are generated # by doxygen. Possible values are YES and NO. If left blank NO is used. QUIET = NO # The WARNINGS tag can be used to turn on/off the warning messages that are # generated by doxygen. Possible values are YES and NO. If left blank # NO is used. WARNINGS = YES # If WARN_IF_UNDOCUMENTED is set to YES, then doxygen will generate warnings # for undocumented members. If EXTRACT_ALL is set to YES then this flag will # automatically be disabled. WARN_IF_UNDOCUMENTED = NO # If WARN_IF_DOC_ERROR is set to YES, doxygen will generate warnings for # potential errors in the documentation, such as not documenting some # parameters in a documented function, or documenting parameters that # don't exist or using markup commands wrongly. WARN_IF_DOC_ERROR = YES # This WARN_NO_PARAMDOC option can be abled to get warnings for # functions that are documented, but have no documentation for their parameters # or return value. If set to NO (the default) doxygen will only warn about # wrong or incomplete parameter documentation, but not about the absence of # documentation. WARN_NO_PARAMDOC = NO # The WARN_FORMAT tag determines the format of the warning messages that # doxygen can produce. The string should contain the $file, $line, and $text # tags, which will be replaced by the file and line number from which the # warning originated and the warning text. Optionally the format may contain # $version, which will be replaced by the version of the file (if it could # be obtained via FILE_VERSION_FILTER) WARN_FORMAT = "$file:$line: $text" # The WARN_LOGFILE tag can be used to specify a file to which warning # and error messages should be written. If left blank the output is written # to stderr. WARN_LOGFILE = #--------------------------------------------------------------------------- # configuration options related to the input files #--------------------------------------------------------------------------- # The INPUT tag can be used to specify the files and/or directories that contain # documented source files. You may enter file names like "myfile.cpp" or # directories like "/usr/src/myproject". Separate the files or directories # with spaces. INPUT = . # This tag can be used to specify the character encoding of the source files that # doxygen parses. Internally doxygen uses the UTF-8 encoding, which is also the default # input encoding. Doxygen uses libiconv (or the iconv built into libc) for the transcoding. # See http://www.gnu.org/software/libiconv for the list of possible encodings. INPUT_ENCODING = UTF-8 # If the value of the INPUT tag contains directories, you can use the # FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp # and *.h) to filter out the source-files in the directories. If left # blank the following patterns are tested: # *.c *.cc *.cxx *.cpp *.c++ *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh *.hxx # *.hpp *.h++ *.idl *.odl *.cs *.php *.php3 *.inc *.m *.mm *.py FILE_PATTERNS = # The RECURSIVE tag can be used to turn specify whether or not subdirectories # should be searched for input files as well. Possible values are YES and NO. # If left blank NO is used. RECURSIVE = YES # The EXCLUDE tag can be used to specify files and/or directories that should # excluded from the INPUT source files. This way you can easily exclude a # subdirectory from a directory tree whose root is specified with the INPUT tag. EXCLUDE = # The EXCLUDE_SYMLINKS tag can be used select whether or not files or # directories that are symbolic links (a Unix filesystem feature) are excluded # from the input. EXCLUDE_SYMLINKS = NO # If the value of the INPUT tag contains directories, you can use the # EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude # certain files from those directories. Note that the wildcards are matched # against the file with absolute path, so to exclude all test directories # for example use the pattern */test/* EXCLUDE_PATTERNS = # The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names # (namespaces, classes, functions, etc.) that should be excluded from the output. # The symbol name can be a fully qualified name, a word, or if the wildcard * is used, # a substring. Examples: ANamespace, AClass, AClass::ANamespace, ANamespace::*Test EXCLUDE_SYMBOLS = # The EXAMPLE_PATH tag can be used to specify one or more files or # directories that contain example code fragments that are included (see # the \include command). EXAMPLE_PATH = # If the value of the EXAMPLE_PATH tag contains directories, you can use the # EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp # and *.h) to filter out the source-files in the directories. If left # blank all files are included. EXAMPLE_PATTERNS = # If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be # searched for input files to be used with the \include or \dontinclude # commands irrespective of the value of the RECURSIVE tag. # Possible values are YES and NO. If left blank NO is used. EXAMPLE_RECURSIVE = NO # The IMAGE_PATH tag can be used to specify one or more files or # directories that contain image that are included in the documentation (see # the \image command). IMAGE_PATH = # The INPUT_FILTER tag can be used to specify a program that doxygen should # invoke to filter for each input file. Doxygen will invoke the filter program # by executing (via popen()) the command , where # is the value of the INPUT_FILTER tag, and is the name of an # input file. Doxygen will then use the output that the filter program writes # to standard output. If FILTER_PATTERNS is specified, this tag will be # ignored. INPUT_FILTER = # The FILTER_PATTERNS tag can be used to specify filters on a per file pattern # basis. Doxygen will compare the file name with each pattern and apply the # filter if there is a match. The filters are a list of the form: # pattern=filter (like *.cpp=my_cpp_filter). See INPUT_FILTER for further # info on how filters are used. If FILTER_PATTERNS is empty, INPUT_FILTER # is applied to all files. FILTER_PATTERNS = # If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using # INPUT_FILTER) will be used to filter the input files when producing source # files to browse (i.e. when SOURCE_BROWSER is set to YES). FILTER_SOURCE_FILES = NO #--------------------------------------------------------------------------- # configuration options related to source browsing #--------------------------------------------------------------------------- # If the SOURCE_BROWSER tag is set to YES then a list of source files will # be generated. Documented entities will be cross-referenced with these sources. # Note: To get rid of all source code in the generated output, make sure also # VERBATIM_HEADERS is set to NO. If you have enabled CALL_GRAPH or CALLER_GRAPH # then you must also enable this option. If you don't then doxygen will produce # a warning and turn it on anyway SOURCE_BROWSER = YES # Setting the INLINE_SOURCES tag to YES will include the body # of functions and classes directly in the documentation. INLINE_SOURCES = YES # Setting the STRIP_CODE_COMMENTS tag to YES (the default) will instruct # doxygen to hide any special comment blocks from generated source code # fragments. Normal C and C++ comments will always remain visible. STRIP_CODE_COMMENTS = NO # If the REFERENCED_BY_RELATION tag is set to YES (the default) # then for each documented function all documented # functions referencing it will be listed. REFERENCED_BY_RELATION = YES # If the REFERENCES_RELATION tag is set to YES (the default) # then for each documented function all documented entities # called/used by that function will be listed. REFERENCES_RELATION = YES # If the REFERENCES_LINK_SOURCE tag is set to YES (the default) # and SOURCE_BROWSER tag is set to YES, then the hyperlinks from # functions in REFERENCES_RELATION and REFERENCED_BY_RELATION lists will # link to the source code. Otherwise they will link to the documentstion. REFERENCES_LINK_SOURCE = YES # If the USE_HTAGS tag is set to YES then the references to source code # will point to the HTML generated by the htags(1) tool instead of doxygen # built-in source browser. The htags tool is part of GNU's global source # tagging system (see http://www.gnu.org/software/global/global.html). You # will need version 4.8.6 or higher. USE_HTAGS = NO # If the VERBATIM_HEADERS tag is set to YES (the default) then Doxygen # will generate a verbatim copy of the header file for each class for # which an include is specified. Set to NO to disable this. VERBATIM_HEADERS = YES #--------------------------------------------------------------------------- # configuration options related to the alphabetical class index #--------------------------------------------------------------------------- # If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index # of all compounds will be generated. Enable this if the project # contains a lot of classes, structs, unions or interfaces. ALPHABETICAL_INDEX = YES # If the alphabetical index is enabled (see ALPHABETICAL_INDEX) then # the COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns # in which this list will be split (can be a number in the range [1..20]) COLS_IN_ALPHA_INDEX = 5 # In case all classes in a project start with a common prefix, all # classes will be put under the same header in the alphabetical index. # The IGNORE_PREFIX tag can be used to specify one or more prefixes that # should be ignored while generating the index headers. IGNORE_PREFIX = #--------------------------------------------------------------------------- # configuration options related to the HTML output #--------------------------------------------------------------------------- # If the GENERATE_HTML tag is set to YES (the default) Doxygen will # generate HTML output. GENERATE_HTML = YES # The HTML_OUTPUT tag is used to specify where the HTML docs will be put. # If a relative path is entered the value of OUTPUT_DIRECTORY will be # put in front of it. If left blank `html' will be used as the default path. HTML_OUTPUT = html # The HTML_FILE_EXTENSION tag can be used to specify the file extension for # each generated HTML page (for example: .htm,.php,.asp). If it is left blank # doxygen will generate files with .html extension. HTML_FILE_EXTENSION = .html # The HTML_HEADER tag can be used to specify a personal HTML header for # each generated HTML page. If it is left blank doxygen will generate a # standard header. HTML_HEADER = # The HTML_FOOTER tag can be used to specify a personal HTML footer for # each generated HTML page. If it is left blank doxygen will generate a # standard footer. HTML_FOOTER = # The HTML_STYLESHEET tag can be used to specify a user-defined cascading # style sheet that is used by each HTML page. It can be used to # fine-tune the look of the HTML output. If the tag is left blank doxygen # will generate a default style sheet. Note that doxygen will try to copy # the style sheet file to the HTML output directory, so don't put your own # stylesheet in the HTML output directory as well, or it will be erased! HTML_STYLESHEET = # If the HTML_ALIGN_MEMBERS tag is set to YES, the members of classes, # files or namespaces will be aligned in HTML using tables. If set to # NO a bullet list will be used. HTML_ALIGN_MEMBERS = YES # If the GENERATE_HTMLHELP tag is set to YES, additional index files # will be generated that can be used as input for tools like the # Microsoft HTML help workshop to generate a compressed HTML help file (.chm) # of the generated HTML documentation. GENERATE_HTMLHELP = NO # If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML # documentation will contain sections that can be hidden and shown after the # page has loaded. For this to work a browser that supports # JavaScript and DHTML is required (for instance Mozilla 1.0+, Firefox # Netscape 6.0+, Internet explorer 5.0+, Konqueror, or Safari). HTML_DYNAMIC_SECTIONS = NO # If the GENERATE_HTMLHELP tag is set to YES, the CHM_FILE tag can # be used to specify the file name of the resulting .chm file. You # can add a path in front of the file if the result should not be # written to the html output directory. CHM_FILE = # If the GENERATE_HTMLHELP tag is set to YES, the HHC_LOCATION tag can # be used to specify the location (absolute path including file name) of # the HTML help compiler (hhc.exe). If non-empty doxygen will try to run # the HTML help compiler on the generated index.hhp. HHC_LOCATION = # If the GENERATE_HTMLHELP tag is set to YES, the GENERATE_CHI flag # controls if a separate .chi index file is generated (YES) or that # it should be included in the master .chm file (NO). GENERATE_CHI = NO # If the GENERATE_HTMLHELP tag is set to YES, the BINARY_TOC flag # controls whether a binary table of contents is generated (YES) or a # normal table of contents (NO) in the .chm file. BINARY_TOC = NO # The TOC_EXPAND flag can be set to YES to add extra items for group members # to the contents of the HTML help documentation and to the tree view. TOC_EXPAND = NO # The DISABLE_INDEX tag can be used to turn on/off the condensed index at # top of each HTML page. The value NO (the default) enables the index and # the value YES disables it. DISABLE_INDEX = NO # This tag can be used to set the number of enum values (range [1..20]) # that doxygen will group on one line in the generated HTML documentation. ENUM_VALUES_PER_LINE = 4 # If the GENERATE_TREEVIEW tag is set to YES, a side panel will be # generated containing a tree-like index structure (just like the one that # is generated for HTML Help). For this to work a browser that supports # JavaScript, DHTML, CSS and frames is required (for instance Mozilla 1.0+, # Netscape 6.0+, Internet explorer 5.0+, or Konqueror). Windows users are # probably better off using the HTML help feature. GENERATE_TREEVIEW = YES # If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be # used to set the initial width (in pixels) of the frame in which the tree # is shown. TREEVIEW_WIDTH = 250 #--------------------------------------------------------------------------- # configuration options related to the LaTeX output #--------------------------------------------------------------------------- # If the GENERATE_LATEX tag is set to YES (the default) Doxygen will # generate Latex output. GENERATE_LATEX = NO # The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. # If a relative path is entered the value of OUTPUT_DIRECTORY will be # put in front of it. If left blank `latex' will be used as the default path. LATEX_OUTPUT = latex # The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be # invoked. If left blank `latex' will be used as the default command name. LATEX_CMD_NAME = latex # The MAKEINDEX_CMD_NAME tag can be used to specify the command name to # generate index for LaTeX. If left blank `makeindex' will be used as the # default command name. MAKEINDEX_CMD_NAME = makeindex # If the COMPACT_LATEX tag is set to YES Doxygen generates more compact # LaTeX documents. This may be useful for small projects and may help to # save some trees in general. COMPACT_LATEX = NO # The PAPER_TYPE tag can be used to set the paper type that is used # by the printer. Possible values are: a4, a4wide, letter, legal and # executive. If left blank a4wide will be used. PAPER_TYPE = a4wide # The EXTRA_PACKAGES tag can be to specify one or more names of LaTeX # packages that should be included in the LaTeX output. EXTRA_PACKAGES = # The LATEX_HEADER tag can be used to specify a personal LaTeX header for # the generated latex document. The header should contain everything until # the first chapter. If it is left blank doxygen will generate a # standard header. Notice: only use this tag if you know what you are doing! LATEX_HEADER = # If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated # is prepared for conversion to pdf (using ps2pdf). The pdf file will # contain links (just like the HTML output) instead of page references # This makes the output suitable for online browsing using a pdf viewer. PDF_HYPERLINKS = NO # If the USE_PDFLATEX tag is set to YES, pdflatex will be used instead of # plain latex in the generated Makefile. Set this option to YES to get a # higher quality PDF documentation. USE_PDFLATEX = NO # If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \\batchmode. # command to the generated LaTeX files. This will instruct LaTeX to keep # running if errors occur, instead of asking the user for help. # This option is also used when generating formulas in HTML. LATEX_BATCHMODE = NO # If LATEX_HIDE_INDICES is set to YES then doxygen will not # include the index chapters (such as File Index, Compound Index, etc.) # in the output. LATEX_HIDE_INDICES = NO #--------------------------------------------------------------------------- # configuration options related to the RTF output #--------------------------------------------------------------------------- # If the GENERATE_RTF tag is set to YES Doxygen will generate RTF output # The RTF output is optimized for Word 97 and may not look very pretty with # other RTF readers or editors. GENERATE_RTF = NO # The RTF_OUTPUT tag is used to specify where the RTF docs will be put. # If a relative path is entered the value of OUTPUT_DIRECTORY will be # put in front of it. If left blank `rtf' will be used as the default path. RTF_OUTPUT = rtf # If the COMPACT_RTF tag is set to YES Doxygen generates more compact # RTF documents. This may be useful for small projects and may help to # save some trees in general. COMPACT_RTF = NO # If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated # will contain hyperlink fields. The RTF file will # contain links (just like the HTML output) instead of page references. # This makes the output suitable for online browsing using WORD or other # programs which support those fields. # Note: wordpad (write) and others do not support links. RTF_HYPERLINKS = NO # Load stylesheet definitions from file. Syntax is similar to doxygen's # config file, i.e. a series of assignments. You only have to provide # replacements, missing definitions are set to their default value. RTF_STYLESHEET_FILE = # Set optional variables used in the generation of an rtf document. # Syntax is similar to doxygen's config file. RTF_EXTENSIONS_FILE = #--------------------------------------------------------------------------- # configuration options related to the man page output #--------------------------------------------------------------------------- # If the GENERATE_MAN tag is set to YES (the default) Doxygen will # generate man pages GENERATE_MAN = NO # The MAN_OUTPUT tag is used to specify where the man pages will be put. # If a relative path is entered the value of OUTPUT_DIRECTORY will be # put in front of it. If left blank `man' will be used as the default path. MAN_OUTPUT = man # The MAN_EXTENSION tag determines the extension that is added to # the generated man pages (default is the subroutine's section .3) MAN_EXTENSION = .3 # If the MAN_LINKS tag is set to YES and Doxygen generates man output, # then it will generate one additional man file for each entity # documented in the real man page(s). These additional files # only source the real man page, but without them the man command # would be unable to find the correct page. The default is NO. MAN_LINKS = NO #--------------------------------------------------------------------------- # configuration options related to the XML output #--------------------------------------------------------------------------- # If the GENERATE_XML tag is set to YES Doxygen will # generate an XML file that captures the structure of # the code including all documentation. GENERATE_XML = NO # The XML_OUTPUT tag is used to specify where the XML pages will be put. # If a relative path is entered the value of OUTPUT_DIRECTORY will be # put in front of it. If left blank `xml' will be used as the default path. XML_OUTPUT = xml # The XML_SCHEMA tag can be used to specify an XML schema, # which can be used by a validating XML parser to check the # syntax of the XML files. XML_SCHEMA = # The XML_DTD tag can be used to specify an XML DTD, # which can be used by a validating XML parser to check the # syntax of the XML files. XML_DTD = # If the XML_PROGRAMLISTING tag is set to YES Doxygen will # dump the program listings (including syntax highlighting # and cross-referencing information) to the XML output. Note that # enabling this will significantly increase the size of the XML output. XML_PROGRAMLISTING = YES #--------------------------------------------------------------------------- # configuration options for the AutoGen Definitions output #--------------------------------------------------------------------------- # If the GENERATE_AUTOGEN_DEF tag is set to YES Doxygen will # generate an AutoGen Definitions (see autogen.sf.net) file # that captures the structure of the code including all # documentation. Note that this feature is still experimental # and incomplete at the moment. GENERATE_AUTOGEN_DEF = NO #--------------------------------------------------------------------------- # configuration options related to the Perl module output #--------------------------------------------------------------------------- # If the GENERATE_PERLMOD tag is set to YES Doxygen will # generate a Perl module file that captures the structure of # the code including all documentation. Note that this # feature is still experimental and incomplete at the # moment. GENERATE_PERLMOD = NO # If the PERLMOD_LATEX tag is set to YES Doxygen will generate # the necessary Makefile rules, Perl scripts and LaTeX code to be able # to generate PDF and DVI output from the Perl module output. PERLMOD_LATEX = NO # If the PERLMOD_PRETTY tag is set to YES the Perl module output will be # nicely formatted so it can be parsed by a human reader. This is useful # if you want to understand what is going on. On the other hand, if this # tag is set to NO the size of the Perl module output will be much smaller # and Perl will parse it just the same. PERLMOD_PRETTY = YES # The names of the make variables in the generated doxyrules.make file # are prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. # This is useful so different doxyrules.make files included by the same # Makefile don't overwrite each other's variables. PERLMOD_MAKEVAR_PREFIX = #--------------------------------------------------------------------------- # Configuration options related to the preprocessor #--------------------------------------------------------------------------- # If the ENABLE_PREPROCESSING tag is set to YES (the default) Doxygen will # evaluate all C-preprocessor directives found in the sources and include # files. ENABLE_PREPROCESSING = YES # If the MACRO_EXPANSION tag is set to YES Doxygen will expand all macro # names in the source code. If set to NO (the default) only conditional # compilation will be performed. Macro expansion can be done in a controlled # way by setting EXPAND_ONLY_PREDEF to YES. MACRO_EXPANSION = NO # If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES # then the macro expansion is limited to the macros specified with the # PREDEFINED and EXPAND_AS_DEFINED tags. EXPAND_ONLY_PREDEF = NO # If the SEARCH_INCLUDES tag is set to YES (the default) the includes files # in the INCLUDE_PATH (see below) will be search if a #include is found. SEARCH_INCLUDES = YES # The INCLUDE_PATH tag can be used to specify one or more directories that # contain include files that are not input files but should be processed by # the preprocessor. INCLUDE_PATH = # You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard # patterns (like *.h and *.hpp) to filter out the header-files in the # directories. If left blank, the patterns specified with FILE_PATTERNS will # be used. INCLUDE_FILE_PATTERNS = # The PREDEFINED tag can be used to specify one or more macro names that # are defined before the preprocessor is started (similar to the -D option of # gcc). The argument of the tag is a list of macros of the form: name # or name=definition (no spaces). If the definition and the = are # omitted =1 is assumed. To prevent a macro definition from being # undefined via #undef or recursively expanded use the := operator # instead of the = operator. PREDEFINED = # If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then # this tag can be used to specify a list of macro names that should be expanded. # The macro definition that is found in the sources will be used. # Use the PREDEFINED tag if you want to use a different macro definition. EXPAND_AS_DEFINED = # If the SKIP_FUNCTION_MACROS tag is set to YES (the default) then # doxygen's preprocessor will remove all function-like macros that are alone # on a line, have an all uppercase name, and do not end with a semicolon. Such # function macros are typically used for boiler-plate code, and will confuse # the parser if not removed. SKIP_FUNCTION_MACROS = YES #--------------------------------------------------------------------------- # Configuration::additions related to external references #--------------------------------------------------------------------------- # The TAGFILES option can be used to specify one or more tagfiles. # Optionally an initial location of the external documentation # can be added for each tagfile. The format of a tag file without # this location is as follows: # TAGFILES = file1 file2 ... # Adding location for the tag files is done as follows: # TAGFILES = file1=loc1 "file2 = loc2" ... # where "loc1" and "loc2" can be relative or absolute paths or # URLs. If a location is present for each tag, the installdox tool # does not have to be run to correct the links. # Note that each tag file must have a unique name # (where the name does NOT include the path) # If a tag file is not located in the directory in which doxygen # is run, you must also specify the path to the tagfile here. TAGFILES = # When a file name is specified after GENERATE_TAGFILE, doxygen will create # a tag file that is based on the input files it reads. GENERATE_TAGFILE = # If the ALLEXTERNALS tag is set to YES all external classes will be listed # in the class index. If set to NO only the inherited external classes # will be listed. ALLEXTERNALS = NO # If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed # in the modules index. If set to NO, only the current project's groups will # be listed. EXTERNAL_GROUPS = YES # The PERL_PATH should be the absolute path and name of the perl script # interpreter (i.e. the result of `which perl'). PERL_PATH = /usr/bin/perl #--------------------------------------------------------------------------- # Configuration options related to the dot tool #--------------------------------------------------------------------------- # If the CLASS_DIAGRAMS tag is set to YES (the default) Doxygen will # generate a inheritance diagram (in HTML, RTF and LaTeX) for classes with base # or super classes. Setting the tag to NO turns the diagrams off. Note that # this option is superseded by the HAVE_DOT option below. This is only a # fallback. It is recommended to install and use dot, since it yields more # powerful graphs. CLASS_DIAGRAMS = NO # You can define message sequence charts within doxygen comments using the \msc # command. Doxygen will then run the mscgen tool (see http://www.mcternan.me.uk/mscgen/) to # produce the chart and insert it in the documentation. The MSCGEN_PATH tag allows you to # specify the directory where the mscgen tool resides. If left empty the tool is assumed to # be found in the default search path. MSCGEN_PATH = # If set to YES, the inheritance and collaboration graphs will hide # inheritance and usage relations if the target is undocumented # or is not a class. HIDE_UNDOC_RELATIONS = NO # If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is # available from the path. This tool is part of Graphviz, a graph visualization # toolkit from AT&T and Lucent Bell Labs. The other options in this section # have no effect if this option is set to NO (the default) HAVE_DOT = YES # If the CLASS_GRAPH and HAVE_DOT tags are set to YES then doxygen # will generate a graph for each documented class showing the direct and # indirect inheritance relations. Setting this tag to YES will force the # the CLASS_DIAGRAMS tag to NO. CLASS_GRAPH = NO # If the COLLABORATION_GRAPH and HAVE_DOT tags are set to YES then doxygen # will generate a graph for each documented class showing the direct and # indirect implementation dependencies (inheritance, containment, and # class references variables) of the class with other documented classes. COLLABORATION_GRAPH = YES # If the GROUP_GRAPHS and HAVE_DOT tags are set to YES then doxygen # will generate a graph for groups, showing the direct groups dependencies GROUP_GRAPHS = YES # If the UML_LOOK tag is set to YES doxygen will generate inheritance and # collaboration diagrams in a style similar to the OMG's Unified Modeling # Language. UML_LOOK = NO # If set to YES, the inheritance and collaboration graphs will show the # relations between templates and their instances. TEMPLATE_RELATIONS = YES # If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDE_GRAPH, and HAVE_DOT # tags are set to YES then doxygen will generate a graph for each documented # file showing the direct and indirect include dependencies of the file with # other documented files. INCLUDE_GRAPH = YES # If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDED_BY_GRAPH, and # HAVE_DOT tags are set to YES then doxygen will generate a graph for each # documented header file showing the documented files that directly or # indirectly include this file. INCLUDED_BY_GRAPH = YES # If the CALL_GRAPH, SOURCE_BROWSER and HAVE_DOT tags are set to YES then doxygen will # generate a call dependency graph for every global function or class method. # Note that enabling this option will significantly increase the time of a run. # So in most cases it will be better to enable call graphs for selected # functions only using the \callgraph command. CALL_GRAPH = YES # If the CALLER_GRAPH, SOURCE_BROWSER and HAVE_DOT tags are set to YES then doxygen will # generate a caller dependency graph for every global function or class method. # Note that enabling this option will significantly increase the time of a run. # So in most cases it will be better to enable caller graphs for selected # functions only using the \callergraph command. CALLER_GRAPH = YES # If the GRAPHICAL_HIERARCHY and HAVE_DOT tags are set to YES then doxygen # will graphical hierarchy of all classes instead of a textual one. GRAPHICAL_HIERARCHY = YES # If the DIRECTORY_GRAPH, SHOW_DIRECTORIES and HAVE_DOT tags are set to YES # then doxygen will show the dependencies a directory has on other directories # in a graphical way. The dependency relations are determined by the #include # relations between the files in the directories. DIRECTORY_GRAPH = YES # The DOT_IMAGE_FORMAT tag can be used to set the image format of the images # generated by dot. Possible values are png, jpg, or gif # If left blank png will be used. DOT_IMAGE_FORMAT = png # The tag DOT_PATH can be used to specify the path where the dot tool can be # found. If left blank, it is assumed the dot tool can be found in the path. DOT_PATH = # The DOTFILE_DIRS tag can be used to specify one or more directories that # contain dot files that are included in the documentation (see the # \dotfile command). DOTFILE_DIRS = # The MAX_DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of # nodes that will be shown in the graph. If the number of nodes in a graph # becomes larger than this value, doxygen will truncate the graph, which is # visualized by representing a node as a red box. Note that doxygen if the number # of direct children of the root node in a graph is already larger than # MAX_DOT_GRAPH_NOTES then the graph will not be shown at all. Also note # that the size of a graph can be further restricted by MAX_DOT_GRAPH_DEPTH. DOT_GRAPH_MAX_NODES = 90 # The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the # graphs generated by dot. A depth value of 3 means that only nodes reachable # from the root by following a path via at most 3 edges will be shown. Nodes # that lay further from the root node will be omitted. Note that setting this # option to 1 or 2 may greatly reduce the computation time needed for large # code bases. Also note that the size of a graph can be further restricted by # DOT_GRAPH_MAX_NODES. Using a depth of 0 means no depth restriction. MAX_DOT_GRAPH_DEPTH = 0 # Set the DOT_TRANSPARENT tag to YES to generate images with a transparent # background. This is disabled by default, which results in a white background. # Warning: Depending on the platform used, enabling this option may lead to # badly anti-aliased labels on the edges of a graph (i.e. they become hard to # read). DOT_TRANSPARENT = NO # Set the DOT_MULTI_TARGETS tag to YES allow dot to generate multiple output # files in one run (i.e. multiple -o and -T options on the command line). This # makes dot run faster, but since only newer versions of dot (>1.8.10) # support this, this feature is disabled by default. DOT_MULTI_TARGETS = NO # If the GENERATE_LEGEND tag is set to YES (the default) Doxygen will # generate a legend page explaining the meaning of the various boxes and # arrows in the dot generated graphs. GENERATE_LEGEND = YES # If the DOT_CLEANUP tag is set to YES (the default) Doxygen will # remove the intermediate dot files that are used to generate # the various graphs. DOT_CLEANUP = YES #--------------------------------------------------------------------------- # Configuration::additions related to the search engine #--------------------------------------------------------------------------- # The SEARCHENGINE tag specifies whether or not a search engine should be # used. If set to NO the values of all tags below this one will be ignored. SEARCHENGINE = NO odc-1.6.3/src/odc/0000775000175000017500000000000015146027420014007 5ustar alastairalastairodc-1.6.3/src/odc/DispatchingWriter.cc0000664000175000017500000000230415146027420017747 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// /// \file DispatchingWriter.cc /// /// @author Piotr Kuchta, June 2009 #include "odc/DispatchingWriter.h" #include "eckit/config/Resource.h" using namespace eckit; namespace odc { DispatchingWriter::DispatchingWriter(const std::string& outputFileTemplate, int maxOpenFiles, bool append) : outputFileTemplate_(outputFileTemplate), maxOpenFiles_(maxOpenFiles ? maxOpenFiles : Resource("$ODBAPI_MAX_OPEN_FILES;-maxOpenFiles;maxOpenFiles", 250)), append_(append) {} DispatchingWriter::~DispatchingWriter() {} DispatchingWriter::iterator_class* DispatchingWriter::writer() { NOTIMP; // TODO: return 0; } DispatchingWriter::iterator DispatchingWriter::begin() { return iterator(new iterator_class(*this, maxOpenFiles_, append_)); } } // namespace odc odc-1.6.3/src/odc/sql/0000775000175000017500000000000015146027420014606 5ustar alastairalastairodc-1.6.3/src/odc/sql/SQLOutputConfig.cc0000664000175000017500000000511715146027420020127 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/sql/SQLSimpleOutput.h" #include "odc/DispatchingWriter.h" #include "odc/TemplateParameters.h" #include "odc/Writer.h" #include "odc/sql/ODAOutput.h" #include "odc/sql/SQLOutputConfig.h" namespace odc { namespace sql { //---------------------------------------------------------------------------------------------------------------------- SQLOutputConfig::SQLOutputConfig(bool noColumnNames, bool noNULL, const std::string& delimiter, const std::string& format, bool bitfieldsBinary, bool noColumnAlignment, bool fullPrecision) : eckit::sql::SQLOutputConfig(noColumnNames, noNULL, delimiter, format, bitfieldsBinary, noColumnAlignment, fullPrecision), outStream_(std::cout) {} SQLOutputConfig::SQLOutputConfig(const std::string& odbFilename) : SQLOutputConfig() { outputFormat_ = "odb"; outputFile_ = odbFilename; } SQLOutputConfig::~SQLOutputConfig() {} eckit::sql::SQLOutput* SQLOutputConfig::buildOutput(const eckit::PathName& path) const { // TODO: maxOpenFiles configuration for output. Was disabled in Feb 2016 const size_t maxOpenFiles = 100; std::string format; if (outputFormat_ == "default") { format = (path.asString().empty() ? "ascii" : "odb"); } else { format = outputFormat_; } if (format == "wide" || format == "ascii") { return new eckit::sql::SQLSimpleOutput(*this, outStream_.get()); } else if (format == "odb") { ASSERT(path.asString().size()); TemplateParameters templateParameters; TemplateParameters::parse(path, templateParameters); if (templateParameters.size()) { return new odc::sql::ODAOutput(new DispatchingWriter(path, maxOpenFiles)); } else { return new odc::sql::ODAOutput>(new Writer<>(path)); // TODO: toODAColumns } } NOTIMP; } void SQLOutputConfig::setOutputStream(std::ostream& s) { outStream_ = s; } //---------------------------------------------------------------------------------------------------------------------- } // namespace sql } // namespace odc odc-1.6.3/src/odc/sql/ODAOutput.cc0000664000175000017500000001362315146027420016746 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/log/Log.h" #include "eckit/sql/SQLSelect.h" #include "eckit/sql/expression/SQLExpressions.h" #include "odc/DispatchingWriter.h" #include "odc/LibOdc.h" #include "odc/Writer.h" #include "odc/api/ColumnType.h" #include "odc/sql/ODAOutput.h" #include "odc/sql/Types.h" using namespace eckit; using namespace eckit::sql; using namespace odc::api; using eckit::sql::expression::SQLExpression; namespace odc { namespace sql { //---------------------------------------------------------------------------------------------------------------------- template ODAOutput::ODAOutput(WRITER* writer) : writer_(writer), it_(nullptr), col_(0), count_(0), initted_(false) {} template ODAOutput::~ODAOutput() {} template void ODAOutput::print(std::ostream& s) const { s << "ODAOutput: iterator: " << it_ << std::endl; } template unsigned long long ODAOutput::count() { return count_; } template void ODAOutput::reset() { count_ = 0; } template void ODAOutput::flush() {} template void ODAOutput::cleanup(SQLSelect& sql) { sql.outputFiles((**it_).outputFiles()); } template bool ODAOutput::output(const expression::Expressions& results) { size_t nCols(results.size()); for (col_ = 0; col_ < nCols; ++col_) { results[col_]->output(*this); } ++it_; ++count_; return true; } template void ODAOutput::preprepare(SQLSelect& sql) {} template void ODAOutput::prepare(SQLSelect& sql) { initUpdateTypes(sql); } template void ODAOutput::updateTypes(eckit::sql::SQLSelect& sql) { initUpdateTypes(sql); } template void ODAOutput::initUpdateTypes(eckit::sql::SQLSelect& sql) { const expression::Expressions& columns(sql.output()); if (!initted_) { initted_ = true; it_ = writer_->begin(); it_->setNumberOfColumns(columns.size()); columnSizes_.resize(columns.size()); missingValues_.resize(columns.size()); for (size_t i = 0; i < columns.size(); ++i) { SQLExpression& c(*columns[i]); api::ColumnType typ = sqlToOdbType(*c.type()); columnSizes_[i] = sizeof(double); if (typ == BITFIELD) { it_->setBitfieldColumn(i, c.title(), typ, c.bitfieldDef()); } else { it_->setColumn(i, c.title(), typ); if (typ == STRING) { size_t maxlen = c.type()->size(); columnSizes_[i] = maxlen; ASSERT(maxlen % sizeof(double) == 0); it_->columns()[i]->dataSizeDoubles(maxlen / sizeof(double)); } } missingValues_[i] = it_->missingValue(i); } it_->writeHeader(); LOG_DEBUG_LIB(LibOdc) << " => ODAOutput: " << std::endl << it_->columns() << std::endl; } else { std::map resetColumnSizeDoubles; for (size_t i = 0; i < columns.size(); ++i) { SQLExpression& c(*columns[i]); api::ColumnType typ = sqlToOdbType(*c.type()); ASSERT(typ == it_->columns()[i]->type()); ASSERT(c.title() == it_->columns()[i]->name()); if (typ == STRING) { size_t maxlen = c.type()->size(); ASSERT(maxlen % sizeof(double) == 0); size_t sizeDoubles = maxlen / sizeof(double); if (sizeDoubles > it_->columns()[i]->dataSizeDoubles()) { columnSizes_[i] = maxlen; resetColumnSizeDoubles[c.title()] = sizeDoubles; } } else { ASSERT(it_->columns()[i]->dataSizeDoubles() == 1); } } // And if we _do_ need to adjust the column sizes... if (!resetColumnSizeDoubles.empty()) { it_->flushAndResetColumnSizes(resetColumnSizeDoubles); } } } // Direct output functions removed in order output template void ODAOutput::outputNumber(double val, bool missing) { (*it_)[col_] = (missing ? missingValues_[col_] : val); } template void ODAOutput::outputReal(double val, bool missing) { outputNumber(val, missing); } template void ODAOutput::outputDouble(double val, bool missing) { outputNumber(val, missing); } template void ODAOutput::outputInt(double val, bool missing) { outputNumber(val, missing); } template void ODAOutput::outputUnsignedInt(double val, bool missing) { outputNumber(val, missing); } template void ODAOutput::outputBitfield(double val, bool missing) { outputNumber(val, missing); } template void ODAOutput::outputString(const char* val, size_t len, bool missing) { ::memset(&(*it_)[col_], 0, columnSizes_[col_]); ASSERT(len <= columnSizes_[col_]); if (!missing) ::strncpy(reinterpret_cast(&(*it_)[col_]), val, len); } // Explicit template instantiations. template class ODAOutput>; template class ODAOutput; //---------------------------------------------------------------------------------------------------------------------- } // namespace sql } // namespace odc odc-1.6.3/src/odc/sql/TODATable.h0000664000175000017500000000615315146027420016463 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Piotr Kuchta /// @author Simon Smart /// ECMWF Oct 2010 #ifndef TODATable_H #define TODATable_H #include "eckit/sql/SQLTable.h" #include "odc/Reader.h" #include "odc/csv/TextReader.h" namespace odc { namespace sql { //---------------------------------------------------------------------------------------------------------------------- template class TODATable : public eckit::sql::SQLTable { public: TODATable(eckit::sql::SQLDatabase& owner, const std::string& path, const std::string& name); virtual ~TODATable(); const READER& oda() const; private: // methods void populateMetaData(); // void updateMetaData(const std::vector&); protected: // methods TODATable(eckit::sql::SQLDatabase& owner, const std::string& path, const std::string& name, READER&& oda); private: // methods (overrides) virtual bool hasColumn(const std::string&) const override; virtual const eckit::sql::SQLColumn& column(const std::string&) const override; virtual eckit::sql::SQLTableIterator* iterator( const std::vector>&, std::function metadataUpdateCallback) const override; virtual void print(std::ostream& s) const override; public: READER oda_; private: // members // This is a hack. Avoid calling begin() twice on non-seekable DataHandle if possible typename READER::iterator readerIterator_; }; //---------------------------------------------------------------------------------------------------------------------- // Specific cases for Reader and TextReader extern template class TODATable; extern template class TODATable; struct ODATable : public TODATable { ODATable(eckit::sql::SQLDatabase& owner, const std::string& path, const std::string& name) : TODATable(owner, path, name, Reader(path)) {} ODATable(eckit::sql::SQLDatabase& owner, eckit::DataHandle& dh) : TODATable(owner, "<>", "input", Reader(dh)) {} }; struct ODBCSVTable : public TODATable { ODBCSVTable(eckit::sql::SQLDatabase& owner, const std::string& path, const std::string& name, const std::string& delimiter) : TODATable(owner, path, name, TextReader(path, delimiter)) {} ODBCSVTable(eckit::sql::SQLDatabase& owner, std::istream& is, const std::string& name, const std::string& delimiter) : TODATable(owner, "", name, TextReader(is, delimiter)) {} }; //---------------------------------------------------------------------------------------------------------------------- } // namespace sql } // namespace odc #endif odc-1.6.3/src/odc/sql/Types.cc0000664000175000017500000000310415146027420016217 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/exception/Exceptions.h" #include "eckit/sql/type/SQLType.h" #include "eckit/utils/Translator.h" #include "odc/sql/Types.h" using namespace eckit; using namespace eckit::sql::type; using namespace odc::api; namespace odc { namespace sql { //---------------------------------------------------------------------------------------------------------------------- ColumnType sqlToOdbType(const SQLType& t) { switch (t.getKind()) { case SQLType::realType: return REAL; case SQLType::integerType: return INTEGER; case SQLType::stringType: return STRING; case SQLType::bitmapType: return BITFIELD; case SQLType::doubleType: return DOUBLE; case SQLType::blobType: throw SeriousBug("SQL blob type not supported in odc", Here()); default: throw SeriousBug("Unrecognised type found: " + Translator()(t.getKind()), Here()); }; ASSERT(false); return IGNORE; } //---------------------------------------------------------------------------------------------------------------------- } // namespace sql } // namespace odc odc-1.6.3/src/odc/sql/TODATable.cc0000664000175000017500000001737715146027420016633 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/io/FileHandle.h" #include "eckit/sql/SQLColumn.h" #include "eckit/sql/SQLTableFactory.h" #include "eckit/sql/type/SQLBitfield.h" #include "eckit/utils/StringTools.h" #include "eckit/utils/Translator.h" #include "odc/Reader.h" #include "odc/csv/TextReader.h" #include "odc/csv/TextReaderIterator.h" #include "odc/sql/TODATable.h" #include "odc/sql/TODATableIterator.h" using namespace eckit; using namespace eckit::sql; using namespace odc::api; using namespace odc::core; namespace odc { namespace sql { //---------------------------------------------------------------------------------------------------------------------- namespace { // Provide a factory such that when a table is specified in a from statement, the SQLParser // can construct an appropriate table! class ODAFactory : public eckit::sql::SQLTableFactoryBase { virtual SQLTable* build(SQLDatabase& owner, const std::string& name, const std::string& location) const override { PathName path(location); if (!path.exists()) return 0; // Check that this is an ODB file FileHandle fh(path, false); fh.openForRead(); AutoClose closer(fh); char buf[5]; char oda[5]{'\xff', '\xff', 'O', 'D', 'A'}; if (fh.read(buf, 5) != 5 || ::memcmp(buf, oda, 5) != 0) return 0; return new odc::sql::ODATable(owner, location, name); } }; ODAFactory odaFactoryInstance; } // namespace //--------------------------------------------------------------------------------------------------------------------- template TODATable::TODATable(SQLDatabase& owner, const std::string& path, const std::string& name, READER&& oda) : SQLTable(owner, path, name), oda_(std::move(oda)), readerIterator_(oda_.begin()) { populateMetaData(); } template TODATable::~TODATable() {} template const READER& TODATable::oda() const { return oda_; } template void TODATable::populateMetaData() { auto& it(readerIterator_); size_t count = it->columns().size(); for (size_t i = 0; i < count; i++) { Column& column(*it->columns()[i]); const std::string name(column.name()); bool hasMissing(column.hasMissing()); double missing(column.missingValue()); BitfieldDef bitfieldDef(column.bitfieldDef()); std::string sqlType; size_t typeSizeDoubles = it->dataSizeDoubles(i); switch (column.type()) { case INTEGER: sqlType = "integer"; break; case STRING: sqlType = "string"; break; case REAL: sqlType = "real"; break; case DOUBLE: sqlType = "double"; break; case BITFIELD: { std::string typeSignature = type::SQLBitfield::make("Bitfield", bitfieldDef.first, bitfieldDef.second, "DummyTypeAlias"); addColumn(name, i, type::SQLType::lookup(typeSignature), hasMissing, missing, true, bitfieldDef); continue; } default: throw SeriousBug("Unknown type: " + Translator()(column.type()), Here()); } addColumn(name, i, type::SQLType::lookup(sqlType, typeSizeDoubles), hasMissing, missing, column.type() == BITFIELD, bitfieldDef); } } // void TODATable::updateMetaData(const std::vector& selected) //{ // // TODO: Whoah! whoah! whoah! // // n.b. we don't really want to modify the table. We should probabyl deal with this in the iterator... // NOTIMP; // //// MetaData newColumns (it_->columns()); //// for(size_t i = 0; i < selected.size(); i++) //// { //// ODAColumn *c = dynamic_cast(selected[i]); //// ASSERT(c); //// if (newColumns.size() <= c->index() || newColumns[c->index()]->name() != c->name()) //// { //// Log::warning() << "Column '" << c->fullName() << "': index has changed in new dataset." << endl //// << "Was: " << c->index() << "." << endl; //// bool newIndexFound = false; //// for (size_t j = 0; j < newColumns.size(); ++j) //// { //// Column &other(*newColumns[j]); //// if (other.name() == c->name() || other.name() == c->fullName()) //// { //// newIndexFound = true; //// Log::warning() << "New index: " << j << endl; //// c->index(j); //// break; //// } //// } //// if (! newIndexFound) //// { //// // TODO: if user specified MAYBE keyword, then use a constant NULL column. //// //if (maybe_) { //// // Log::warning() << "Column '" << c->name() << "' not found." << endl; //// // selected[i] = new NullColumn(*selected[i]); //// //} else { //// stringstream ss; //// ss << "One of selected columns, '" << c->name() << "', does not exist in new data set."; //// throw UserError(ss.str()); //// //} //// } //// } //// //c->value(&data_[i]); //// } //} template bool TODATable::hasColumn(const std::string& name) const { // If the column is simply in the table, then use it. if (SQLTable::hasColumn(name)) { return true; } // Find columns that also have an (unspecified) section name std::string colName(name + "@"); int n = 0; for (const auto& column : columnsByName_) { const std::string& s(column.first); if (StringTools::startsWith(s, colName)) { n++; } } if (n == 1) return true; if (n > 1) { throw UserError(std::string("TODATable:hasColumn(\"") + name + "\"): ambiguous name"); } return false; } template const SQLColumn& TODATable::column(const std::string& name) const { // If the column is simply in the table, then use it. if (SQLTable::hasColumn(name)) { return SQLTable::column(name); } // Find columns that also have an (unspecified) section name const std::string colName(name + "@"); SQLColumn* column = 0; for (const auto& col : columnsByName_) { const std::string& s(col.first); if (StringTools::startsWith(s, colName)) { if (column) throw UserError(std::string("TODATable:hasColumn(\"") + name + "\"): ambiguous name"); column = col.second; } } if (!column) throw SeriousBug("Requesting column \"" + name + "\": not found", Here()); return *column; } template SQLTableIterator* TODATable::iterator( const std::vector>& columns, std::function metadataUpdateCallback) const { return new TODATableIterator(*this, columns, metadataUpdateCallback, readerIterator_); } template void TODATable::print(std::ostream& s) const { s << "TODATable(" << path_ << ")"; } // Explicit instantiation template class TODATable; template class TODATable; //---------------------------------------------------------------------------------------------------------------------- } // namespace sql } // namespace odc odc-1.6.3/src/odc/sql/SQLOutputConfig.h0000664000175000017500000000301515146027420017764 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Simon Smart /// Aug 2018 #ifndef odc_sql_SQLOutputConfig_H #define odc_sql_SQLOutputConfig_H #include "eckit/sql/SQLOutputConfig.h" namespace odc { namespace sql { //---------------------------------------------------------------------------------------------------------------------- class SQLOutputConfig : public eckit::sql::SQLOutputConfig { public: // methods SQLOutputConfig(bool noColumnNames = false, bool noNULL = false, const std::string& delimiter = defaultDelimiter, const std::string& format = defaultOutputFormat, bool bitfieldsBinary = false, bool noColumnAlignment = false, bool fullPrecision = false); SQLOutputConfig(const std::string& odbFilename); ~SQLOutputConfig() override; eckit::sql::SQLOutput* buildOutput(const eckit::PathName& path) const override; void setOutputStream(std::ostream& s); private: // members std::reference_wrapper outStream_; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace sql } // namespace odc #endif odc-1.6.3/src/odc/sql/TODATableIterator.cc0000664000175000017500000001165115146027420020332 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/exception/Exceptions.h" #include "eckit/sql/SQLColumn.h" #include "odc/Reader.h" #include "odc/csv/TextReader.h" #include "odc/csv/TextReaderIterator.h" #include "odc/sql/TODATable.h" #include "odc/sql/TODATableIterator.h" namespace odc { namespace sql { //---------------------------------------------------------------------------------------------------------------------- namespace { /// Return the index of column `columnName` in metadata. If not found or ambiguous, /// throw an exception. size_t columnIndex(const std::string& columnName, const core::MetaData& md) { size_t idx; try { idx = md.columnIndex(columnName); // Make some error messages more precise } catch (const core::AmbiguousColumnException& e) { throw eckit::UserError("Ambiguous column name \"" + columnName + "\" specified in SQL request.", Here()); } catch (const core::ColumnNotFoundException& e) { throw eckit::UserError("Column \"" + columnName + "\" not found in table, but required in SQL request.", Here()); } return idx; } } // namespace //---------------------------------------------------------------------------------------------------------------------- // TODO: It is not appropriate for parent_.oda() to be const_cast<>-ed. // Not entirely clear how to resolve this. But there is no reason for us intrinsically // to be modifying the parent. Perhaps we should take a copy of somethnig (oda, dh?) template TODATableIterator::TODATableIterator( const TODATable& parent, const std::vector>& columns, std::function metadataUpdateCallback, const typename READER::iterator& seedIterator) : parent_(parent), it_(seedIterator), end_(parent_.oda().end()), columns_(columns), metadataUpdateCallback_(metadataUpdateCallback), firstRow_(true) { if (it_ != end_) updateMetaData(); } template void TODATableIterator::rewind() { if (!firstRow_) { it_ = const_cast(parent_.oda()).begin(); end_ = parent_.oda().end(); firstRow_ = true; } } template TODATableIterator::~TODATableIterator() {} template bool TODATableIterator::next() { // We don't need to increment pointer on first row. begin() just called. if (firstRow_) { firstRow_ = false; } else { ++it_; } if (it_ == end_) return false; if (it_->isNewDataset()) { // TODO: Need to update the column pointers in the SQLSelect. AARGH. updateMetaData(); metadataUpdateCallback_(*this); } return true; } template void TODATableIterator::updateMetaData() { const core::MetaData& md = it_->columns(); columnOffsets_.clear(); columnDoublesSizes_.clear(); columnsHaveMissing_.clear(); columnMissingValues_.clear(); for (const eckit::sql::SQLColumn& col : columns_) { const size_t idx = columnIndex(col.name(), md); columnOffsets_.push_back(it_->dataOffset(idx)); columnDoublesSizes_.push_back(it_->dataSizeDoubles(idx)); columnsHaveMissing_.push_back(it_->hasMissing(idx)); columnMissingValues_.push_back(it_->missingValue(idx)); } } template std::vector TODATableIterator::columnOffsets() const { ASSERT(columnOffsets_.size() == columns_.size()); return columnOffsets_; } template std::vector TODATableIterator::doublesDataSizes() const { ASSERT(columnDoublesSizes_.size() == columns_.size()); return columnDoublesSizes_; } template std::vector TODATableIterator::columnsHaveMissing() const { ASSERT(columnsHaveMissing_.size() == columns_.size()); return columnsHaveMissing_; } template std::vector TODATableIterator::missingValues() const { ASSERT(columnMissingValues_.size() == columns_.size()); return columnMissingValues_; } template const double* TODATableIterator::data() const { return it_->data(); } // Explicit instantiation template class TODATableIterator; template class TODATableIterator; //---------------------------------------------------------------------------------------------------------------------- } // namespace sql } // namespace odc odc-1.6.3/src/odc/sql/TODATableIterator.h0000664000175000017500000000475415146027420020202 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Piotr Kuchta /// @author Simon Smart /// ECMWF Oct 2010 #ifndef odc_sql_TODATableIterator_H #define odc_sql_TODATableIterator_H #include "eckit/sql/SQLTable.h" #include "odc/Reader.h" #include "odc/csv/TextReader.h" namespace odc { namespace sql { template class TODATable; //---------------------------------------------------------------------------------------------------------------------- template class TODATableIterator : public eckit::sql::SQLTableIterator { public: // methods TODATableIterator(const TODATable& parent, const std::vector>& columns, std::function metadataUpdateCallback, const typename READER::iterator& seedIterator); virtual ~TODATableIterator(); private: // methods (override> virtual void rewind() override; virtual bool next() override; virtual std::vector columnOffsets() const override; virtual std::vector doublesDataSizes() const override; virtual std::vector columnsHaveMissing() const override; virtual std::vector missingValues() const override; virtual const double* data() const override; private: // methods void updateMetaData(); private: // members const TODATable& parent_; typename READER::iterator it_; typename READER::iterator end_; const std::vector>& columns_; std::vector columnOffsets_; std::vector columnDoublesSizes_; std::vector columnsHaveMissing_; std::vector columnMissingValues_; std::function metadataUpdateCallback_; bool firstRow_; }; extern template class TODATableIterator; extern template class TODATableIterator; //---------------------------------------------------------------------------------------------------------------------- } // namespace sql } // namespace odc #endif odc-1.6.3/src/odc/sql/Types.h0000664000175000017500000000200715146027420016062 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Simon Smart /// @date Aug 2018 #ifndef odc_sql_Types_H #define odc_sql_Types_H #include "odc/api/ColumnType.h" namespace eckit { namespace sql { namespace type { class SQLType; } } // namespace sql } // namespace eckit namespace odc { namespace sql { //---------------------------------------------------------------------------------------------------------------------- odc::api::ColumnType sqlToOdbType(const eckit::sql::type::SQLType& t); //---------------------------------------------------------------------------------------------------------------------- } // namespace sql } // namespace odc #endif odc-1.6.3/src/odc/sql/SQLSelectOutput.h0000664000175000017500000000777615146027420020020 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Simon Smart /// @date Aug 2018 /// @note This output is designed to generalise and standardise the output mechanism for the /// odc::Select class. Prior to this, the class had two problems: /// /// i) It invasively used the internals of the SQLSelect and TableIterators, accessing their /// internal data buffers, and the results objects directly. This is nasty and results in /// significant duplicate code. /// /// ii) As a result of the original IFS origins of this code, integers are returned as DOUBLES. /// this is reasonable, but a bit odd, and it is worth having in code the support to output /// integers as integers for future/other use - without having to reimplement and duplicate /// everything. /// /// This output mechanism allows output to be made to a specified buffer (which may be changed /// on a per-step basis) with a semantically correct behaviour depending on the column type. /// /// This has been put in odc, not in eckit, on the understanding that the handling of /// integers as doubles is esoteric and should probably not be replicated anywhere else. #ifndef odc_SQLSelectOutput_H #define odc_SQLSelectOutput_H #include #include "eckit/sql/SQLOutput.h" #include "odc/core/MetaData.h" namespace odc { namespace sql { //---------------------------------------------------------------------------------------------------------------------- class SQLSelectOutput : public eckit::sql::SQLOutput { public: // methods /// out/count specify an output buffer and its size in doubles. SQLSelectOutput(bool manageOwnBuffer = true); virtual ~SQLSelectOutput(); void resetBuffer(double* out, size_t count); // Enable access to the aggregated data from the SelectIterator const double* data() const { return out_; } double& data(size_t i) { return out_[offsets_[i]]; } size_t rowDataSizeDoubles() const { return requiredBufferSize_; } const core::MetaData& metadata() const { return metaData_; } size_t dataOffset(size_t i) const { return offsets_[i]; } bool isNewDataset() const { return isNewDataset_; } private: // utility void outputNumber(double val, bool missing); private: // methods (overrides) virtual void print(std::ostream&) const; virtual void reset(); virtual void flush(); virtual bool output(const eckit::sql::expression::Expressions&); virtual void prepare(eckit::sql::SQLSelect&); virtual void updateTypes(eckit::sql::SQLSelect&); virtual void cleanup(eckit::sql::SQLSelect&); virtual unsigned long long count(); virtual void outputReal(double, bool); virtual void outputDouble(double, bool); virtual void outputInt(double, bool); virtual void outputUnsignedInt(double, bool); virtual void outputString(const char*, size_t, bool); virtual void outputBitfield(double, bool); private: // members /// Only used if managing own buffer. std::vector data_; /// Where are we writing data to (and how many elements can we write) double* out_; double* pos_; double* end_; size_t bufferElements_; size_t requiredBufferSize_; /// How are writes carried out std::vector columnSizesDoubles_; std::vector offsets_; std::vector missingValues_; core::MetaData metaData_; /// How much output have we done unsigned long long count_; size_t currentColumn_; bool manageOwnBuffer_; bool isNewDataset_; bool newDatasetOutputted_; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace sql } // namespace odc #endif odc-1.6.3/src/odc/sql/ODAOutput.h0000664000175000017500000000603315146027420016605 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Piotr Kuchta /// @author Simon Smart /// @date Jan 2009 /// @date Aug 2018 #ifndef odc_sql_ODAOutput_H #define odc_sql_ODAOutput_H #include "eckit/sql/SQLOutput.h" #include "odc/core/MetaData.h" #include "odc/DispatchingWriter.h" #include "odc/WriterBufferingIterator.h" namespace odc { namespace sql { //---------------------------------------------------------------------------------------------------------------------- template class ODAOutput : public eckit::sql::SQLOutput { public: ODAOutput(WRITER*); // ODAOutput(WRITER*, const MetaData&); virtual ~ODAOutput(); // Change to virtual if base class private: // methods virtual void print(std::ostream&) const override; // -- Members std::unique_ptr writer_; typename WRITER::iterator it_; int col_; std::vector columnSizes_; std::vector missingValues_; // MetaData metaData_; unsigned long long count_; bool initted_; // -- Overridden methods virtual void reset() override; virtual void flush() override; virtual bool output(const eckit::sql::expression::Expressions&) override; virtual void preprepare(eckit::sql::SQLSelect&) override; virtual void prepare(eckit::sql::SQLSelect&) override; virtual void cleanup(eckit::sql::SQLSelect&) override; virtual void updateTypes(eckit::sql::SQLSelect&) override; virtual unsigned long long count() override; void initUpdateTypes(eckit::sql::SQLSelect&); // Overridden (and removed) functions virtual void outputReal(double, bool) override; virtual void outputDouble(double, bool) override; virtual void outputInt(double, bool) override; virtual void outputUnsignedInt(double, bool) override; virtual void outputString(const char*, size_t, bool) override; virtual void outputBitfield(double, bool) override; void outputNumber(double, bool); }; //---------------------------------------------------------------------------------------------------------------------- } // namespace sql //---------------------------------------------------------------------------------------------------------------------- // We explicitly instantiate this template in the .cc file, so don't instantiate in each // translation unit. class WriterBufferingIterator; template class Writer; class DispatchingWriter; extern template class sql::ODAOutput>; extern template class sql::ODAOutput; //---------------------------------------------------------------------------------------------------------------------- } // namespace odc #endif odc-1.6.3/src/odc/sql/SQLSelectOutput.cc0000664000175000017500000001463115146027420020142 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include #include "eckit/eckit.h" #include "eckit/log/Number.h" #include "eckit/sql/SQLSelect.h" #include "eckit/sql/expression/SQLExpression.h" #include "odc/sql/SQLSelectOutput.h" #include "odc/sql/Types.h" using namespace eckit; using namespace eckit::sql; namespace odc { namespace sql { //---------------------------------------------------------------------------------------------------------------------- // TODO: n.b. We can implement an optimised case if the output buffer matches the buffer in the // innards of the select, which just does a memcpy. It will be a bit messy, but this // is probably the place to do it (although it may rather belong in eckit). SQLSelectOutput::SQLSelectOutput(bool manageOwnBuffer) : out_(0), pos_(0), end_(0), bufferElements_(0), count_(0), manageOwnBuffer_(manageOwnBuffer), isNewDataset_(true), newDatasetOutputted_(false) {} SQLSelectOutput::~SQLSelectOutput() {} void SQLSelectOutput::resetBuffer(double* out, size_t count) { // The fortran interface doesn't pass in the buffer size, so just assume it is correct // if the size is specified to be zero if (count == 0) count = requiredBufferSize_; ASSERT(!manageOwnBuffer_); out_ = pos_ = out; end_ = out_ + count; bufferElements_ = count; ASSERT(bufferElements_ >= requiredBufferSize_); } void SQLSelectOutput::print(std::ostream& s) const { s << "SQLSelectOutput"; } void SQLSelectOutput::reset() { pos_ = out_; } void SQLSelectOutput::flush() {} bool SQLSelectOutput::output(const expression::Expressions& results) { ASSERT(results.size() == columnSizesDoubles_.size()); pos_ = out_; for (currentColumn_ = 0; currentColumn_ < columnSizesDoubles_.size(); currentColumn_++) { results[currentColumn_]->output(*this); } ASSERT(pos_ == end_); count_++; if (isNewDataset_) { if (newDatasetOutputted_) { isNewDataset_ = false; newDatasetOutputted_ = false; } else { newDatasetOutputted_ = true; } } return true; } void SQLSelectOutput::outputNumber(double x, bool missing) { ASSERT(pos_ >= out_ && pos_ < end_); *pos_++ = (missing ? missingValues_[currentColumn_] : x); } // TODO: We can add special missing-value behaviour here --- with user specified missing values! void SQLSelectOutput::outputReal(double x, bool missing) { outputNumber(x, missing); } void SQLSelectOutput::outputDouble(double x, bool missing) { outputNumber(x, missing); } void SQLSelectOutput::outputInt(double x, bool missing) { outputNumber(x, missing); } void SQLSelectOutput::outputUnsignedInt(double x, bool missing) { outputNumber(x, missing); } void SQLSelectOutput::outputBitfield(double x, bool missing) { outputNumber(x, missing); } void SQLSelectOutput::outputString(const char* s, size_t len, bool missing) { ASSERT(pos_ >= out_ && (pos_ + columnSizesDoubles_[currentColumn_]) <= end_); size_t charSize = columnSizesDoubles_[currentColumn_] * sizeof(double); if (len > charSize) { std::ostringstream ss; ss << "String too long for configured output: " << len << " > " << charSize; throw SeriousBug(ss.str(), Here()); } if (missing) { len = 0; } else { ::memcpy(reinterpret_cast(pos_), s, len); } if (len < charSize) { ::memset(&reinterpret_cast(pos_)[len], 0, charSize - len); } pos_ += columnSizesDoubles_[currentColumn_]; } void SQLSelectOutput::prepare(SQLSelect& sql) { updateTypes(sql); } void SQLSelectOutput::updateTypes(SQLSelect& sql) { size_t offset = 0; expression::Expressions output(sql.output()); metaData_.setSize(output.size()); offsets_.clear(); offsets_.reserve(output.size()); columnSizesDoubles_.clear(); columnSizesDoubles_.reserve(output.size()); missingValues_.clear(); missingValues_.reserve(output.size()); // TODO: What happens here if the metadata/columns change during an odb? // --> We need to update this allocation as we go. isNewDataset_ = true; newDatasetOutputted_ = false; for (size_t i = 0; i < output.size(); i++) { // Column sizes for output const auto& column(output[i]); size_t colSizeBytes = column->type()->size(); ASSERT(colSizeBytes % 8 == 0); columnSizesDoubles_.push_back(colSizeBytes / 8); offsets_.push_back(offset); offset += columnSizesDoubles_.back(); // Update the metadata metaData_[i]->name(column->title()); metaData_[i]->type(sqlToOdbType(*column->type())); metaData_[i]->hasMissing(column->hasMissingValue()); metaData_[i]->bitfieldDef(column->bitfieldDef()); metaData_[i]->dataSizeDoubles(columnSizesDoubles_.back()); // n.b. missing value in the target can be different than for the source. And it can change in the source // between frames. So don't set the target missing value to the sources, use the default one associated // with the default encoder. missingValues_.push_back(metaData_[i]->missingValue()); } requiredBufferSize_ = std::accumulate(columnSizesDoubles_.begin(), columnSizesDoubles_.end(), 0); // Buffer allocation if necessary if (manageOwnBuffer_) { data_.resize(offset); pos_ = out_ = &data_[0]; end_ = pos_ + offset; bufferElements_ = offset; // If a buffer is being provided at the time of doing each request, then we need to // test against the buffer size then, not now. ASSERT(bufferElements_ >= requiredBufferSize_); } } void SQLSelectOutput::cleanup(SQLSelect& sql) {} unsigned long long SQLSelectOutput::count() { return count_; } //---------------------------------------------------------------------------------------------------------------------- } // namespace sql } // namespace odc odc-1.6.3/src/odc/ODAUpdatingIterator.cc0000664000175000017500000000430215146027420020126 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// /// \file ODAUpdatingIterator.cc /// /// @author Piotr Kuchta, June 2009 #include "odc/core/MetaData.h" namespace odc { template ODAUpdatingIterator::ODAUpdatingIterator(T& ii, const T& end, const std::vector& columns, const std::vector& values) : ii_(ii), end_(end), columns_(columns), columnIndices_(columns.size()), values_(values), data_(0), refCount_(0), noMore_(false) { ASSERT(columns.size() == values.size()); updateIndices(); std::copy(ii_->data(), ii_->data() + ii_->columns().size(), data_); update(); } template void ODAUpdatingIterator::updateIndices() { const core::MetaData& md(ii_->columns()); delete[] data_; data_ = new double[md.size()]; for (size_t i = 0; i < columns_.size(); ++i) columnIndices_[i] = md.columnIndex(columns_[i]); } template ODAUpdatingIterator::ODAUpdatingIterator(const T& end) : ii_(end), end_(end), columnIndices_(), values_(), data_(0), refCount_(0), noMore_(true) {} template ODAUpdatingIterator::~ODAUpdatingIterator() { delete[] data_; } template void ODAUpdatingIterator::update() { for (size_t i = 0; i < columnIndices_.size(); ++i) data_[columnIndices_[i]] = values_[i]; } template bool ODAUpdatingIterator::isNewDataset() { return ii_->isNewDataset(); } template bool ODAUpdatingIterator::next() { if (noMore_) return noMore_; ++ii_; bool r = ii_ != end_; if (r) { if (ii_->isNewDataset()) updateIndices(); std::copy(ii_->data(), ii_->data() + ii_->columns().size(), data_); update(); } noMore_ = !r; return r; } } // namespace odc odc-1.6.3/src/odc/ReaderIterator.cc0000664000175000017500000001536215146027420017241 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/io/DataHandle.h" #include "eckit/log/Log.h" #include "odc/LibOdc.h" #include "odc/Reader.h" #include "odc/ReaderIterator.h" #include "odc/core/Codec.h" #include "odc/core/Header.h" using namespace eckit; using namespace odc::core; namespace odc { ReaderIterator::ReaderIterator(Reader& owner) : owner_(owner), columns_(0), lastValues_(0), columnOffsets_(0), rowDataSizeDoubles_(0), nrows_(0), rowsRemainingInTable_(0), f_(owner_.dataHandle()->clone()), newDataset_(false), rowDataBuffer_(0), noMore_(false), headerCounter_(0), byteOrder_(BYTE_ORDER_INDICATOR), refCount_(0) { ASSERT(f_); f_->openForRead(); loadHeaderAndBufferData(); } eckit::DataHandle* ReaderIterator::dataHandle() { ASSERT(f_); return f_.get(); } ReaderIterator::ReaderIterator(Reader& owner, const PathName& pathName) : owner_(owner), columns_(0), lastValues_(0), columnOffsets_(0), rowDataSizeDoubles_(0), nrows_(0), rowsRemainingInTable_(0), f_(pathName.fileHandle()), newDataset_(false), rowDataBuffer_(0), noMore_(false), headerCounter_(0), byteOrder_(BYTE_ORDER_INDICATOR), refCount_(0) { ASSERT(f_); f_->openForRead(); loadHeaderAndBufferData(); } bool ReaderIterator::loadHeaderAndBufferData() { if (noMore_) return false; ASSERT(rowsRemainingInTable_ == 0); // Keep going until we find a valid header, or run out of data // n.b. an empty table is legit, so we need a loop. while (true) { // Check the magic. If no more data, we are done if (!Header::readMagic(*f_)) { noMore_ = true; return false; } // Read in the rest of the header Header header(columns_, properties_); header.loadAfterMagic(*f_); byteOrder_ = header.byteOrder(); rowDataSizeDoubles_ = rowDataSizeDoublesInternal(); ++headerCounter_; // Ensure the decode buffers are all set up initRowBuffer(); // Read in the data into a buffer and initialise the DataStream. size_t dataSize = header.dataSize(); // It is perfectly legitimate to have zero rows in an ODB. If that is the case, // then loop around again. if (dataSize == 0) { ASSERT(header.rowsNumber() == 0); } else { // Read the expected data into the rows buffer. ASSERT(header.rowsNumber() != 0); ASSERT(dataSize >= 2); if (!readBuffer(dataSize)) { // See ODB-376 throw SeriousBug("Expected row data to follow table header"); } // And we are done newDataset_ = true; rowsRemainingInTable_ = header.rowsNumber(); return true; } } } ReaderIterator::~ReaderIterator() noexcept(false) { LOG_DEBUG_LIB(LibOdc) << "ReaderIterator::~ReaderIterator: headers read: " << headerCounter_ << " rows:" << nrows_ << std::endl; close(); delete[] lastValues_; delete[] columnOffsets_; } bool ReaderIterator::operator!=(const ReaderIterator& other) { // ASSERT(&other == 0); return noMore_; } void ReaderIterator::initRowBuffer() { int32_t numDoubles = rowDataSizeDoubles(); size_t nCols = columns().size(); delete[] lastValues_; lastValues_ = new double[numDoubles]; codecs_.clear(); codecs_.resize(nCols, 0); delete[] columnOffsets_; columnOffsets_ = new size_t[nCols]; size_t offset = 0; for (size_t i = 0; i < nCols; i++) { codecs_[i] = &columns()[i]->coder(); lastValues_[offset] = codecs_[i]->missingValue(); columnOffsets_[i] = offset; offset += columns()[i]->dataSizeDoubles(); } } size_t ReaderIterator::readBuffer(size_t dataSize) { // Ensure we have enough buffer space if (rowDataBuffer_.size() < dataSize) { rowDataBuffer_ = eckit::Buffer(dataSize); } // Read the data into a buffer size_t bytesRead = f_->read(rowDataBuffer_, dataSize); if (bytesRead == 0) return 0; if (bytesRead != dataSize) { std::stringstream ss; ss << "Failed to read " << dataSize << " bytes of encoded data"; throw ODBIncomplete(ss.str(), Here()); } // Assign the data to a DataStream. rowDataStream_ = GeneralDataStream(byteOrder_ != BYTE_ORDER_INDICATOR, rowDataBuffer_); // Assign the appropriate data stream to each of the codecs. for (auto& codec : codecs_) codec->setDataStream(rowDataStream_); return bytesRead; } bool ReaderIterator::next() { newDataset_ = false; if (noMore_) return false; if (rowsRemainingInTable_ == 0) { if (!loadHeaderAndBufferData()) return false; ASSERT(rowsRemainingInTable_ != 0); } unsigned char marker[2]; rowDataStream_.readBytes(marker, sizeof(marker)); int startCol = (marker[0] * 256) + marker[1]; size_t nCols = columns().size(); for (size_t i = startCol; i < nCols; i++) { codecs_[i]->decode(&lastValues_[columnOffsets_[i]]); } ++nrows_; --rowsRemainingInTable_; return nCols; } size_t ReaderIterator::rowDataSizeDoublesInternal() const { size_t total = 0; for (const auto& column : columns()) { total += column->dataSizeDoubles(); } return total; } bool ReaderIterator::isNewDataset() { return newDataset_; } double& ReaderIterator::data(size_t i) { ASSERT(i >= 0 && i < columns().size()); return lastValues_[columnOffsets_[i]]; } int ReaderIterator::close() { if (f_) f_->close(); f_.reset(); return 0; } std::string ReaderIterator::property(std::string key) { return properties_[key]; } api::ColumnType ReaderIterator::columnType(unsigned long index) { return columns_[index]->type(); } const std::string& ReaderIterator::columnName(unsigned long index) const { return columns_[index]->name(); } const std::string& ReaderIterator::codecName(unsigned long index) const { return columns_[index]->coder().name(); } double ReaderIterator::columnMissingValue(unsigned long index) { return columns_[index]->missingValue(); } const eckit::sql::BitfieldDef& ReaderIterator::bitfieldDef(unsigned long index) { return columns_[index]->bitfieldDef(); } } // namespace odc odc-1.6.3/src/odc/Writer.cc0000664000175000017500000001025515146027420015575 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// /// \file Writer.cc /// /// @author Piotr Kuchta, Feb 2009 #include "odc/Writer.h" #include #include #include #include #include // #include using namespace std; #include "eckit/config/Resource.h" #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/PathName.h" #include "eckit/io/DataHandle.h" #include "eckit/io/FileDescHandle.h" #include "odc/ODBAPISettings.h" #include "odc/WriterBufferingIterator.h" #include "odc/codec/CodecOptimizer.h" #include "odc/core/Codec.h" #include "odc/core/Column.h" #include "odc/core/DataStream.h" #include "odc/core/MetaData.h" namespace odc { template Writer::Writer() : path_(""), dataHandle_(0), rowsBufferSize_( eckit::Resource("$ODB_ROWS_BUFFER_SIZE;-rowsBufferSize;rowsBufferSize", DEFAULT_ROWS_BUFFER_SIZE)), openDataHandle_(true), deleteDataHandle_(true) {} template Writer::Writer(const eckit::PathName& path) : path_(path), dataHandle_(0), rowsBufferSize_( eckit::Resource("$ODB_ROWS_BUFFER_SIZE;-rowsBufferSize;rowsBufferSize", DEFAULT_ROWS_BUFFER_SIZE)), openDataHandle_(true), deleteDataHandle_(true) { if (path_ == "/dev/stdout" || path_ == "stdout") { eckit::Log::info() << "Writing to stdout" << std::endl; dataHandle_ = new eckit::FileDescHandle(1); openDataHandle_ = false; } } template Writer::Writer(eckit::DataHandle* dh, bool openDataHandle, bool deleteDataHandle) : path_(""), dataHandle_(dh), rowsBufferSize_( eckit::Resource("$ODB_ROWS_BUFFER_SIZE;-rowsBufferSize;rowsBufferSize", DEFAULT_ROWS_BUFFER_SIZE)), openDataHandle_(openDataHandle), deleteDataHandle_(deleteDataHandle) {} template Writer::Writer(eckit::DataHandle& dh, bool openDataHandle) : path_(""), dataHandle_(&dh), rowsBufferSize_( eckit::Resource("$ODB_ROWS_BUFFER_SIZE;-rowsBufferSize;rowsBufferSize", DEFAULT_ROWS_BUFFER_SIZE)), openDataHandle_(openDataHandle), deleteDataHandle_(false) {} template Writer::~Writer() { if (deleteDataHandle_) delete dataHandle_; } template typename Writer::iterator Writer::begin(bool openDataHandle) { eckit::DataHandle* dh = 0; if (dataHandle_ == 0) { dh = ODBAPISettings::instance().writeToFile(path_, eckit::Length(0), false); return typename Writer::iterator(new ITERATOR(*this, dh, openDataHandle)); } else { ASSERT(dataHandle_); dh = dataHandle_; return typename Writer::iterator(new ITERATOR(*this, *dh, openDataHandle)); } } template ITERATOR* Writer::createWriteIterator(eckit::PathName pathName, bool append) { eckit::Length estimatedLength = 10 * 1024 * 1024; eckit::DataHandle* h = append ? ODBAPISettings::instance().appendToFile(pathName, estimatedLength) : ODBAPISettings::instance().writeToFile(pathName, estimatedLength); return new ITERATOR(*this, h, false); } // Explicit templates' instantiations. template Writer::Writer(); template Writer::Writer(const eckit::PathName&); template Writer::Writer(eckit::DataHandle&, bool); template Writer::Writer(eckit::DataHandle*, bool, bool); template Writer::~Writer(); template Writer::iterator Writer::begin(bool); template WriterBufferingIterator* Writer::createWriteIterator(eckit::PathName, bool); } // namespace odc odc-1.6.3/src/odc/Select.cc0000664000175000017500000000616615146027420015546 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Piotr Kuchta /// @author Simon Smart /// @date April 2010 #include "eckit/filesystem/PathName.h" #include "eckit/io/DataHandle.h" #include "odc/Select.h" #include "odc/SelectIterator.h" #include "odc/sql/SQLOutputConfig.h" #include "odc/sql/SQLSelectOutput.h" #include "odc/sql/TODATable.h" using namespace eckit; namespace odc { //---------------------------------------------------------------------------------------------------------------------- // TODO: Select should BE a SelectIterator, not posess one. Select::Select(const std::string& selectStatement, bool manageOwnBuffer) : selectStatement_(selectStatement), session_(std::unique_ptr(new sql::SQLSelectOutput(manageOwnBuffer)), std::unique_ptr(new odc::sql::SQLOutputConfig)), initted_(false), it_(nullptr) {} Select::Select(const std::string& selectStatement, DataHandle& dh, bool /* manageOwnBuffer */) : Select(selectStatement, true) { dh.openForRead(); eckit::sql::SQLDatabase& db(session_.currentDatabase()); db.addImplicitTable(new odc::sql::ODATable(db, dh)); } Select::Select(const std::string& selectStatement, const eckit::PathName& path, bool /* manageOwnBuffer */) : Select(selectStatement) { ownDH_.reset(path.fileHandle()); ownDH_->openForRead(); eckit::sql::SQLDatabase& db(session_.currentDatabase()); db.addImplicitTable(new odc::sql::ODATable(db, *ownDH_)); } Select::Select(const std::string& selectStatement, const char* path, bool manageOwnBuffer) : Select(selectStatement, std::string(path), manageOwnBuffer) {} Select::~Select() noexcept(false) { if (ownDH_) { ownDH_->close(); } } eckit::sql::SQLDatabase& Select::database() { return session_.currentDatabase(); } SelectIterator* Select::createSelectIterator(const std::string& sql) { sql::SQLSelectOutput* output = dynamic_cast(&session_.output()); ASSERT(output); return new SelectIterator(sql, session_, *output); } const Select::iterator Select::end() { return iterator(nullptr); } // This is horrible, but the TextReader, and any stream based iteraton, can only // iterate once, so we MUST NOT create two iterators if begin() is called twice. Select::iterator Select::begin() { if (!initted_) { SelectIterator* it = createSelectIterator(selectStatement_); ASSERT(it); it->next(); it_ = iterator(it); initted_ = true; } return it_; } #ifdef SWIGPYTHON template odc::IteratorProxy; #endif //---------------------------------------------------------------------------------------------------------------------- } // namespace odc odc-1.6.3/src/odc/ODBAPIVersion.cc0000664000175000017500000000205315146027420016622 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// /// This file will be updated autmatically by the make.sms script /// #include "odc/ODBAPIVersion.h" #include "odc/core/Header.h" #include "odc_config.h" namespace odc { const char* ODBAPIVersion::version() { return odc_VERSION_STR; } const char* gitsha1() { return odc_GIT_SHA1; } unsigned int ODBAPIVersion::formatVersionMajor() { return core::FORMAT_VERSION_NUMBER_MAJOR; } unsigned int ODBAPIVersion::formatVersionMinor() { return core::FORMAT_VERSION_NUMBER_MINOR; } const char* ODBAPIVersion::installPrefix() { return odc_INSTALL_PREFIX; } const char* ODBAPIVersion::buildDirectory() { return odc_BINARY_DIR; } } // namespace odc odc-1.6.3/src/odc/StringTool.h0000664000175000017500000000357715146027420016300 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// \file StringTool.h /// /// @author Piotr Kuchta, ECMWF, Oct 2010 #ifndef StringTool_H #define StringTool_H #include "eckit/eckit.h" #include "odc/api/ColumnType.h" namespace eckit { class PathName; class CodeLocation; } // namespace eckit namespace odc { class StringTool { typedef int (*ctypeFun)(int); public: static std::string readFile(const eckit::PathName fileName, bool logging = false); static std::vector readLines(const eckit::PathName fileName, bool logging = false); static void trimInPlace(std::string&); static bool match(const std::string& regex, const std::string&); static bool matchAny(const std::vector& regs, const std::string&); static bool check(const std::string&, ctypeFun); static bool isInQuotes(const std::string&); static std::string unQuote(const std::string&); static double cast_as_double(const std::string&); static std::string double_as_string(double); // static std::string stringAsDouble(double v) static std::string int_as_double2string(double); static int shell(std::string cmd, const eckit::CodeLocation& where, bool assertSuccess = true); static double translate(const std::string& v); static std::string valueAsString(double, api::ColumnType); static std::string patchTimeForMars(const std::string& v); static bool isSelectStatement(const std::string&); }; std::ostream& operator<<(std::ostream&, const std::vector&); } // namespace odc #endif odc-1.6.3/src/odc/Writer.h0000664000175000017500000000346315146027420015442 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// /// \file Writer.h /// /// @author Piotr Kuchta, Feb 2009 #ifndef ODAWRITER_H #define ODAWRITER_H #include "odc/IteratorProxy.h" #include "odc/WriterBufferingIterator.h" namespace eckit { class PathName; } namespace eckit { class DataHandle; } namespace odc { typedef WriterBufferingIterator DefaultWritingIterator; template class Writer { enum { DEFAULT_ROWS_BUFFER_SIZE = 10000 }; public: typedef ITERATOR iterator_class; typedef IteratorProxy iterator; Writer(const eckit::PathName& path); Writer(eckit::DataHandle&, bool openDataHandle = true); Writer(eckit::DataHandle*, bool openDataHandle = true, bool deleteDataHandle = false); Writer(); virtual ~Writer(); iterator begin(bool openDataHandle = true); eckit::DataHandle& dataHandle() { return *dataHandle_; } ITERATOR* createWriteIterator(eckit::PathName, bool append = false); unsigned long rowsBufferSize() { return rowsBufferSize_; } Writer& rowsBufferSize(unsigned long n) { rowsBufferSize_ = n; } const eckit::PathName path() { return path_; } private: // No copy allowed Writer(const Writer&); Writer& operator=(const Writer&); const eckit::PathName path_; eckit::DataHandle* dataHandle_; unsigned long rowsBufferSize_; bool openDataHandle_; bool deleteDataHandle_; }; } // namespace odc #endif odc-1.6.3/src/odc/odc_config.f90.in0000664000175000017500000000025015146027420017023 0ustar alastairalastair module odc_config character(*), parameter :: odc_version_str = "@odc_VERSION_STR@" character(*), parameter :: odc_git_sha1_str = "@odc_GIT_SHA1@" end module odc-1.6.3/src/odc/tools/0000775000175000017500000000000015146027420015147 5ustar alastairalastairodc-1.6.3/src/odc/tools/TestFunctionDateAndTime.cc0000664000175000017500000000357015146027420022150 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// \file UnitTest.cc /// /// @author ECMWF, July 2010 const double EPS = 1e-6; #include #include "eckit/log/Timer.h" #include "odc/Select.h" #include "TestCase.h" #include "odc/Writer.h" using namespace std; using namespace eckit; using namespace odc; static void setUp() { Timer t("Test DateAndTime function"); odc::Writer<> oda("test_date_and_time.odb"); odc::Writer<>::iterator row = oda.begin(); row->setNumberOfColumns(2); row->setColumn(0, "date", odc::api::INTEGER); row->setColumn(1, "time", odc::api::INTEGER); row->writeHeader(); (*row)[0] = 20090706.0; (*row)[1] = 210109.0; ++row; } static void tearDown() { PathName("test_date_and_time.odb").unlink(); } static void test() { const string sql = "select julian(date,time), year(date),month(date),day(date),hour(time),minute(time),second(time), " "timestamp(date,time) from \"test_date_and_time.odb\";"; Log::info() << "Executing: '" << sql << "'" << std::endl; odc::Select oda(sql); odc::Select::iterator it = oda.begin(); ASSERT(fabs((*it)[0] - 2455019.) < EPS); // ASSERT((*it)[1] == 2009); // ASSERT((*it)[2] == 7); // ASSERT((*it)[3] == 6); // ASSERT((*it)[4] == 21); // ASSERT((*it)[5] == 1); // ASSERT((*it)[6] == 9); // ASSERT((*it)[7] == 20090706210109ll); // } SIMPLE_TEST(FunctionDateAndTime) odc-1.6.3/src/odc/tools/TestCase.cc0000664000175000017500000000167615146027420017203 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// \file TestCase.h /// /// @author Piotr Kuchta, ECMWF, Feb 2009 #include #include #include using namespace std; #include "TestCase.h" #include "Tool.h" #include "ToolFactory.h" namespace odc { namespace tool { namespace test { TestCase::TestCase(int argc, char** argv) : Tool(argc, argv) {} void TestCase::run() { setUp(); test(); tearDown(); } TestCase::~TestCase() {} void TestCase::setUp() {} void TestCase::test() {} void TestCase::tearDown() {} } // namespace test } // namespace tool } // namespace odc odc-1.6.3/src/odc/tools/ImportTool.h0000664000175000017500000000247315146027420017436 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef ImportTool_H #define ImportTool_H #include "eckit/filesystem/PathName.h" #include "odc/tools/Tool.h" namespace odc { namespace tool { class ImportTool : public Tool { public: ImportTool(int argc, char* argv[]); void run(); static void help(std::ostream& o); static void usage(const std::string& name, std::ostream& o); static void importFile(const eckit::PathName& in, const eckit::PathName& out, const std::string& delimiter = defaultDelimiter()); static void filterAndImportFile(const eckit::PathName& in, const eckit::PathName& out, const std::string& sql, const std::string& delimiter = defaultDelimiter()); static std::string defaultDelimiter() { return ","; } private: // No copy allowed ImportTool(const ImportTool&); ImportTool& operator=(const ImportTool&); }; } // namespace tool } // namespace odc #endif odc-1.6.3/src/odc/tools/TestFunctionsForAngleConversion.cc0000664000175000017500000000510115146027420023747 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// \file UnitTest.cc /// /// @author ECMWF, July 2010 const double EPS = 7e-6; #include "eckit/log/Timer.h" #include "odc/Select.h" #include "TestCase.h" #include "odc/Writer.h" #include using namespace std; using namespace eckit; using namespace odc; static void setUp() { Timer t("Test various functions to convert angles (radians to degrees, etc.)"); odc::Writer<> oda("test_angleconv.odb"); odc::Writer<>::iterator row = oda.begin(); row->setNumberOfColumns(2); row->setColumn(0, "radian_col", odc::api::REAL); row->setColumn(1, "degrees_col", odc::api::REAL); row->writeHeader(); (*row)[0] = M_PI; (*row)[1] = 180.0e0; ++row; (*row)[0] = 0.0e0; (*row)[1] = 0.0e0; ++row; (*row)[0] = M_PI / 4.0e0; (*row)[1] = 45.0e0; ++row; } static void tearDown() { PathName("test_angleconv.odb").unlink(); } static void test() { const string sql = "select degrees(radian_col),radians(degrees_col), rad2deg(radian_col), deg2rad(degrees_col), " "radians(degrees(radian_col)), degrees(radians(degrees_col)) from \"test_angleconv.odb\";"; Log::info() << "Executing: '" << sql << "'" << std::endl; odc::Select oda(sql); odc::Select::iterator it = oda.begin(); // because stored as single real precision; we loose some accuracy ASSERT(fabs((*it)[0] - 180) < EPS); // ASSERT(fabs((*it)[1] - M_PI) < EPS); // ASSERT(fabs((*it)[2] - 180.0) < EPS); // ASSERT(fabs((*it)[3] - M_PI) < EPS); // ASSERT(fabs((*it)[4] - M_PI) < EPS); // ASSERT(fabs((*it)[5] - 180.0e0) < EPS); // ++it; ASSERT(fabs((*it)[0]) < EPS); // ASSERT(fabs((*it)[1]) < EPS); // ASSERT(fabs((*it)[2]) < EPS); // ASSERT(fabs((*it)[3]) < EPS); // ASSERT(fabs((*it)[4]) < EPS); // ASSERT(fabs((*it)[5]) < EPS); // ++it; ASSERT(fabs((*it)[0] - 45) < EPS); // ASSERT(fabs((*it)[1] - M_PI / 4.0) < EPS); // ASSERT(fabs((*it)[2] - 45.0) < EPS); // ASSERT(fabs((*it)[3] - M_PI / 4.0) < EPS); // ASSERT(fabs((*it)[4] - M_PI / 4.0) < EPS); // ASSERT(fabs((*it)[5] - 45.0e0) < EPS); // } SIMPLE_TEST(FunctionsForAngleConversion) odc-1.6.3/src/odc/tools/TestFunctionTypeConversion.cc0000664000175000017500000000300415146027420023010 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// \file UnitTest.cc /// /// @author ECMWF, July 2010 #include "eckit/log/Timer.h" #include "odc/Select.h" #include "TestCase.h" #include "odc/Writer.h" using namespace std; using namespace eckit; using namespace odc; static void setUp() { Timer t("Test TypeConversion function"); odc::Writer<> oda("test_type_conversion.odb"); odc::Writer<>::iterator row = oda.begin(); row->setNumberOfColumns(1); row->setColumn(0, "obsvalue", odc::api::REAL); row->writeHeader(); (*row)[0] = 247.53; ++row; } static void tearDown() { PathName("test_type_conversion.odb").unlink(); } static void test() { const string sql = "select ceil(obsvalue),floor(obsvalue), trunc(obsvalue),int(obsvalue),nint(obsvalue) from " "\"test_type_conversion.odb\";"; Log::info() << "Executing: '" << sql << "'" << std::endl; odc::Select oda(sql); odc::Select::iterator it = oda.begin(); ASSERT((*it)[0] == 248); // ASSERT((*it)[1] == 247); // ASSERT((*it)[2] == 247); // ASSERT((*it)[3] == 247); // ASSERT((*it)[4] == 248); // } SIMPLE_TEST(FunctionTypeConversion) odc-1.6.3/src/odc/tools/IndexTool.cc0000664000175000017500000000357215146027420017372 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "odc/tools/IndexTool.h" #include "eckit/eckit.h" #include "eckit/exception/Exceptions.h" #include "eckit/io/Length.h" #include "eckit/io/Offset.h" #include "eckit/io/PartFileHandle.h" #include "odc/Indexer.h" #include "odc/Reader.h" #include "odc/Select.h" #include "odc/core/MetaData.h" #include "odc/tools/CountTool.h" using namespace eckit; namespace odc { namespace tool { IndexTool::IndexTool(int argc, char* argv[]) : Tool(argc, argv) {} void IndexTool::help(std::ostream& o) { o << "Creates index of reports for a given file"; } void IndexTool::usage(const std::string& name, std::ostream& o) { o << name << " [] " << std::endl << std::endl << "\tSpecifically the index file is an ODB file with (INTEGER) columns: block_begin, block_length, seqno, n_rows" << std::endl << "\tOne entry is made for each unique seqno - block pair within the source ODB file." << std::endl; } void IndexTool::run() { if (!(parameters().size() == 2 || parameters().size() == 3)) { Log::error() << "Usage: "; usage(parameters(0), Log::error()); Log::error() << std::endl; std::stringstream ss; ss << "Expected exactly 2 or 3 command line parameters"; throw UserError(ss.str()); } PathName dataFile(parameters(1)); PathName indexFile(parameters().size() == 3 ? parameters(2) : parameters(1) + ".idx"); Indexer::createIndex(dataFile, indexFile); } } // namespace tool } // namespace odc odc-1.6.3/src/odc/tools/ImportTool.cc0000664000175000017500000000736215146027420017576 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "odc/tools/ImportTool.h" #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/PathName.h" #include "eckit/io/FileHandle.h" #include "eckit/log/Log.h" #include "eckit/sql/SQLParser.h" #include "eckit/sql/SQLSession.h" #include "eckit/sql/SQLStatement.h" #include "eckit/utils/StringTools.h" #include "odc/api/Odb.h" #include "odc/sql/SQLOutputConfig.h" #include "odc/sql/TODATable.h" using namespace std; using namespace eckit; namespace odc { namespace tool { // TODO: A test with wide strings // TODO: A test with SQL filtering // TODO: A test with non-comma delimiters ImportTool::ImportTool(int argc, char* parameters[]) : Tool(argc, parameters) { registerOptionWithArgument("-d"); // Delimiter registerOptionWithArgument("-sql"); // SQL to filter input CSV with } void ImportTool::help(std::ostream& o) { o << "Imports data from a text file"; } void ImportTool::usage(const std::string& name, std::ostream& o) { o << name << "\t[-d delimiter] " << std::endl << std::endl << "\tdelimiter can be a single character (e.g.: ',') or TAB. As a data example:" << std::endl << std::endl << "\tcol1:INTEGER,col2:REAL" << std::endl << "\t1,2.0" << std::endl << "\t3,4.0" << std::endl; } void ImportTool::run() { if (parameters().size() != 3) { Log::error() << "Usage: "; usage(parameters(0), Log::error()); Log::error() << std::endl; std::stringstream ss; ss << "Expected exactly 3 command line parameters"; throw UserError(ss.str()); } PathName inFile(parameters(1)), outFile(parameters(2)); Log::info() << "ImportTool::run: inFile: " << inFile << ", outFile: " << outFile << std::endl; std::string delimiter(StringTools::upper(optionArgument("-d", defaultDelimiter()))); delimiter = delimiter == "TAB" ? "\t" : delimiter == "SPACE" ? " " : delimiter; std::string sql(optionArgument("-sql", std::string("select *;"))); if (sql == "select *;") { FileHandle dh_in(inFile); FileHandle dh_out(outFile); dh_in.openForRead(); AutoClose close_in(dh_in); dh_out.openForWrite(0); AutoClose close_out(dh_out); size_t n = api::odbFromCSV(dh_in, dh_out); Log::info() << "ImportTool::odbFromCSV: Copied " << n << " rows." << std::endl; } else { filterAndImportFile(inFile, outFile, sql, delimiter); } } void ImportTool::importFile(const PathName& in, const PathName& out, const std::string& delimiter) { filterAndImportFile(in, out, "select *;", delimiter); } void ImportTool::filterAndImportFile(const PathName& in, const PathName& out, const std::string& sql, const std::string& delimiter) { // TODO: Why are we not using the ODAOutput directly, rather than going via a Select, Writer combination? eckit::sql::SQLSession session(std::unique_ptr(new odc::sql::SQLOutputConfig(out))); eckit::sql::SQLDatabase& db(session.currentDatabase()); db.addImplicitTable(new odc::sql::ODBCSVTable(db, in, in, delimiter)); session.currentDatabase(); eckit::sql::SQLParser().parseString(session, sql); size_t n = session.statement().execute(); Log::info() << "ImportTool::importFile: Copied " << n << " rows." << std::endl; } } // namespace tool } // namespace odc odc-1.6.3/src/odc/tools/TestRunner.cc0000664000175000017500000001525315146027420017575 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// \file TestRunner.h /// /// @author Piotr Kuchta, ECMWF, Feb 2009 #include #include #include #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/PathName.h" #include "eckit/log/Log.h" #include "eckit/log/Timer.h" #include "eckit/utils/StringTools.h" #include "odc/CommandLineParser.h" #include "odc/LibOdc.h" #include "odc/tools/TestCase.h" #include "odc/tools/TestRunner.h" #include "odc/tools/Tool.h" #include "odc/tools/ToolFactory.h" using namespace eckit; using namespace std; namespace odc { namespace tool { namespace test { TestRunner::TestRunner(CommandLineParser& clp) : clp_(clp), mars_sms_label_(false), label_() { if (getenv("MARS_SMS_LABEL")) { mars_sms_label_ = true; label_ = getenv("MARS_SMS_LABEL"); } } TestRunner::~TestRunner() {} void TestRunner::run() { ASSERT(getenv("odc_TEST_DATA_PATH") && "odc_TEST_DATA_PATH must be set"); stringstream totalRunningTime; unique_ptr allTestsTimer(new Timer("Total", totalRunningTime)); unique_ptr tests; failed_.clear(); if (clp_.parameters().size() == 1) { tests.reset(AbstractToolFactory::testCases()); Log::info() << "clp_.parameters()" << clp_.parameters() << endl; runTests(*tests); } else { // TODO: keep the config in the odc_TEST_DATA_PATH // readConfig("../../../odc/src/odb/TestRunnerApplication.cfg"); readConfig("/tmp/Dropbox/work/odc/src/odb/TestRunnerApplication.cfg"); readConfig("/tmp/Dropbox/work/odc/src/tools/TestRunnerApplication.cfg"); tests.reset(new TestCases()); for (size_t i = 1; i < clp_.parameters().size(); ++i) { string suiteName = clp_.parameters()[i]; ASSERT("Suite does not exist" && suites_.find(suiteName) != suites_.end()); vector& suite = suites_[suiteName]; unique_ptr > tsts(AbstractToolFactory::testCases(suite)); runTests(*tsts); tests->insert(tests->end(), tsts->begin(), tsts->end()); } } allTestsTimer.reset(); ofstream xmlf("testresults.xml"); xmlf << "" << endl; xmlf << "" << endl; xmlf << xml_.str(); xmlf << "" << endl; size_t nTests = tests->size(); for (size_t i = 0; i < nTests; ++i) delete (*tests)[i]; if (failed_.size() == 0) { Log::info() << endl << "+- Phew, made it! All " << nTests << " tests passed successfully. " << endl << endl; Log::info() << runningTimes_.str() << endl; ; Log::info() << totalRunningTime.str() << endl; } else { Log::error() << endl << "+- Summary: " << failed_.size() << " test(s) failed." << endl; for (vector::iterator it = failed_.begin(); it != failed_.end(); ++it) { const string& name = it->first; const string& what = it->second; Log::error() << "\t" << name << ": " << endl << what; } Log::error() << endl; stringstream ss; ss << " " << failed_.size() << " test(s) failed"; throw eckit::SeriousBug(ss.str()); } } void TestRunner::runTests(const TestCases& tests) { for (TestCases::const_iterator it = tests.begin(); it != tests.end(); ++it) { bool exceptionThrown = false; string what; TestCase* tst = *it; const string& name = tst->name(); Log::info() << "+- Running " << name << " ..." << endl; smslabel(name); stringstream runningTime; unique_ptr timer(new Timer(name, runningTime)); try { tst->setUp(); tst->test(); } catch (std::exception& e) { Log::warning() << "+- FAILED" << endl; exceptionThrown = true; what += string(e.what()) + '\n'; } catch (...) { Log::warning() << "+- FAILED: unknown exception!" << endl; exceptionThrown = true; what += string("Uknown exception") + '\n'; } try { tst->tearDown(); } catch (std::exception& e) { Log::warning() << "+- Exception thrown from tearDown." << endl; exceptionThrown = true; what += string("[In tearDown:]") + string(e.what()) + '\n'; } catch (...) { Log::warning() << "+- FAILED: unknown exception!" << endl; exceptionThrown = true; what += string("Uknown exception") + '\n'; } if (exceptionThrown) { failed_.push_back(make_pair(name, what)); xml_ << "" << endl; xml_ << " " << endl; xml_ << "" << endl; } else { timer.reset(); runningTimes_ << runningTime.str(); Log::info() << "+- Passed." << endl << endl; xml_ << "" << endl; } } } void TestRunner::readConfig(const PathName fileName) { LOG_DEBUG_LIB(LibOdc) << "TestRunner::readConfig: reading file '" << fileName << "'" << endl; suites_.clear(); vector lines = StringTool::readLines(fileName); for (size_t i = 0; i < lines.size(); ++i) { vector words = StringTools::split(":", lines[i]); if (words.size() == 0) continue; ASSERT("Each line of config file should be like: ' : TestPattern1 TestPattern2 ...'" && words.size() == 2); suites_[words[0]] = StringTools::split(" \t", words[1]); LOG_DEBUG_LIB(LibOdc) << "TestRunner::readConfig(\"" << fileName << "\"): " << words[0] << ": " << suites_[words[0]].size() << " entries." << endl; } } void TestRunner::smslabel(const string& s) { if (!mars_sms_label_) return; string cmd = "smslabel "; cmd += label_ + " " + s; std::system(cmd.c_str()); } } // namespace test } // namespace tool } // namespace odc odc-1.6.3/src/odc/tools/TestDistinct.cc0000664000175000017500000000270415146027420020102 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// \file UnitTest.h /// /// @author Piotr Kuchta, ECMWF, September 2010 #include "eckit/filesystem/PathName.h" #include "eckit/io/FileHandle.h" #include "odc/Select.h" #include "odc/api/Odb.h" #include "TestCase.h" using namespace std; using namespace eckit; using namespace odc; /// static void test() { std::string sql = "select distinct a from \"TestDistinct_a1to10twice.odb\";"; Log::info() << "Executing: '" << sql << "'" << std::endl; odc::Select sel(sql); odc::Select::iterator it2 = sel.begin(); odc::Select::iterator end2 = sel.end(); int i = 0; for (; it2 != end2; ++it2) ASSERT((*it2)[0] == ++i); ASSERT((*it2)[0] == 10); } static void setUp() { stringstream s; s << "a:REAL" << std::endl; for (size_t i = 1; i <= 10; ++i) s << i << std::endl; for (size_t i = 1; i <= 10; ++i) s << i << std::endl; FileHandle dh("TestDistinct_a1to10twice.odb"); dh.openForWrite(0); AutoClose close(dh); odc::api::odbFromCSV(s, dh); } static void tearDown() {} SIMPLE_TEST(Distinct) odc-1.6.3/src/odc/tools/MDSetTool.h0000664000175000017500000000215315146027420017133 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef MDSetTool_H #define MDSetTool_H #include "eckit/sql/SQLTypedefs.h" #include "odc/tools/Tool.h" namespace odc { namespace tool { class MDSetTool : public Tool { public: MDSetTool(int argc, char* argv[]); void run(); static void help(std::ostream& o); static void usage(const std::string& name, std::ostream& o); private: // No copy allowed MDSetTool(const MDSetTool&); MDSetTool& operator=(const MDSetTool&); void parseUpdateList(const std::string& s, std::vector& columns, std::vector& types, std::vector& values, std::vector& bitfieldDefs); }; } // namespace tool } // namespace odc #endif odc-1.6.3/src/odc/tools/MockReader.cc0000664000175000017500000000064515146027420017477 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "MockReader.h" odc-1.6.3/src/odc/tools/TestIntegerValues.cc0000664000175000017500000000432215146027420021074 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// \file TestIntegerValues.h /// /// @author Piotr Kuchta, ECMWF, Jan 2011 #include "eckit/filesystem/PathName.h" #include "eckit/io/FileHandle.h" #include "odc/Reader.h" #include "odc/api/Odb.h" #include "TestCase.h" using namespace std; using namespace eckit; using namespace odc; static void setUp() { const char* data = "date:REAL\n" "20101130\n" "20101201\n" "20101202\n"; FileHandle dh("TestIntegerValues.odb"); dh.openForWrite(0); AutoClose close(dh); odc::api::odbFromCSV(data, dh); } static void test() { // Log::info() << fixed; ////////////////////////////// ODB from MARS ////////////////////////////// const std::string fileNameOdb = "TestIntegerValues.odb"; odc::Reader odb(fileNameOdb); odc::Reader::iterator it = odb.begin(); for (unsigned int i = 0; i < it->columns().size(); ++i) { std::cout << "Name = " << it->columns()[i]->name() << " "; } std::cout << std::endl; for (; it != odb.end(); ++it) { for (size_t i = 0; i < it->columns().size(); ++i) { // float nr = ((*it)[i]); /// <- WRONG! double nr = ((*it)[i]); switch (it->columns()[i]->type()) { case odc::api::INTEGER: case odc::api::BITFIELD: std::cout << static_cast(nr) << " "; // cout << "* should be: " << it->integer(i) << " "; break; case odc::api::REAL: std::cout << nr << " "; break; case odc::api::IGNORE: default: ASSERT("Unknown type" && false); break; } } std::cout << std::endl; } } static void tearDown() {} SIMPLE_TEST(IntegerValues) odc-1.6.3/src/odc/tools/CompactTool.cc0000664000175000017500000000337315146027420017710 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "odc/tools/CompactTool.h" #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/PathName.h" #include "eckit/log/Log.h" #include "odc/Comparator.h" #include "odc/Reader.h" #include "odc/Writer.h" using namespace std; using namespace eckit; namespace odc { namespace tool { CompactTool::CompactTool(int argc, char* argv[]) : Tool(argc, argv) {} void CompactTool::run() { if (parameters().size() != 3) { Log::error() << "Usage: "; usage(parameters(0), Log::error()); Log::error() << std::endl; std::stringstream ss; ss << "Expected exactly 3 command line parameters"; throw UserError(ss.str()); } PathName inFile = parameters(1); PathName outFile = parameters(2); odc::Reader in(inFile); odc::Writer<> out(outFile); odc::Reader::iterator it(in.begin()); odc::Reader::iterator end(in.end()); odc::Writer<>::iterator writer(out.begin()); writer->pass1(it, end); odc::Reader outReader(outFile); Log::info() << "Verifying." << std::endl; odc::Reader::iterator it1 = in.begin(); odc::Reader::iterator end1 = in.end(); odc::Reader::iterator it2 = outReader.begin(); odc::Reader::iterator end2 = outReader.end(); odc::Comparator comparator; comparator.compare(it1, end1, it2, end2, inFile, outFile); } } // namespace tool } // namespace odc odc-1.6.3/src/odc/tools/ToolRunnerApplication.h0000664000175000017500000000211515146027420021612 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// \file ToolRunnerApplication.h /// /// @author Piotr Kuchta, ECMWF, Feb 2009 #ifndef ToolRunnerApplication_H #define ToolRunnerApplication_H #include "odc/ODBApplication.h" namespace odc { namespace tool { class Tool; class ToolRunnerApplication : public ODBApplication { public: ToolRunnerApplication(int argc, char** argv, bool createCommandLineTool = true, bool deleteTool = true); ToolRunnerApplication(odc::tool::Tool& tool, int argc, char** argv); ~ToolRunnerApplication(); void tool(odc::tool::Tool*); void run(); int printHelp(std::ostream& out); private: odc::tool::Tool* tool_; bool deleteTool_; }; } // namespace tool } // namespace odc #endif odc-1.6.3/src/odc/tools/SQLTool.h0000664000175000017500000000577415146027420016632 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Piotr Kuchta /// @author Simon Smart /// @date Aug 2018 #ifndef odc_SQLTool_H #define odc_SQLTool_H #include #include "eckit/sql/SQLOutputConfig.h" #include "odc/tools/Tool.h" namespace eckit { class Offset; class Length; namespace sql { class SQLSession; class SQLParser; class SQLOutputConfig; } // namespace sql } // namespace eckit namespace odc { namespace sql { class SQLOutputConfig; } namespace tool { //---------------------------------------------------------------------------------------------------------------------- class SQLTool : public Tool { public: SQLTool(int argc, char** argv); virtual ~SQLTool(); virtual void run(); static void help(std::ostream& o) { o << "Executes SQL statement"; } static void usage(const std::string& name, std::ostream& o) { o << name << " | " << std::endl; o << " [-T] Disables printing of column names" << std::endl; o << " [-offset ] Start processing file at a given offset" << std::endl; o << " [-length ] Process only given bytes of data" << std::endl; o << " [-N] Do not write NULLs, but proper missing data values" << std::endl; o << " [-i ] ODB input file" << std::endl; o << " [-o ] ODB output file" << std::endl; o << " [-f default|wide|ascii|odb] ODB output format (odb is binary ODB, ascii and wide are ascii " "formatted with bitfield definitions in header. Default is ascii on stdout and odb to file)" << std::endl; o << " [-delimiter ] Changes the default values' delimiter (TAB by default)" << std::endl; o << " delim can be any character or string" << std::endl; o << " [--binary|--bin] Print bitfields in binary notation" << std::endl; o << " [--no_alignment] Do not align columns" << std::endl; o << " [--full_precision] Print with full precision" << std::endl; } private: std::unique_ptr sqlOutputConfig_; std::string inputFile_; // -i eckit::Offset offset_; // -offset eckit::Length length_; // -length }; //---------------------------------------------------------------------------------------------------------------------- } // namespace tool } // namespace odc #endif odc-1.6.3/src/odc/tools/MDSetTool.cc0000664000175000017500000001417215146027420017275 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "odc/tools/MDSetTool.h" #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/PathName.h" #include "eckit/log/Log.h" #include "eckit/sql/SQLTypedefs.h" #include "eckit/utils/StringTools.h" #include "eckit/utils/Tokenizer.h" #include "odc/ODBAPISettings.h" #include "odc/core/Header.h" #include "odc/core/MetaData.h" #include "odc/core/TablesReader.h" using namespace eckit; using namespace std; typedef eckit::StringTools S; using namespace odc::core; namespace odc { namespace tool { void MDSetTool::help(std::ostream& o) { o << "Creates a new file resetting types or values (constants only) of columns."; } void MDSetTool::usage(const std::string& name, std::ostream& o) { o << name << " " << endl << endl << "\t is a comma separated list of expressions of the form:" << endl << "\t : = " << endl << endl << "\t can be one of: integer, real, double, string. If ommited, the existing type of the column will not " "be changed." << endl << "\tBoth type and value are optional; at least one of the two should be present. For example:" << endl << "\t odb mdset \"expver=' 0008'\" input.odb patched.odb " << endl; } MDSetTool::MDSetTool(int argc, char* parameters[]) : Tool(argc, parameters) {} void MDSetTool::run() { if (parameters().size() != 4) { Log::error() << "Usage: "; usage(parameters(0), Log::error()); Log::error() << std::endl; std::stringstream ss; ss << "Expected exactly 4 command line parameters"; throw UserError(ss.str()); } PathName inFile = parameters(2), outFile = parameters(3); std::unique_ptr outHandle(ODBAPISettings::instance().writeToFile(outFile)); std::vector columns, types, values; std::vector bitfieldDefs; parseUpdateList(parameters(1), columns, types, values, bitfieldDefs); odc::core::TablesReader reader(inFile); for (auto it = reader.begin(), end = reader.end(); it != end; ++it) { const MetaData& md(it->columns()); for (size_t i = 0; i < columns.size(); ++i) { Column& c(*md[md.columnIndex(columns[i])]); Log::info() << "" << columns[i] << ": " << c << endl; if (types[i].size() && types[i] != "NONE") c.type(Column::type(types[i])); if (bitfieldDefs[i].first.size()) c.bitfieldDef(bitfieldDefs[i]); if (values[i].size() && values[i] != "NONE") { Codec& codec(c.coder()); if (codec.name().find("constant") == std::string::npos) { stringstream ss; ss << "Column '" << columns[i] << "' is not constant (codec: " << codec.name() << ")" << endl; throw UserError(ss.str()); } double v(StringTool::translate(values[i])); c.min(v); c.max(v); } } size_t sizeOfEncodedData = it->encodedDataSize(); eckit::Buffer encodedData(it->readEncodedData()); ASSERT(encodedData.size() == sizeOfEncodedData); // See if the file was created on a different order architecture if (it->byteOrder() == BYTE_ORDER_INDICATOR) { Log::info() << "MDSetTool::run: SAME ORDER " << sizeOfEncodedData << std::endl; auto encodedHeader = core::Header::serializeHeader(sizeOfEncodedData, md.rowsNumber(), it->properties(), md); outHandle->write(encodedHeader.first, encodedHeader.second); } else { Log::info() << "MDSetTool::run: OTHER ORDER " << sizeOfEncodedData << std::endl; auto encodedHeader = core::Header::serializeHeaderOtherByteOrder(sizeOfEncodedData, md.rowsNumber(), it->properties(), md); outHandle->write(encodedHeader.first, encodedHeader.second); } outHandle->write(encodedData.data(), sizeOfEncodedData); } } // // static std::vector split(const std::string& delim, const std::string& text); void MDSetTool::parseUpdateList(const std::string& s, std::vector& columns, std::vector& types, std::vector& values, std::vector& bitfieldDefs) { std::vector assignments(S::split(",", s)); for (size_t i = 0; i < assignments.size(); ++i) { vector assignment(S::split("=", assignments[i])); string value(assignment.size() == 2 ? assignment[1] : "NONE"); vector columnNameAndType(S::split(":", assignment[0])); string type(columnNameAndType.size() == 2 ? columnNameAndType[1] : "NONE"); string column(columnNameAndType[0]); eckit::sql::BitfieldDef bf; if (type.size() && type[0] == '[' && type[type.size() - 1] == ']') { std::vector parts(StringTools::split(";", type.substr(1, type.size() - 2))); for (size_t p = 0; p < parts.size(); ++p) { std::vector field(S::split(":", parts[p])); bf.first.push_back(field[0]); bf.second.push_back(atoi(field[1].c_str())); } } Log::info() << "MDSetTool::parseUpdateList: " << column << " : " << type << " = '" << value << "'" << std::endl; columns.push_back(column); types.push_back(type); values.push_back(value); bitfieldDefs.push_back(bf); } ASSERT(columns.size() == types.size()); ASSERT(columns.size() == values.size()); ASSERT(columns.size() == bitfieldDefs.size()); } } // namespace tool } // namespace odc odc-1.6.3/src/odc/tools/TestMetaDataReader.ksh0000664000175000017500000000160715146027420021325 0ustar alastairalastairset -e function log { if [ x$MARS_SMS_LABEL != x ]; then smslabel $MARS_SMS_LABEL "$*"; else echo ++++++++++++++++++++++++++++++++++++++++++++ echo + $* echo ++++++++++++++++++++++++++++++++++++++++++++ fi } #set -v set -x log +++ Test: odb import cat >TestMetaDataReader1.csv <<@@ x:INTEGER 1 @@ ./odb import TestMetaDataReader1.csv TestMetaDataReader1.odb cat >TestMetaDataReader2.csv <<@@ x:INTEGER 2 @@ ./odb import TestMetaDataReader2.csv TestMetaDataReader2.odb cat >TestMetaDataReader3.csv <<@@ x:INTEGER 3 @@ ./odb import TestMetaDataReader3.csv TestMetaDataReader3.odb cat TestMetaDataReader1.odb TestMetaDataReader2.odb TestMetaDataReader3.odb >TestMetaDataReader.odb ./odb sql select \* from \"TestMetaDataReader.odb\" log +++ All tests completed OK JUST_ONE_TEST(test, setUp, tearDown) void test(){} void setUp(){} void tearDown(){} JUST_ONE_TEST(test, setUp, tearDown) odc-1.6.3/src/odc/tools/TestOdaCAPI.h0000664000175000017500000000151315146027420017320 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef TEST_ODA_C_API_H #define TEST_ODA_C_API_H namespace odc { namespace tool { namespace test { int test_odacapi(int argc, char* argv[]); int test_odacapi_setup_in_C(int argc, char* argv[]); int test_odacapi_setup(int argc, char* argv[]); int test_odacapi1(int argc, char* argv[]); int test_odacapi2(int argc, char* argv[]); int test_odacapi3(int argc, char* argv[]); } // namespace test } // namespace tool } // namespace odc #endif odc-1.6.3/src/odc/tools/CountTool.cc0000664000175000017500000000317115146027420017406 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/eckit.h" #include "eckit/exception/Exceptions.h" #include "eckit/log/Log.h" #include "odc/LibOdc.h" #include "odc/Reader.h" #include "odc/core/MetaData.h" #include "odc/core/TablesReader.h" #include "odc/tools/CountTool.h" using namespace eckit; namespace odc { namespace tool { CountTool::CountTool(int argc, char* argv[]) : Tool(argc, argv) {} size_t CountTool::rowCount(const PathName& db) { odc::core::TablesReader reader(db); auto it = reader.begin(); auto end = reader.end(); size_t n = 0; for (; it != end; ++it) { n += it->rowCount(); } return n; } void CountTool::run() { if (parameters().size() < 2) { Log::error() << "Usage: "; usage(parameters(0), Log::error()); Log::error() << std::endl; std::stringstream ss; ss << "Expected at least 2 command line parameters"; throw UserError(ss.str()); } unsigned long long n(0); for (size_t i(1); i < parameters().size(); ++i) { const std::string fileName(parameters(i)); LOG_DEBUG_LIB(LibOdc) << "CountTool: counting " << fileName << std::endl; n += rowCount(fileName); } std::cout << n << std::endl; } } // namespace tool } // namespace odc odc-1.6.3/src/odc/tools/TestAtTableInTheOutput.cc0000664000175000017500000000505615146027420022011 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// \file TestAtTableInTheOutput.h /// /// @author Piotr Kuchta, ECMWF, Feb 2009 #include "odc/Comparator.h" #include "odc/Reader.h" #include "odc/Select.h" #include "odc/core/MetaData.h" #include "TestCase.h" #include "odc/Writer.h" using namespace std; using namespace eckit; using namespace odc; static void setUp() { odc::Writer<> f("TestAtTableInTheOutput_A.odb"); odc::Writer<>::iterator it = f.begin(); it->setNumberOfColumns(4); it->setColumn(0, "lat@hdr", odc::api::REAL); it->setColumn(1, "lon@hdr", odc::api::REAL); it->setColumn(2, "obsvalue", odc::api::REAL); eckit::sql::BitfieldDef bfDef; bfDef.first.push_back("x"); bfDef.second.push_back(1); bfDef.first.push_back("y"); bfDef.second.push_back(2); it->setBitfieldColumn(3, "bf", odc::api::BITFIELD, bfDef); it->writeHeader(); for (size_t i = 1; i <= 2; i++) { (*it)[0] = i; // col 0 (*it)[1] = i; // col 1 (*it)[2] = i; // col 2 ++it; } } static void selectIntoSecondFile() { const string fileName = "TestAtTableInTheOutput_A.odb"; string sql = "select lat,lon,obsvalue,bf into \"TestAtTableInTheOutput_B.odb\""; sql += " from \"" + fileName + "\" ;"; odc::Select f(sql); //, fileName); odc::Select::iterator it = f.begin(); // string c0 = it.columns()[0]->name(); // string c1 = it.columns()[1]->name(); // string c2 = it.columns()[2]->name(); ++it; // this is needed to push the second row to the INTO file // Log::info() << "c0=" << c0 << ", c1=" << c1 << ", c2=" << c2 << std::endl; /// ASSERT(""); } static void compareFiles() { odc::Reader oda1("TestAtTableInTheOutput_A.odb"); odc::Reader oda2("TestAtTableInTheOutput_B.odb"); odc::Reader::iterator it1(oda1.begin()); odc::Reader::iterator end1(oda1.end()); odc::Reader::iterator it2(oda2.begin()); odc::Reader::iterator end2(oda2.end()); odc::Comparator().compare(it1, end1, it2, end2, "TestAtTableInTheOutput_A.odb", "TestAtTableInTheOutput_B.odb"); } static void test() { selectIntoSecondFile(); compareFiles(); } static void tearDown() {} SIMPLE_TEST(AtTableInTheOutput) odc-1.6.3/src/odc/tools/ODAHeaderTool.cc0000664000175000017500000001332615146027420020035 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/exception/Exceptions.h" #include "ODAHeaderTool.h" #include "odc/core/TablesReader.h" using namespace eckit; using namespace odc::core; namespace odc { namespace tool { class MDPrinter { public: virtual void print(std::ostream&, const core::Table&) = 0; virtual void printSummary(std::ostream&) {} }; class VerbosePrinter : public MDPrinter { public: VerbosePrinter() : headerCount_() {} void print(std::ostream& o, const core::Table& tbl) { o << std::endl << "Header " << ++headerCount_ << ". " << "Begin offset: " << tbl.startPosition() << ", end offset: " << tbl.nextPosition() << ", number of rows in block: " << tbl.rowCount() << ", byteOrder: " << ((tbl.byteOrder() == 1) ? "same" : "other") << std::endl << tbl.columns(); } private: unsigned long headerCount_; }; class OffsetsPrinter : public MDPrinter { public: OffsetsPrinter() {} void print(std::ostream& o, const core::Table& tbl) { Offset offset(tbl.startPosition()); Length length(tbl.nextPosition() - tbl.startPosition()); o << offset << " " << length << " " << tbl.rowCount() << " " << tbl.columnCount() << std::endl; } }; class DDLPrinter : public MDPrinter { public: DDLPrinter(const std::string& path, const std::string& tableName) : path_(path), tableName_(tableName) {} void print(std::ostream& o, const core::Table& tbl) { if (md_.empty() || md_.back() != tbl.columns()) { md_.push_back(tbl.columns()); return; } } void printSummary(std::ostream& o) { for (size_t i(0); i < md_.size(); ++i) printTable(o, md_[i], tableName_, path_); } static std::string typeName(const Column& c) { using namespace api; switch (c.type()) { case STRING: return "STRING"; case INTEGER: return "INTEGER"; case BITFIELD: return "INTEGER"; case REAL: return "REAL"; case DOUBLE: return "DOUBLE"; default: throw new Exception("unknown type"); } } static std::pair typeDefinitionAndName(const std::string& tableName, const Column& c) { std::stringstream definition; std::string type_name(typeName(c)); if (c.type() == api::BITFIELD) { const eckit::sql::BitfieldDef& bd(c.bitfieldDef()); const std::vector& fieldNames(bd.first); const std::vector& sizes(bd.second); type_name = stripAtTable(tableName, c.name()) + "_at_" + tableName + "_t"; definition << "CREATE TYPE " << type_name << " AS ("; for (size_t i(0); i < sizes.size(); ++i) definition << fieldNames[i] << " bit" << sizes[i] << ((i + 1 < sizes.size()) ? ", " : ""); definition << ");\n"; } return make_pair(definition.str(), type_name); } static std::string stripAtTable(const std::string& tableName, const std::string& columnName) { std::string suffix(std::string("@") + tableName); if (columnName.size() >= suffix.size() && columnName.compare(columnName.size() - suffix.size(), suffix.size(), suffix) == 0) return columnName.substr(0, columnName.size() - suffix.size()); return columnName; } static void printTable(std::ostream& s, const MetaData& md, const std::string& tableName, const std::string& path) { std::stringstream create_type, create_table; create_table << "CREATE TABLE " << tableName << " AS (\n"; for (size_t i(0); i < md.size(); ++i) { std::pair p(typeDefinitionAndName(tableName, *md[i])); const std::string &def(p.first), type(p.second); create_type << def; create_table << " " << stripAtTable(tableName, md[i]->name()) << " " << type << ",\n"; } create_table << ") ON '" << path << "';\n"; s << create_type.str(); s << create_table.str(); } private: std::vector md_; const std::string path_; const std::string tableName_; }; HeaderTool::HeaderTool(int argc, char* argv[]) : Tool(argc, argv) {} void HeaderTool::run() { registerOptionWithArgument("-table"); if (parameters().size() != 2) { Log::error() << "Usage: "; usage(parameters(0), Log::error()); std::stringstream ss; ss << "Expected exactly 2 command line parameters"; throw UserError(ss.str()); } const std::string db(parameters(1)); std::ostream& o(std::cout); VerbosePrinter verbosePrinter; OffsetsPrinter offsetsPrinter; DDLPrinter ddlPrinter(db, optionArgument("-table", std::string("foo"))); MDPrinter& printer(*(optionIsSet("-offsets") ? static_cast(&offsetsPrinter) : optionIsSet("-ddl") ? static_cast(&ddlPrinter) : static_cast(&verbosePrinter))); core::TablesReader reader(db); auto it = reader.begin(); auto end = reader.end(); for (; it != end; ++it) { printer.print(o, *it); } printer.printSummary(o); } } // namespace tool } // namespace odc odc-1.6.3/src/odc/tools/xxx0000664000175000017500000000030515146027420015717 0ustar alastairalastairFILES=$(grep -l SIMPLE_TEST *cc) for f in $FILES; do test_name=$(echo $f|sed s/.cc//g|sed s/^Test//g) echo $f $test_name perl -p -i -e s/SIMPLE_TEST/SIMPLE_TEST\($test_name\)/g $f done odc-1.6.3/src/odc/tools/MergeTool.h0000664000175000017500000000251715146027420017222 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef MergeTool_H #define MergeTool_H #include "eckit/filesystem/PathName.h" #include "odc/tools/Tool.h" namespace odc { namespace tool { class MergeTool : public Tool { public: MergeTool(int argc, char* argv[]); void run(); static void help(std::ostream& o); static void usage(const std::string& name, std::ostream& o); static void merge(const std::vector& inputFiles, const eckit::PathName& outputFileName); static void merge(const std::vector& inputFiles, const std::vector& sqls, const eckit::PathName& outputFileName); private: // No copy allowed MergeTool(const MergeTool&); MergeTool& operator=(const MergeTool&); static char* dummyArgv_[]; std::vector inputFiles_; std::vector sql_; eckit::PathName outputFile_; bool sqlFiltering_; }; } // namespace tool } // namespace odc #endif odc-1.6.3/src/odc/tools/FixedSizeRowTool.cc0000664000175000017500000000337015146027420020701 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "odc/tools/FixedSizeRowTool.h" #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/PathName.h" #include "eckit/log/Log.h" #include "odc/Comparator.h" #include "odc/Reader.h" #include "odc/Writer.h" using namespace eckit; namespace odc { namespace tool { FixedSizeRowTool::FixedSizeRowTool(int argc, char* argv[]) : Tool(argc, argv) {} void FixedSizeRowTool::run() { if (parameters().size() != 3) { Log::error() << "Usage: "; usage(parameters(0), Log::error()); Log::error() << std::endl; std::stringstream ss; ss << "Expected exactly 3 command line parameters"; throw UserError(ss.str()); } PathName inFile = parameters(1); PathName outFile = parameters(2); odc::Reader in(inFile); odc::Writer<> out(outFile); odc::Reader::iterator it = in.begin(); odc::Reader::iterator end = in.end(); odc::Writer<>::iterator outIt(out.begin()); outIt->pass1(it, end); odc::Reader outReader(outFile); Log::info() << "Verifying." << std::endl; odc::Reader::iterator it1 = in.begin(); odc::Reader::iterator end1 = in.end(); odc::Reader::iterator it2 = outReader.begin(); odc::Reader::iterator end2 = outReader.end(); odc::Comparator comparator; comparator.compare(it1, end1, it2, end2, inFile, outFile); } } // namespace tool } // namespace odc odc-1.6.3/src/odc/tools/TestMetaData.cc0000664000175000017500000000310415146027420017774 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// \file UnitTest.h /// /// @author Piotr Kuchta, ECMWF, Feb 2009 #include "odc/core/MetaData.h" #include "TestCase.h" using namespace std; using namespace eckit; using namespace odc; using namespace odc::core; static void test() { MetaData md1(0); md1.addColumn /**/ ("A", "REAL"); //, false, 0.); md1.addColumn /**/ ("B", "INTEGER"); //, false, 0.); Log::info() << "md1: " << std::endl << md1 << std::endl; MetaData md2(0); md2.addColumn /* */ ("C", "STRING"); //, false, 0.); Log::info() << "md2:" << std::endl << md2 << std::endl; MetaData sum = md1 + md2; Log::info() << "md1 + md2: " << std::endl << sum << std::endl; ASSERT(sum.size() == md1.size() + md2.size()); ASSERT(sum == md1 + md2); MetaData sum2 = md1; sum2 += md2; ASSERT(sum2.size() == md1.size() + md2.size()); ASSERT(sum == sum2); ASSERT(columnNameMatches("column@body", "column@body")); ASSERT(columnNameMatches("column@body", "column")); ASSERT(!columnNameMatches("columns@body", "column")); ASSERT(!columnNameMatches("another_column@body", "column")); } static void setUp() {} static void tearDown() {} SIMPLE_TEST(MetaData) odc-1.6.3/src/odc/tools/TestRunnerApplication.cfg0000664000175000017500000000100315146027420022117 0ustar alastairalastairdisp: TestODADispatchingWriter sql: TestAggregateFunctions TestSelectIterator TestSelectStarAt TestStar TestBitfields TestODASelectDataHandle TestDistinct TestOrderBy basic: Test.*Codec.* TestCommandLineParsing TestMinMax TestOdaCAPI .*selectAggregatedAndNonAggregated.* Test_vector_syntax Test_bitfieldsLength Test_stringInWhere Test_vector_syntax2 Test_filterInPlace Test_filterInPlace2 Test_rownumber1 codecs: Test.*Codec.* functions: Test.*Function.* include: Test_include Test_include odc-1.6.3/src/odc/tools/CountTool.h0000664000175000017500000000175215146027420017253 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef odc_CountTool_H #define odc_CountTool_H #include "odc/tools/Tool.h" namespace odc { namespace tool { class CountTool : public Tool { public: CountTool(int argc, char* argv[]); static size_t rowCount(const eckit::PathName&); void run(); static void help(std::ostream& o) { o << "Counts number of rows in files"; } static void usage(const std::string& name, std::ostream& o) { o << name << " "; } private: // No copy allowed CountTool(const CountTool&); CountTool& operator=(const CountTool&); }; } // namespace tool } // namespace odc #endif odc-1.6.3/src/odc/tools/TestSelectStarAt.cc0000664000175000017500000000303315146027420020653 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// \file UnitTest.h /// /// @author Piotr Kuchta, ECMWF, Feb 2009 #include "odc/Select.h" #include "odc/core/MetaData.h" #include "TestCase.h" using namespace std; using namespace eckit; using namespace odc; /// UnitTest expansion of '*@hdr' into a list of columns of the hdr ODB table. /// static void test() { // TODO: make sure a 'select ... into ... from ...', e.g.: // // const string SELECT = "select *@hdr into \"out.odb\" from \"2000010106-reduced.odb\";"; // is not returning a result set (iterator). Or perhaps it is returning an empty result set. const string SELECT = "select *@hdr from \"2000010106-reduced.odb\";"; odc::Select oda(SELECT); Log::info() << "Executing: '" << SELECT << "'" << std::endl; odc::Select::iterator it = oda.begin(); Log::info() << "it->columns().size() => " << it->columns().size() << std::endl; ASSERT(it->columns().size() == 27); #if 0 unsigned long long i = 0; for ( ; it != oda.end(); ++it) ++i; Log::info() << "i == " << i << std::endl; ASSERT(i == 3321753); #endif } static void setUp() {} static void tearDown() {} SIMPLE_TEST(SelectStarAt) odc-1.6.3/src/odc/tools/ToolFactory.cc0000664000175000017500000001071615146027420017730 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// \file ToolFactory.cc /// /// @author Piotr Kuchta, ECMWF, Feb 2009 /// // #include // #include // #include // #include // #include "eckit/utils/Regex.h" #include "eckit/exception/Exceptions.h" #include "TestCase.h" #include "Tool.h" #include "ToolFactory.h" using namespace std; using namespace eckit; namespace odc { namespace tool { std::map* AbstractToolFactory::toolFactories = 0; class MatchAll : public std::vector { public: MatchAll() { push_back(".*"); } }; const std::vector AbstractToolFactory::matchAll = MatchAll(); std::vector* AbstractToolFactory::testCases(const std::vector& patterns) { ASSERT(toolFactories != 0); std::vector* v = new std::vector(); for (std::map::iterator it = toolFactories->begin(); it != toolFactories->end(); it++) { std::string testName = it->first; if (testName.find("Test") == std::string::npos || !Tool::matchAny(patterns, testName)) continue; AbstractToolFactory* factory = it->second; char* argv[] = {const_cast("test"), 0}; Tool* tool = factory->create(1, argv); ASSERT(tool); test::TestCase* testCase = dynamic_cast(tool); if (testCase != 0) { testCase->name(testName); v->push_back(testCase); } else { Log::warning() << "AbstractToolFactory::testCases: " << testName << " is not a TestCase. Skipping" << std::endl; delete tool; } } return v; } AbstractToolFactory& AbstractToolFactory::findTool(const std::string& name) { ASSERT(toolFactories); std::map::const_iterator it = toolFactories->find(name); ASSERT("Unknown tool" && it != toolFactories->end()); return *it->second; } void AbstractToolFactory::printToolHelp(const std::string& name, std::ostream& s) { findTool(name).help(s); s << std::endl; } void AbstractToolFactory::printToolUsage(const std::string& name, std::ostream& s) { findTool(name).usage(name, s); s << std::endl; } void AbstractToolFactory::printToolsHelp(std::ostream& s) { ASSERT(toolFactories); for (std::map::iterator it = toolFactories->begin(); it != toolFactories->end(); it++) { std::string toolName = it->first; AbstractToolFactory* toolFactory = it->second; if (toolName.find("Test") == std::string::npos && !toolFactory->experimental()) { s << toolName << ":\t"; toolFactory->help(s); s << std::endl << "Usage:" << std::endl << "\t"; toolFactory->usage(toolName, s); s << std::endl << std::endl; } } } void AbstractToolFactory::listTools(std::ostream& s) { ASSERT(toolFactories); for (std::map::iterator it = toolFactories->begin(); it != toolFactories->end(); it++) { std::string toolName = it->first; AbstractToolFactory* toolFactory = it->second; if (toolName.find("Test") == std::string::npos && !toolFactory->experimental()) { s << " " << toolName << " "; toolFactory->help(s); s << std::endl; } } } Tool* AbstractToolFactory::createTool(const std::string& name, int argc, char** argv) { AbstractToolFactory* factory = (*toolFactories)[name]; if (factory == 0) return 0; return factory->create(argc, argv); }; AbstractToolFactory::AbstractToolFactory(const std::string& name) { if (toolFactories == 0) toolFactories = new std::map(); (*toolFactories)[name] = this; } AbstractToolFactory::~AbstractToolFactory() { if (toolFactories) { delete toolFactories; toolFactories = 0; } } } // namespace tool } // namespace odc odc-1.6.3/src/odc/tools/ODA2RequestTool.cc0000664000175000017500000001540615146027420020360 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/config/Resource.h" #include "eckit/filesystem/PathName.h" #include "eckit/log/Log.h" #include "eckit/utils/StringTools.h" #include "eckit/utils/Tokenizer.h" #include "odc/FastODA2Request.h" #include "odc/GribCodes.h" #include "odc/LibOdc.h" #include "odc/Select.h" #include "odc/tools/ODA2RequestTool.h" using namespace std; using namespace eckit; namespace odc { namespace tool { char* static_argv[] = {const_cast("oda2request")}; ODA2RequestTool::ODA2RequestTool(int argc, char** argv) : Tool(argc, argv) { registerOptionWithArgument("-c"); readConfig(); } ODA2RequestTool::ODA2RequestTool() : Tool(1, static_argv) { registerOptionWithArgument("-c"); readConfig(); } ODA2RequestTool::~ODA2RequestTool() {} void ODA2RequestTool::help(std::ostream& o) { o << "Creates MARS ARCHIVE request for a given file"; } void ODA2RequestTool::usage(const std::string& name, std::ostream& o) { o << name << " [-c configFile] [-q] []"; } void ODA2RequestTool::run() { eckit::PathName inputFile; string outputFile; switch (parameters().size()) { case 3: outputFile = parameters(2); case 2: inputFile = parameters(1); break; default: Log::error() << "Usage: "; usage(parameters(0), Log::error()); Log::error() << std::endl; return; // 1; break; } readConfig(); string request = generateMarsRequest(inputFile, optionIsSet("-q")); if (outputFile.size() == 0) std::cout << request << std::endl; else { ofstream out(outputFile.c_str()); out << request << std::endl; out.close(); } return; } PathName ODA2RequestTool::config() { return optionArgument("-c", std::string("~odc/codes/ODA2RequestTool.cfg")); } void ODA2RequestTool::readConfig() { readConfig(config()); } void ODA2RequestTool::readConfig(const PathName& fileName) { LOG_DEBUG_LIB(LibOdc) << "ODA2RequestTool::readConfig: reading file '" << fileName << "'" << std::endl; columnName2requestKey_.clear(); string s = readFile(fileName); LOG_DEBUG_LIB(LibOdc) << "ODA2RequestTool::readConfig: parsing '" << s << "'" << std::endl; parseConfig(s); } void ODA2RequestTool::parseConfig(const std::string& s) { LOG_DEBUG_LIB(LibOdc) << "ODA2RequestTool::parseConfig: '" << s << "'" << std::endl; vector lines; Tokenizer("\n")(s, lines); Tokenizer tokenizer(": \t"); for (size_t i = 0; i < lines.size(); ++i) { vector words; tokenizer(lines[i], words); if (words.size() == 0) continue; ASSERT("Each line of config file should be like: 'MARS_KEYWORD : oda_column_name'" && words.size() == 2); columnName2requestKey_[words[1]] = words[0]; } } inline string int_as_double2string(double v) { stringstream s; s.precision(0); s << fixed << v; return s.str(); } string ODA2RequestTool::gatherStatsFast(const PathName& inputFile) { FastODA2Request o; o.parseConfig(readFile(config())); o.scanFile(inputFile); return o.genRequest(); } void ODA2RequestTool::gatherStats(const PathName& inputFile) { size_t n = columnName2requestKey_.size(); values_ = vector(n); string columnList; for (std::map::iterator it = columnName2requestKey_.begin(); it != columnName2requestKey_.end(); ++it) { if (it != columnName2requestKey_.begin()) columnList += ", "; columnList += it->first; } const string select = std::string("select ") + columnList + " from \"" + inputFile + "\";"; Log::info() << "Executing '" << select << "'" << std::endl; Translator double2string; odc::Select oda(select, inputFile); odc::Select::iterator end = oda.end(); for (odc::Select::iterator row = oda.begin(); row != end; ++row) for (size_t i = 0; i < n; ++i) { odc::api::ColumnType type = row->columns()[i]->type(); Value v = type == odc::api::STRING ? (*row).string(i) : type == odc::api::INTEGER ? int_as_double2string((*row)[i]) : double2string((*row)[i]); values_[i].insert(v); } } string ODA2RequestTool::generateMarsRequest(const PathName& inputFile, bool fast) { stringstream request; if (fast) request << gatherStatsFast(inputFile); else { gatherStats(inputFile); size_t i = 0; std::map::iterator end = columnName2requestKey_.end(); for (std::map::iterator it = columnName2requestKey_.begin(); it != end; ++it) { if (request.str().size()) request << ",\n"; const std::string& key = it->second; const string k = StringTools::upper(key); string valuesList; Values& vs = values_[i++]; for (Values::iterator vi = vs.begin(); vi != vs.end(); ++vi) { string v = *vi; LOG_DEBUG_LIB(LibOdc) << "ODA2RequestTool::genRequest: v = '" << v << "', key = " << key << std::endl; if (k == "TIME") v = StringTool::patchTimeForMars(v); else if (k == "CLASS" || k == "TYPE" || k == "STREAM") { LOG_DEBUG_LIB(LibOdc) << "ODA2RequestTool::genRequest: checking if '" << v << "' is numeric" << std::endl; if (StringTool::check(v, isdigit)) { v = StringTools::trim(v); LOG_DEBUG_LIB(LibOdc) << "ODA2RequestTool::genRequest: replacing " << v << " with "; v = GribCodes::alphanumeric(StringTools::lower(key), v); LOG_DEBUG_LIB(LibOdc) << v << std::endl; } v = StringTools::upper(v); } if (vi != vs.begin()) valuesList += "/"; valuesList += v; } request << key << " = " << valuesList; } } stringstream str; str << "ODB," << std::endl; str << request.str(); return str.str(); } } // namespace tool } // namespace odc odc-1.6.3/src/odc/tools/Tool.cc0000664000175000017500000000374115146027420016400 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// \file Tool.h /// /// @author Piotr Kuchta, ECMWF, Feb 2009 #include "odc/tools/Tool.h" #include #include "odc/tools/CompactTool.h" #include "odc/tools/CompareTool.h" #include "odc/tools/CountTool.h" #include "odc/tools/FixedSizeRowTool.h" #include "odc/tools/ImportTool.h" #include "odc/tools/IndexTool.h" #include "odc/tools/LSTool.h" #include "odc/tools/MDSetTool.h" #include "odc/tools/MergeTool.h" #include "odc/tools/ODAHeaderTool.h" #include "odc/tools/SQLTool.h" #include "odc/tools/SetTool.h" #include "odc/tools/SplitTool.h" #include "odc/tools/ToolFactory.h" #include "odc/tools/XYVTool.h" namespace odc { namespace tool { Tool::~Tool() {} Tool::Tool(int argc, char** argv) : CommandLineParser(argc, argv) {} Tool::Tool(const CommandLineParser& clp) : CommandLineParser(clp) {} void Tool::registerTools() { static ToolFactory compact("compact"); static ToolFactory compare("compare"); static ToolFactory countTool("count"); static ToolFactory indexTool("index"); static ToolFactory fixedSizeRow("fixrowsize"); static ToolFactory import("import"); static ToolFactory lsTool("ls"); static ToolFactory mdset("mdset"); static ToolFactory mergeTool("merge"); static ToolFactory odaHeader("header"); static ToolFactory sqlTool("sql"); static ToolFactory set("set"); static ToolFactory split("split"); static ToolFactory xyvTool("xyv"); } } // namespace tool } // namespace odc odc-1.6.3/src/odc/tools/TestFunctionTdiff.cc0000664000175000017500000000260215146027420021060 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// \file UnitTest.cc /// /// @author ECMWF, July 2010 // const double EPS = 7e-6 #include "eckit/log/Timer.h" #include "odc/Select.h" #include "TestCase.h" #include "odc/Writer.h" using namespace std; using namespace eckit; using namespace odc; static void setUp() { Timer t("Test tdiff function"); odc::Writer<> oda("test_tdiff.odb"); odc::Writer<>::iterator row = oda.begin(); row->setNumberOfColumns(2); row->setColumn(0, "date", odc::api::INTEGER); row->setColumn(1, "time", odc::api::INTEGER); row->writeHeader(); (*row)[0] = 20090706.0; (*row)[1] = 210109.0; ++row; } static void tearDown() { PathName("test_tdiff.odb").unlink(); } static void test() { const string sql = "select tdiff(date,time,20090707.0,0.0) from \"test_tdiff.odb\";"; Log::info() << "Executing: '" << sql << "'" << std::endl; odc::Select oda(sql); odc::Select::iterator it = oda.begin(); ASSERT((*it)[0] == -10731); // } SIMPLE_TEST(FunctionTdiff) odc-1.6.3/src/odc/tools/TestRunner.h0000664000175000017500000000251215146027420017431 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// \file TestRunner.h /// /// @author Piotr Kuchta, ECMWF, Feb 2009 #ifndef TestRunner_H #define TestRunner_H #include "eckit/filesystem/PathName.h" #include "odc/CommandLineParser.h" namespace odc { namespace tool { namespace test { class TestRunner { public: TestRunner(CommandLineParser&); virtual ~TestRunner(); size_t numberOfFailed() { return failed_.size(); } void run(); private: typedef std::pair FailedTest; typedef std::map > Suites; void readConfig(const eckit::PathName fileName); void runTests(const TestCases& tests); void smslabel(const std::string&); CommandLineParser clp_; Suites suites_; std::vector failed_; std::stringstream runningTimes_; std::stringstream xml_; bool mars_sms_label_; std::string label_; }; } // namespace test } // namespace tool } // namespace odc #endif odc-1.6.3/src/odc/tools/ODA2RequestTool.h0000664000175000017500000000263115146027420020216 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef ODA2RequestTool_H #define ODA2RequestTool_H #include "odc/tools/Tool.h" namespace odc { namespace tool { class ODA2RequestTool : public Tool { typedef std::string Value; typedef std::set Values; public: ODA2RequestTool(); ODA2RequestTool(int argc, char** argv); ~ODA2RequestTool(); static void help(std::ostream& o); static void usage(const std::string& name, std::ostream& o); virtual void run(); void readConfig(); void readConfig(const eckit::PathName&); void parseConfig(const std::string&); std::string generateMarsRequest(const eckit::PathName& inputFile, bool fast = false); protected: std::vector& values() { return values_; } void gatherStats(const eckit::PathName& inputFile); std::string gatherStatsFast(const eckit::PathName& inputFile); eckit::PathName config(); private: std::map columnName2requestKey_; std::vector values_; }; } // namespace tool } // namespace odc #endif odc-1.6.3/src/odc/tools/CMakeLists.txt0000664000175000017500000000416515146027420017715 0ustar alastairalastairlist( APPEND odctools_src_files TestCase.h TestCase.cc TestRunner.cc TestRunner.h TestRunnerApplication.cc TestRunnerApplication.h TestRunnerApplication.cfg CompactTool.cc CompactTool.h CompareTool.cc CompareTool.h CountTool.cc CountTool.h IndexTool.cc IndexTool.h FixedSizeRowTool.cc FixedSizeRowTool.h ImportTool.cc ImportTool.h LSTool.cc LSTool.h MDSetTool.cc MDSetTool.h MergeTool.cc MergeTool.h #ODA2RequestTool.cc #ODA2RequestTool.h ODAHeaderTool.cc ODAHeaderTool.h SQLTool.cc SQLTool.h SetTool.cc SetTool.h SplitTool.cc SplitTool.h Tool.cc Tool.h ToolFactory.cc ToolFactory.h ToolRunnerApplication.cc ToolRunnerApplication.h XYVTool.cc XYVTool.h ) list( APPEND odctest_src_files Examples.cc CAPIExamples.cc UnitTests.cc TestAggregateFunctions.cc TestAggregateFunctions2.cc TestAggregateFunctions3.cc TestAtTableInTheOutput.cc TestBitfields.cc TestCommandLineParsing.cc TestDispatchingWriter.cc TestDistinct.cc TestFunctionDateAndTime.cc TestFunctionTdiff.cc TestFunctionThin.cc TestFunctionTypeConversion.cc TestFunctionsForAngleConversion.cc TestFunctionsForTemperatureConversion.cc TestInt16_MissingCodec.cc TestIntegerValues.cc TestMetaData.cc TestMetaDataReader.ksh TestMissingValue.cc TestOdaCAPI.cc TestOdaCAPI.h TestOrderBy.cc TestSelectDataHandle.cc TestSelectStarAt.cc TestSelectTwoFiles.cc TestSetvbuffer.cc TestStar.cc MockReader.h MockReader.cc ) ecbuild_add_library( TARGET odctools INSTALL_HEADERS LISTED HEADER_DESTINATION ${INSTALL_INCLUDE_DIR}/odc/tools SOURCES ${odctools_src_files} TEMPLATES ${odctools_templates} PUBLIC_LIBS odccore ) ecbuild_add_library( TARGET odctest #INSTALL_HEADERS LISTED #HEADER_DESTINATION ${INSTALL_INCLUDE_DIR}/odc/tools SOURCES ${odctest_src_files} TEMPLATES ${odctest_templates} PUBLIC_LIBS odccore odctools ) ## TODO: Re-enable odctest ecbuild_add_executable( TARGET odc SOURCES odc.cc LIBS odccore odctools odctest ) odc-1.6.3/src/odc/tools/test.ksh0000664000175000017500000000573615146027420016650 0ustar alastairalastairset -e function log { if [ x$MARS_SMS_LABEL != x ]; then smslabel $MARS_SMS_LABEL "$*"; else echo ++++++++++++++++++++++++++++++++++++++++++++ echo + $* echo ++++++++++++++++++++++++++++++++++++++++++++ fi } #set -v set -x log +++ Test: odb set export INPUT=disp.4.0.odb export OUTPUT=disp.4.0.modified.odb log Check the input data is there and has the expected values ls -l $INPUT ./odb header $INPUT | grep expver | grep 0018 ./odb header $INPUT | grep andate | grep 20000101 ./odb sql select expver,andate -i $INPUT | grep 0018 | grep 20000101 >/dev/null log Create new file with changed values of expver and andate ./odb set "expver@desc='0019 ',andate=20100101" $INPUT $OUTPUT log Check new files have the correct values ls -l $OUTPUT ./odb ls -s expver,andate $OUTPUT | head -n 1 ./odb sql select expver from \"$OUTPUT\" |uniq #|grep 0019 | head -n 1 ./odb sql select andate from \"$OUTPUT\" |uniq #|grep 20100101 | head -n 1 ./odb header $OUTPUT | grep expver | grep 0019 ./odb header $OUTPUT | grep andate | grep 20100101 log +++ odb set Completed OK log +++ Test: odb import cat >test_odb_text_input.csv <<@@ stream:STRING,expver:INTEGER,value:REAL,report_status@hdr:BITFIELD[active:1;passive:1;rejected:1;blacklisted:1] '001',1,0.1,0000 '001',2,0.2,0001 '001',3,0.3,5 '001',4,0.4,15 @@ ./odb import test_odb_text_input.csv test_import.odb ./odb sql select \* from \"test_import.odb\" >test_odb_text_input.out #diff test_odb_text_input.csv test_odb_text_input.out #grep "VARCHAR(30)\|INT" ../obsdb/obsdb.ddl|xargs|sed 's/ VARCHAR(30)/:STRING/g'|sed 's/ INT/:INTEGER/g'|sed 's/ //g'|sed 's/,/\&/g' >rtt.csv #python ../obsdb/convertRTT.py >>rtt.csv #./odb import -d \& rtt.csv rtt.odb #./odb sql select \* from \"rtt.csv\" log +++ odb import Completed OK log +++ Test: SQL Functions #./odb test functions log +++ Completed. SQL Functions OK log +++ Test: Input file command line option: -i. Also, test printing of column names by default ./odb sql select \* -i test_import.odb |grep stream |grep expver|grep value log +++ odb Completed OK log +++ Test: Input file command line option: -i. Also, test option -T: do not print column names [ `./odb sql select \* -i test_import.odb -T|grep stream|wc|awk '{print $1}'` == 0 ] log +++ odb Completed OK ./odb sql select time,lat,lon,obsvalue order by time,lat,lon -i 2000010106.1.0.odb -f odb -o order_by_out.odb ./odb header order_by_out.odb log +++ Test SELECT with mixed aggregated and non-aggregated functions cat >test_group_by.csv <<@@ stream:STRING,expver:INTEGER,value:REAL '001',1,0.1 '001',2,0.2 NULL,4,0.4 NULL,5,0.5 @@ ./odb import test_group_by.csv test_group_by.odb ./odb sql select stream,min\(expver\) from \"test_group_by.odb\" log +++ Test portability of the split tool rm -rf split.*.*.odb ./odb split $ODB_API_TEST_DATA_PATH/split_crash_on_andate_and_antime.odb split.{andate}.{antime}.odb ls -l split.*.*.odb log +++ odb Completed OK #log +++ Test import tool #og +++ odb Completed OK log +++ All tests completed OK odc-1.6.3/src/odc/tools/XYVTool.cc0000664000175000017500000000355215146027420017007 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "odc/tools/XYVTool.h" #include #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/PathName.h" #include "eckit/log/Log.h" #include "odc/Select.h" using namespace std; using namespace eckit; namespace odc { namespace tool { XYVTool::XYVTool(int argc, char** argv) : Tool(argc, argv) {} XYVTool::~XYVTool() {} void XYVTool::run() { if (parameters().size() != 4) { Log::error() << "Usage: "; usage(parameters(0), Log::error()); Log::error() << std::endl; std::stringstream ss; ss << "Expected exactly 4 command line parameters"; throw UserError(ss.str()); } PathName inputFile = parameters(1); std::string valueColumn = parameters(2); PathName outputFile = parameters(3); std::ofstream out; out.open(outputFile.asString().c_str()); out << "#GEO" << std::endl << std::endl; out << "#FORMAT XYV" << std::endl << std::endl; out << "PARAMETER = 12004" << std::endl << std::endl; out << "x/long y/lat value" << std::endl; out << "#DATA" << std::endl << std::endl; std::string select = std::string("select lat, lon, ") + valueColumn + " from \"" + inputFile + "\";"; Log::info() << select << std::endl; odc::Select oda(select); for (odc::Select::iterator it = oda.begin(); it != oda.end(); ++it) { out << (*it)[0] << "\t" << (*it)[1] << "\t" << (*it)[2] << std::endl; } out.close(); } } // namespace tool } // namespace odc odc-1.6.3/src/odc/tools/TestRunnerApplication.h0000664000175000017500000000154515146027420021622 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// \file TestRunnerApplication.h /// /// @author Piotr Kuchta, ECMWF, Feb 2009 #ifndef TestRunnerApplication_H #define TestRunnerApplication_H #include "odc/ODBApplication.h" namespace odc { namespace tool { namespace test { class TestRunnerApplication : public ODBApplication { public: TestRunnerApplication(int argc, char** argv); virtual ~TestRunnerApplication(); void run(); }; } // namespace test } // namespace tool } // namespace odc #endif odc-1.6.3/src/odc/tools/LSTool.h0000664000175000017500000000207515146027420016500 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef LSTool_H #define LSTool_H #include "odc/tools/Tool.h" namespace odc { namespace tool { class LSTool : public Tool { public: LSTool(int argc, char* argv[]); void run(); static void help(std::ostream& o) { o << "Shows file's contents"; } static void usage(const std::string& name, std::ostream& o) { o << name << " [-o ] " << std::endl << std::endl; } unsigned long long printData(const std::string& db, std::ostream& out); private: // No copy allowed LSTool(const LSTool&); LSTool& operator=(const LSTool&); static const std::string nullString; }; } // namespace tool } // namespace odc #endif odc-1.6.3/src/odc/tools/SetTool.cc0000664000175000017500000000651415146027420017055 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/PathName.h" #include "eckit/log/Log.h" #include "eckit/utils/Tokenizer.h" #include "odc/ConstantSetter.h" #include "odc/LibOdc.h" #include "odc/Reader.h" #include "odc/Writer.h" #include "SetTool.h" using namespace eckit; using namespace odc::core; namespace odc { namespace tool { SetTool::SetTool(int argc, char* parameters[]) : Tool(argc, parameters) {} void SetTool::run() { if (parameters().size() != 4) { Log::error() << "Usage: "; usage(parameters(0), Log::error()); Log::error() << std::endl; std::stringstream ss; ss << "Expected exactly 4 command line parameters"; throw UserError(ss.str()); } std::vector columns; std::vector values; PathName inFile = parameters(2); PathName outFile = parameters(3); Reader in(inFile); Writer<> out(outFile); Writer<>::iterator writer(out.begin()); Reader::iterator sourceIt = in.begin(); const Reader::iterator sourceEnd = in.end(); parseUpdateList(parameters(1), columns, values); typedef ConstantSetter Setter; Setter setter(sourceIt, sourceEnd, columns, values); Setter::iterator begin = setter.begin(); const Setter::iterator end = setter.end(); writer->pass1(begin, end); } void SetTool::parseUpdateList(std::string s, std::vector& columns, std::vector& values) { Tokenizer splitAssignments(","); std::vector assignments; splitAssignments(s, assignments); Tokenizer splitEq("="); for (size_t i = 0; i < assignments.size(); ++i) { std::vector assignment; splitEq(assignments[i], assignment); ASSERT(assignment.size() == 2); std::string colName = assignment[0]; std::string value = assignment[1]; Log::info() << "SetTool::parseUpdateList: " << colName << "='" << value << "'" << std::endl; double v = 0; if (value.find("0x") != 0) v = translate(value); else { value = value.substr(2); ASSERT("Format of the hexadecimal value is not correct" && (value.size() % 2) == 0); ASSERT("Hexadecimal literal is too long" && (value.size() / 2) <= sizeof(double)); bzero(&v, sizeof(double)); for (size_t i = 0; i < value.size() / 2; ++i) { std::string byteInHex = value.substr(i * 2, 2); char* p = 0; unsigned char x; reinterpret_cast(&v)[i] = x = static_cast(strtoul(byteInHex.c_str(), &p, 16)); LOG_DEBUG_LIB(LibOdc) << "SetTool::parseUpdateList: '" << byteInHex << "' => " << x << std::endl; } } columns.push_back(colName); values.push_back(v); } } } // namespace tool } // namespace odc odc-1.6.3/src/odc/tools/CompareTool.h0000664000175000017500000000246615146027420017554 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef CompareTool_H #define CompareTool_H #include "odc/tools/Tool.h" #include "eckit/filesystem/PathName.h" namespace odc { class RowsReaderIterator; namespace tool { class CompareTool : public Tool { public: CompareTool(int argc, char* argv[]); void run(); static void help(std::ostream& o) { o << "Compares two ODB files"; } static void usage(const std::string& name, std::ostream& o) { o << name << " [-excludeColumns ] [-excludeColumnsTypes ] [-dontCheckMissing] " " "; } private: // No copy allowed CompareTool(const CompareTool&); CompareTool& operator=(const CompareTool&); static char* dummyArgv_[]; eckit::PathName file1_; eckit::PathName file2_; odc::RowsReaderIterator* reader1_; odc::RowsReaderIterator* reader2_; }; } // namespace tool } // namespace odc #endif odc-1.6.3/src/odc/tools/CompareTool.cc0000664000175000017500000000501415146027420017702 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "odc/tools/CompareTool.h" #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/PathName.h" #include "eckit/log/Log.h" #include "eckit/log/Timer.h" #include "eckit/utils/StringTools.h" #include "odc/Comparator.h" #include "odc/Reader.h" using namespace std; using namespace eckit; namespace odc { namespace tool { CompareTool::CompareTool(int argc, char* argv[]) : Tool(argc, argv) { registerOptionWithArgument("-excludeColumnsTypes"); registerOptionWithArgument("-excludeColumns"); if (parameters().size() != 3) { Log::error() << "Usage: "; usage(parameters(0), Log::error()); Log::error() << std::endl; std::stringstream ss; ss << "Expected exactly 3 command line parameters"; throw UserError(ss.str()); } PathName p; if (!(p = PathName(parameters()[1])).exists() || !(p = PathName(parameters()[2])).exists()) { stringstream s; s << "File " << p << " does not exist."; throw Exception(s.str()); } file1_ = parameters()[1]; file2_ = parameters()[2]; } void CompareTool::run() { Timer t(std::string("Comparing files ") + file1_ + " and " + file2_); odc::Reader oda1(file1_); odc::Reader oda2(file2_); odc::Reader::iterator it1(oda1.begin()); odc::Reader::iterator end1(oda1.end()); odc::Reader::iterator it2(oda2.begin()); odc::Reader::iterator end2(oda2.end()); std::vector excludedColumnsTypes = StringTools::split(",", optionArgument("-excludeColumnsTypes", std::string(""))); std::vector excludedColumns = StringTools::split(",", optionArgument("-excludeColumns", std::string(""))); if (excludedColumnsTypes.size()) { Log::info() << "excludedColumnsTypes:" << excludedColumnsTypes << std::endl; } if (excludedColumns.size()) { Log::info() << "excludedColumns:" << excludedColumns << std::endl; } bool checkMissing = !optionIsSet("-dontCheckMissing"); odc::Comparator(checkMissing).compare(it1, end1, it2, end2, file1_, file2_, excludedColumnsTypes, excludedColumns); } } // namespace tool } // namespace odc odc-1.6.3/src/odc/tools/SplitTool.h0000664000175000017500000000315415146027420017254 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef SplitTool_H #define SplitTool_H #include #include #include #include "Tool.h" #include "eckit/io/Length.h" #include "eckit/io/Offset.h" namespace odc { namespace tool { class SplitTool : public Tool { public: SplitTool(int argc, char* argv[]); void run(); static void help(std::ostream& o) { o << "Splits file according to given template"; } static void usage(const std::string& name, std::ostream& o) { o << name << " [-no_verification] [-maxopenfiles ] "; } static void split(const eckit::PathName&, const std::string&, size_t, bool verify = true); static void presortAndSplit(const eckit::PathName&, const std::string&); static std::vector > getChunks(const eckit::PathName&, size_t maxExpandedSize = 100 * 1024 * 1024); private: // No copy allowed SplitTool(const SplitTool&); SplitTool& operator=(const SplitTool&); static std::string genOrderBySelect(const std::string&, const std::string&); long maxOpenFiles_; bool sort_; }; } // namespace tool } // namespace odc #endif odc-1.6.3/src/odc/tools/LSTool.cc0000664000175000017500000000634615146027420016643 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/exception/Exceptions.h" #include "odc/Reader.h" #include "odc/tools/LSTool.h" using namespace eckit; namespace odc { namespace tool { LSTool::LSTool(int argc, char* argv[]) : Tool(argc, argv) { registerOptionWithArgument("-o"); // Text Output } const std::string LSTool::nullString; unsigned long long LSTool::printData(const std::string& db, std::ostream& out) { odc::Reader f(db); odc::Reader::iterator it = f.begin(); odc::Reader::iterator end = f.end(); core::MetaData md(0); // Formatting of real values: out << std::fixed; unsigned long long n = 0; for (; it != end; ++it, ++n) { if (md != it->columns()) { md = it->columns(); const char* spacer = ""; for (size_t i = 0; i < md.size(); ++i) { out << spacer << md[i]->name(); spacer = "\t"; } out << std::endl; } const char* spacer = ""; for (size_t i = 0; i < md.size(); ++i) { out << spacer; if (it->isMissing(i)) { out << "."; } else { switch (md[i]->type()) { case odc::api::INTEGER: case odc::api::BITFIELD: if (it->isMissing(i)) { out << "."; } else { out << static_cast((*it)[i]); } break; case odc::api::REAL: case odc::api::DOUBLE: out << (*it)[i]; break; case odc::api::STRING: out << "'" << (*it).string(i) << "'"; break; case odc::api::IGNORE: default: ASSERT("Unknown type" && false); break; } } spacer = "\t"; } out << std::endl; } return n; } void LSTool::run() { if (parameters().size() != 2) { Log::error() << "Usage: "; usage(parameters(0), Log::error()); Log::error() << std::endl; std::stringstream ss; ss << "Expected exactly 2 command line parameters"; throw UserError(ss.str()); } std::string db = parameters(1); std::unique_ptr foutPtr; std::ostream* out = &std::cout; if (optionIsSet("-o")) { foutPtr.reset(new std::ofstream(optionArgument("-o", std::string("")).c_str())); out = foutPtr.get(); } unsigned long long n = 0; n = printData(db, *out); Log::info() << "Selected " << n << " row(s)." << std::endl; } } // namespace tool } // namespace odc odc-1.6.3/src/odc/tools/SplitTool.cc0000664000175000017500000001162715146027420017416 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "SplitTool.h" #include #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/PathName.h" #include "eckit/io/PartFileHandle.h" #include "eckit/log/Log.h" #include "eckit/types/Types.h" #include "odc/DispatchingWriter.h" #include "odc/LibOdc.h" #include "odc/Reader.h" #include "odc/Select.h" #include "odc/TemplateParameters.h" #include "odc/core/TablesReader.h" using namespace eckit; using namespace std; namespace odc { namespace tool { SplitTool::SplitTool(int argc, char* argv[]) : Tool(argc, argv), maxOpenFiles_(200), sort_(false) { registerOptionWithArgument("-maxopenfiles"); } void SplitTool::run() { if (parameters().size() != 3) { Log::error() << "Usage: "; usage(parameters(0), Log::error()); Log::error() << endl; std::stringstream ss; ss << "Expected exactly 3 command line parameters"; throw UserError(ss.str()); } if (optionIsSet("-sort")) sort_ = true; maxOpenFiles_ = optionArgument("-maxopenfiles", maxOpenFiles_); LOG_DEBUG_LIB(LibOdc) << "SplitTool: maxOpenFiles_ = " << maxOpenFiles_ << endl; PathName inFile(parameters(1)); string outFileTemplate(parameters(2)); if (sort_) presortAndSplit(inFile, outFileTemplate); else split(inFile, outFileTemplate, maxOpenFiles_, !optionIsSet("-no_verification")); } /** * @param maxExpandedSize maximum size of the data in chunks after decoding */ vector > SplitTool::getChunks(const PathName& inFile, size_t maxExpandedSize) { LOG_DEBUG_LIB(LibOdc) << "SplitTool::getChunks: " << endl; vector > r; core::TablesReader reader(inFile); auto it(reader.begin()), end(reader.end()); Offset currentOffset(0); Length currentLength(0); size_t currentSize(0); for (; it != end; ++it) { Offset offset(it->startPosition()); Length length(it->nextPosition() - it->startPosition()); size_t numberOfRows(it->rowCount()); size_t numberOfColumns(it->columnCount()); LOG_DEBUG_LIB(LibOdc) << "SplitTool::getChunks: " << offset << " " << length << endl; size_t size(numberOfRows * numberOfColumns * sizeof(double)); if (currentSize + size > maxExpandedSize) { LOG_DEBUG_LIB(LibOdc) << "SplitTool::getChunks: collect " << currentOffset << " " << currentLength << endl; r.push_back(make_pair(currentOffset, currentLength)); currentOffset = offset; currentLength = length; } else { currentLength += length; currentSize += numberOfRows * numberOfColumns * sizeof(double); } } if (r.size() == 0 || r.back().first != currentOffset) r.push_back(make_pair(currentOffset, currentLength)); return r; } std::string SplitTool::genOrderBySelect(const std::string& inFile, const std::string& outFileTemplate) { core::TablesReader reader(inFile); auto it = reader.begin(); TemplateParameters templateParameters; TemplateParameters::parse(outFileTemplate, templateParameters, it->columns()); std::stringstream ss; ss << "select * order by "; for (size_t i = 0; i < templateParameters.size(); ++i) { if (i) ss << ","; ss << templateParameters[i]->name; } std::string sql(ss.str()); Log::info() << "SplitTool::genOrderBySelect: sql: '" << sql << "'" << endl; return sql; } void SplitTool::presortAndSplit(const PathName& inFile, const std::string& outFileTemplate) { odc::DispatchingWriter out(outFileTemplate, 1); odc::DispatchingWriter::iterator outIt(out.begin()); string sql(genOrderBySelect(inFile, outFileTemplate)); vector > chunks(getChunks(inFile)); for (size_t i = 0; i < chunks.size(); ++i) { PartFileHandle h(inFile, chunks[i].first, chunks[i].second); h.openForRead(); AutoClose closer(h); odc::Select in(sql, h); outIt->pass1(in.begin(), in.end()); } } void SplitTool::split(const PathName& inFile, const std::string& outFileTemplate, size_t maxOpenFiles, bool verify) { odc::Reader in(inFile); odc::DispatchingWriter out(outFileTemplate, maxOpenFiles); odc::DispatchingWriter::iterator outIt(out.begin()); outIt->pass1(in.begin(), in.end()); odc::Reader input(inFile); odc::Reader::iterator begin(input.begin()); odc::Reader::iterator end(input.end()); outIt->close(); if (verify) (**outIt).verify(begin, end); } } // namespace tool } // namespace odc odc-1.6.3/src/odc/tools/TestFunctionsForTemperatureConversion.cc0000664000175000017500000000357615146027420025234 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// \file UnitTest.cc /// /// @author ECMWF, July 2010 #include "eckit/log/Timer.h" #include "odc/Select.h" #include "TestCase.h" #include "odc/Writer.h" using namespace std; using namespace eckit; using namespace odc; static void setUp() { Timer t("Test various functions to convert temperatures"); odc::Writer<> oda("test_tempconv.odb"); odc::Writer<>::iterator row = oda.begin(); row->setNumberOfColumns(3); row->setColumn(0, "kelvin_col", odc::api::REAL); row->setColumn(1, "celsius_col", odc::api::REAL); row->setColumn(2, "fahrenheit_col", odc::api::REAL); row->writeHeader(); (*row)[0] = 273.15; (*row)[1] = 0.0; (*row)[2] = 32; ++row; } static void tearDown() { PathName("test_tempconv.odb").unlink(); } static void test() { const string sql = "select celsius(kelvin_col), fahrenheit(kelvin_col), c2k(celsius_col),c2f(celsius_col),f2c(fahrenheit_col), " "f2k(fahrenheit_col), k2f(kelvin_col) from \"test_tempconv.odb\";"; Log::info() << "Executing: '" << sql << "'" << std::endl; odc::Select oda(sql); odc::Select::iterator it = oda.begin(); ASSERT((*it)[0] == 0.0); // celsius(273.15) = 0.0 ASSERT((*it)[1] == 32); // farhenheit(273.15) = 31.73 ASSERT((*it)[2] == 273.15); // c2k ASSERT((*it)[3] == 32); // c2f ASSERT((*it)[4] == 0); // f2c ASSERT((*it)[5] == 273.15); // f2k ASSERT((*it)[6] == 32); // k2f } SIMPLE_TEST(FunctionsForTemperatureConversion) odc-1.6.3/src/odc/tools/TestFunctionThin.cc0000664000175000017500000000307115146027420020727 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// \file UnitTest.cc /// /// @author ECMWF, July 2010 const double EPS = 7e-6; #include #include "eckit/log/Timer.h" #include "odc/Select.h" #include "TestCase.h" #include "odc/Writer.h" using namespace std; using namespace eckit; using namespace odc; static void setUp() { Timer t("Test thin function"); odc::Writer<> oda("test_thin.odb"); odc::Writer<>::iterator row = oda.begin(); row->setNumberOfColumns(1); row->setColumn(0, "lat", odc::api::REAL); row->writeHeader(); (*row)[0] = 45.0; (*++row)[0] = 45.0; (*++row)[0] = 45.0; (*++row)[0] = 45.0; (*++row)[0] = 45.0; (*++row)[0] = 45.0; } static void tearDown() { PathName("test_thin.odb").unlink(); } static void test() { const string sql = "select thin(2.0,lat) from \"test_thin.odb\";"; Log::info() << "Executing: '" << sql << "'" << std::endl; int i = 0; odc::Select oda(sql); for (odc::Select::iterator it = oda.begin(); it != oda.end(); ++it) { if (i % 2 == 1) ASSERT(fabs((*it)[0] - 0.0e0) < EPS); // else ASSERT(fabs((*it)[0] - 1.0) < EPS); // ++i; } } SIMPLE_TEST(FunctionThin) odc-1.6.3/src/odc/tools/Tool.h0000664000175000017500000000221015146027420016230 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// \file Tool.h /// /// @author Piotr Kuchta, ECMWF, Feb 2009 #ifndef Tool_H #define Tool_H #include "odc/CommandLineParser.h" #include "odc/StringTool.h" namespace eckit { class PathName; } class Application; namespace odc { namespace tool { class Tool : public StringTool, public CommandLineParser { public: virtual void run() = 0; virtual ~Tool(); std::string name() { return name_; } void name(const std::string& s) { name_ = s; } static void registerTools(); protected: Tool(int argc, char** argv); Tool(const CommandLineParser&); private: std::string name_; }; template struct ExperimentalTool { enum { experimental = false }; }; } // namespace tool } // namespace odc #endif odc-1.6.3/src/odc/tools/TestBitfields.cc0000664000175000017500000000333715146027420020231 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// \file UnitTest.h /// /// @author Piotr Kuchta, ECMWF, Feb 2009 #include "eckit/log/Log.h" #include "odc/Select.h" #include "odc/core/MetaData.h" #include "TestCase.h" using namespace std; using namespace eckit; using namespace odc; /// UnitTest problem fixed with p4 change 23687 /// static void test() { string statusFields = "status.active@body,status.passive@body,status.rejected@body,status.blacklisted@body,status.monthly@body," "status.constant@body,status.experimental@body,status.whitelist@body"; statusFields = "status.*@body"; const string SELECT = std::string("select status@body, ") + statusFields + " from \"2000010106-reduced.odb\";"; Log::info() << "Executing '" << SELECT << "'" << std::endl; odc::Select oda(SELECT); long int i = 0; odc::Select::iterator it = oda.begin(); ASSERT(it->columns().size() == 9); for (; it != oda.end() && i < 5000; ++it, ++i) { unsigned int sum = int((*it)[1]) | int((*it)[2]) << 1 | int((*it)[3]) << 2 | int((*it)[4]) << 3 | int((*it)[5]) << 4 | int((*it)[6]) << 5 | int((*it)[7]) << 6 | int((*it)[8]) << 7; // Log::info() << i << ": " << (*it)[0] << " " << sum << std::endl; ASSERT((*it)[0] == sum); } } static void setUp() {} static void tearDown() {} SIMPLE_TEST(Bitfields) odc-1.6.3/src/odc/tools/TestStar.cc0000664000175000017500000000175215146027420017234 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// \file UnitTest.h /// /// @author Piotr Kuchta, ECMWF, May 2009 #include "odc/Select.h" #include "odc/core/MetaData.h" #include "TestCase.h" using namespace std; using namespace eckit; using namespace odc; /// UnitTest syntax: select *@odb_table from "file.oda"; /// static void test() { const std::string SELECT = "select *@hdr from \"2000010106-reduced.odb\";"; odc::Select oda(SELECT); odc::Select::iterator it = oda.begin(); ASSERT("hdr has 27 columns excluding @LINKs." && it->columns().size() == 27); } static void setUp() {} static void tearDown() {} SIMPLE_TEST(Star) odc-1.6.3/src/odc/tools/TestSelectTwoFiles.cc0000664000175000017500000000353515146027420021220 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// \file UnitTest.h /// /// @author Piotr Kuchta, ECMWF, Feb 2009 #include "eckit/log/Log.h" #include "odc/LibOdc.h" #include "odc/Select.h" #include "odc/Writer.h" #include "odc/utility/Tracer.h" #include "TestCase.h" using namespace std; using namespace eckit; using namespace odc; using namespace odc::utility; static void setUp() { Tracer t(Log::debug(), "setUp"); { odc::Writer<> f("TestSelectTwoFiles1.odb"); odc::Writer<>::iterator it = f.begin(); it->setNumberOfColumns(1); it->setColumn(0, "a", odc::api::REAL); it->writeHeader(); (*it)[0] = 1; ++it; } { odc::Writer<> f("TestSelectTwoFiles2.odb"); odc::Writer<>::iterator it = f.begin(); it->setNumberOfColumns(1); it->setColumn(0, "b", odc::api::REAL); it->writeHeader(); (*it)[0] = 2; ++it; } } static void test() { Tracer t(Log::debug(), "test"); odc::Select s("select * from \"TestSelectTwoFiles1.odb\", \"TestSelectTwoFiles2.odb\";"); odc::Select::iterator it = s.begin(); odc::Select::iterator end = s.end(); ASSERT(it->columns().size() == 2); unsigned long i = 0; for (; it != end; ++it) { LOG_DEBUG_LIB(LibOdc) << "test: " << (*it)[0] << " " << (*it)[0] << std::endl; ASSERT(((*it)[0] == 1) && ((*it)[1] == 2)); ++i; } ASSERT(i == 1); } static void tearDown() {} SIMPLE_TEST(SelectTwoFiles) odc-1.6.3/src/odc/tools/TestSelectDataHandle.cc0000664000175000017500000000337015146027420021446 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// \file UnitTest.h /// /// @author Piotr Kuchta, ECMWF, Jan 2010 #include "eckit/io/FileHandle.h" #include "eckit/log/Timer.h" #include "odc/Select.h" #include "TestCase.h" using namespace std; using namespace eckit; using namespace odc; /// UnitTest syntax 'select lat, lon' (no file name) /// static void test() { string sql = "select * where obstype = 7;"; // string sql = "select * where obstype = 7;"; // string sql = "select obstype from \"input.oda\";"; const string fileName = "2000010106-reduced.odb"; FileHandle fh(fileName); fh.openForRead(); AutoClose closer(fh); odc::Select oda(sql, fh); Log::info() << "test: Execute '" << sql << "'" << std::endl; long n = 0; { Timer t("test: selecting rows using SQL"); odc::Select::iterator it = oda.begin(); odc::Select::iterator end = oda.end(); for (; it != end; ++it) ++n; } Log::info() << "test: selected " << n << " rows." << std::endl; ASSERT(n == 44969); fh.close(); } static void setUp() { #if 0 string s = "Data to be saved"; TemporaryFile tmp; ofstream os(tmp.c_str()); os << s; os.close(); if(!os) throw WriteError(tmp); string cmd = "ls -l "; cmd += tmp; system(cmd.c_str()); cmd = "cat "; cmd += tmp; system(cmd.c_str()); #endif } static void tearDown() {} SIMPLE_TEST(SelectDataHandle) odc-1.6.3/src/odc/tools/ToolFactory.h0000664000175000017500000000375615146027420017600 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef ToolFactory_H #define ToolFactory_H // #include "TestCase.h" namespace odc { namespace tool { namespace test { class TestCase; } class Tool; class AbstractToolFactory { public: static Tool* createTool(const std::string& name, int argc, char** argv); static void printToolHelp(const std::string&, std::ostream&); static void printToolUsage(const std::string& name, std::ostream&); static void printToolsHelp(std::ostream&); static std::vector* testCases(const std::vector& = matchAll); static void listTools(std::ostream&); virtual Tool* create(int argc, char** argv) = 0; virtual void help(std::ostream&) = 0; virtual void usage(const std::string&, std::ostream&) = 0; virtual bool experimental() = 0; protected: AbstractToolFactory(const std::string& name); virtual ~AbstractToolFactory(); private: static AbstractToolFactory& findTool(const std::string& name); static std::map* toolFactories; static const std::vector matchAll; }; template class ToolFactory : public AbstractToolFactory { public: ToolFactory(const std::string& name) : AbstractToolFactory(name) {} Tool* create(int argc, char** argv) { return new T(argc, argv); } void help(std::ostream& o) { T::help(o); } void usage(const std::string& name, std::ostream& o) { T::usage(name, o); } bool experimental() { return odc::tool::ExperimentalTool::experimental; } }; } // namespace tool } // namespace odc #endif odc-1.6.3/src/odc/tools/TestDispatchingWriter.cc0000664000175000017500000000312215146027420021746 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// \file UnitTest.h /// /// @author Piotr Kuchta, ECMWF, June 2009 #include "odc/DispatchingWriter.h" #include "odc/Reader.h" #include "odc/StringTool.h" #include "CountTool.h" #include "TestCase.h" using namespace std; using namespace eckit; using namespace odc; /// UnitTest DispatchingWriter /// static void test() { const string fileName = "2000010106-reduced.odb"; odc::Reader oda(fileName); odc::Reader::iterator it = oda.begin(); const odc::Reader::iterator end = oda.end(); ASSERT(it->columns().size() > 0); odc::DispatchingWriter writer("disp.{obstype}.{sensor}.odb"); odc::DispatchingWriter::iterator wi = writer.begin(); unsigned long long n1 = wi->pass1(it, end); wi->close(); unsigned long long sum = 0; vector files = (**wi).outputFiles(); for (size_t i = 0; i < files.size(); ++i) { unsigned long long n = odc::tool::CountTool::rowCount(files[i]); Log::info() << i << ". " << files[i] << ": " << n << std::endl; sum += n; } ASSERT(n1 == sum); odc::StringTool::shell("ls -l disp.*.*.odb", Here()); } static void setUp() {} static void tearDown() {} SIMPLE_TEST(DispatchingWriter) odc-1.6.3/src/odc/tools/XYVTool.h0000664000175000017500000000204115146027420016641 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef XYVTool_H #define XYVTool_H #include #include "Tool.h" namespace odc { namespace tool { class XYVTool : public Tool { public: XYVTool(int argc, char** argv); ~XYVTool(); static void help(std::ostream& o) { o << "Creates XYV representation of file for displaying in a graphics program"; } static void usage(const std::string& name, std::ostream& o) { o << name << " "; } virtual void run(); }; template <> struct ExperimentalTool { enum { experimental = true }; }; } // namespace tool } // namespace odc #endif odc-1.6.3/src/odc/tools/odc.cc0000664000175000017500000001255415146027420016232 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/io/FileHandle.h" #include "eckit/sql/expression/function/FunctionFactory.h" #include "odc/ODBAPIVersion.h" #include "odc/odccapi.h" #include "TestOdaCAPI.h" #include "TestRunnerApplication.h" #include "Tool.h" #include "ToolFactory.h" #include "ToolRunnerApplication.h" using namespace std; using namespace eckit; using namespace odc::tool; int executeCommand(int argc, char* argv[]); int gdb(int argc, char* argv[]); int valgrind(int argc, char* argv[]); int sqlhelp(int argc, char* argv[]); // This is a hack to ensure that the odctest library links in properly #include "odc/tools/TestOdaCAPI.h" int (*global_odctest_hack)(int, char**) = &::odc::tool::test::test_odacapi_setup; int main(int argc, char* argv[]) { try { return executeCommand(argc, argv); } catch (std::exception& e) { cerr << argv[0] << ": " << e.what() << std::endl; return 1; } catch (...) { cerr << argv[0] << ": unknown exception" << std::endl; return 1; } } int executeCommand(int argc, char* argv[]) { odc::tool::Tool::registerTools(); if (argc < 2) { odb_start_with_args(argc, argv); cerr << "Usage:" << endl << " " << argv[0] << " []" << endl << " " << argv[0] << " help " << std::endl << endl << "Available commands:" << std::endl; AbstractToolFactory::listTools(cout); return 1; } const string firstArg(argv[1]); if (firstArg == "g") { odb_start_with_args(argc, argv); return gdb(argc, argv); } if (firstArg == "vg") { odb_start_with_args(argc, argv); return valgrind(argc, argv); } /// TODO: reenable // if (firstArg == "testodbcapi") return odc::tool::test::test_odacapi(argc, argv); if (firstArg == "test") { if (argc == 2) // no args => test all { std::cout << "Testing C API" << std::endl; string testCapi = std::string(argv[0]) + " testodbcapi"; cerr << "Executing '" << testCapi << "'" << std::endl; int rc = system(testCapi.c_str()); if (rc) return rc; } std::cout << std::endl << "Running tests." << std::endl; odc::tool::test::TestRunnerApplication testRunner(argc - 1, argv + 1); testRunner.start(); // It never really gets here. return 0; } if (firstArg == "help") { odb_start_with_args(argc, argv); if (argc == 2) AbstractToolFactory::printToolsHelp(cout); else { AbstractToolFactory::printToolHelp(argv[2], cout); std::cout << std::endl << "Usage:" << std::endl << std::endl << "\t"; AbstractToolFactory::printToolUsage(argv[2], cout); } return 0; } if (firstArg == "sqlhelp") return sqlhelp(argc, argv); if (firstArg == "-V" || firstArg == "-v" || firstArg == "--version") { std::cout << "ODBAPI Version: " << odc::ODBAPIVersion::version() << std::endl; std::cout << "File format version: " << odc::ODBAPIVersion::formatVersionMajor() << "." << odc::ODBAPIVersion::formatVersionMinor() << std::endl; return 0; } ToolRunnerApplication runner(argc, argv); return runner.start(); } int gdb(int argc, char* argv[]) { string cmd = argv[0]; string args; string gdbScript = argv[1]; for (int i = 2; i < argc; i++) args += std::string(" ") + argv[i]; PathName scriptFile(std::string(".gdb_") + std::string(argc < 3 ? "odc" : argv[2])); if (!scriptFile.exists()) { string s = std::string("file ") + cmd + "\nbreak main\nrun " + args + "\ncatch throw\n"; FileHandle f(scriptFile); f.openForWrite(1024); f.write(s.c_str(), s.size()); f.close(); } string vi = std::string("vi ") + scriptFile; string gdb = std::string("gdb -x ") + scriptFile; std::cout << "Executing '" << vi << "'" << std::endl; system(vi.c_str()); std::cout << "Executing '" << gdb << "'" << std::endl; return system(gdb.c_str()); } // valgrind --log-file=v.log --show-reachable=yes --leak-check=full ./oda test int valgrind(int argc, char* argv[]) { string cmd = argv[0]; string args; for (int i = 2; i < argc; i++) args += std::string(" ") + argv[i]; string logFile = std::string("vg.") + argv[2] + ".log"; string vg = std::string("valgrind --log-file=") + logFile + " --show-reachable=yes --leak-check=full " + " --db-attach=yes " // --suppressions=eckit.supp " + cmd + " " + args; std::cout << "Executing '" << vg << "'" << std::endl; return system(vg.c_str()); } int sqlhelp(int argc, char* argv[]) { auto info = eckit::sql::expression::function::FunctionFactory::instance().functionsInfo(); for (auto& i : info) { std::cout << i.name << "/" << i.arity << " " << i.help << std::endl; } return 0; } odc-1.6.3/src/odc/tools/TestMissingValue.cc0000664000175000017500000001023115146027420020721 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// \file TestMissingValue.h /// /// @author Piotr Kuchta, ECMWF, Feb 2009 #include "eckit/log/Log.h" #include "eckit/sql/SQLTypedefs.h" #include "odc/Comparator.h" #include "odc/LibOdc.h" #include "odc/Reader.h" #include "odc/Select.h" #include "odc/Writer.h" #include "odc/utility/Tracer.h" #include "TestCase.h" using namespace std; using namespace eckit; using namespace odc; using namespace odc::utility; using namespace odc::core; static void setUp() { Tracer t(Log::debug(), "setUp"); odc::Writer<> f("TestMissingValue.odb"); odc::Writer<>::iterator it = f.begin(); it->setNumberOfColumns(2); it->setColumn(0, "lat@hdr", odc::api::REAL); it->missingValue(0, 1); eckit::sql::BitfieldDef bfDef; bfDef.first.push_back("x"); bfDef.second.push_back(1); bfDef.first.push_back("y"); bfDef.second.push_back(2); it->setBitfieldColumn(1, "bf", odc::api::BITFIELD, bfDef); it->missingValue(1, 0); it->writeHeader(); for (size_t i = 0; i <= 2; i++) { (*it)[0] = i; (*it)[1] = i; ++it; } } static void selectIntoSecondFile() { Tracer t(Log::debug(), "selectIntoSecondFile"); const string fileName = "TestMissingValue.odb"; string sql = "select lat,bf into \"TestMissingValue2.odb\""; sql += " from \"" + fileName + "\" ;"; odc::Select f(sql); //, fileName); odc::Select::iterator it = f.begin(); ++it; // this is needed to push the second row to the INTO file ++it; // this is needed to push the third row to the INTO file } static void test() { selectIntoSecondFile(); odc::Comparator().compare("TestMissingValue.odb", "TestMissingValue2.odb"); { odc::Reader f("TestMissingValue.odb"); odc::Reader::iterator fbegin(f.begin()); odc::Reader::iterator fend(f.end()); odc::Select s("select * from \"TestMissingValue2.odb\";"); odc::Select::iterator sbegin(s.begin()); odc::Select::iterator send(s.end()); odc::Comparator().compare(fbegin, fend, sbegin, send, "TestMissingValue.odb", "SELECT TestMissingValue2.odb"); } { odc::Reader f("TestMissingValue2.odb"); odc::Reader::iterator it = f.begin(); odc::Reader::iterator end = f.end(); Column& column = *it->columns()[0]; Codec& codec = column.coder(); ASSERT(codec.hasMissing()); // We do not preserve missing values across these calls. Missing value is as // configured in the encoder, not the data source // ASSERT(codec.missingValue() == 1); ASSERT(codec.missingValue() == MDI::realMDI()); for (int row = 0; it != end; ++it, ++row) { if (row == 1) { ASSERT((*it).isMissing(0)); } else { ASSERT(!(*it).isMissing(0)); } } } { // Check that we correctly identify missing items in the select api // n.b. We _don't' expose the original missing value... odc::Select s("select * from \"TestMissingValue2.odb\";"); //, fileName); odc::Select::iterator i = s.begin(); odc::Select::iterator e = s.end(); for (int row = 0; i != e; ++i, ++row) { ASSERT((*i).missingValue(0) == MDI::realMDI()); ASSERT((*i).missingValue(1) == MDI::bitfieldMDI()); if (row == 0) { ASSERT(!(*i).isMissing(0)); ASSERT((*i).isMissing(1)); } else if (row == 1) { ASSERT((*i).isMissing(0)); ASSERT(!(*i).isMissing(1)); } else { ASSERT(!(*i).isMissing(0)); ASSERT(!(*i).isMissing(1)); } } } } static void tearDown() {} SIMPLE_TEST(MissingValue) odc-1.6.3/src/odc/tools/TestRunnerApplication.cc0000664000175000017500000000174615146027420021763 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// \file TestRunnerApplication.h /// /// @author Piotr Kuchta, ECMWF, Feb 2009 #include #include "TestCase.h" #include "TestRunner.h" #include "TestRunnerApplication.h" #include "Tool.h" #include "ToolFactory.h" namespace odc { namespace tool { namespace test { TestRunnerApplication::TestRunnerApplication(int argc, char** argv) : ODBApplication(argc, argv) {} TestRunnerApplication::~TestRunnerApplication() {} void TestRunnerApplication::run() { TestRunner testRunner(commandLineParser()); testRunner.run(); } } // namespace test } // namespace tool } // namespace odc odc-1.6.3/src/odc/tools/TestAggregateFunctions3.cc0000664000175000017500000000242515146027420022163 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// \file TestAggregateFunctions3.h /// /// @author Piotr Kuchta, ECMWF, September 2010 #include "eckit/io/FileHandle.h" #include "odc/Select.h" #include "TestCase.h" #include "odc/api/Odb.h" using namespace std; using namespace eckit; using namespace odc; static void test() { string sql = "select sum(a) from \"TestAggregateFunctions3.odb\";"; Log::info() << "Executing: '" << sql << "'" << std::endl; odc::Select sel(sql); odc::Select::iterator it2 = sel.begin(); odc::Select::iterator end2 = sel.end(); ASSERT((*it2)[0] == 55); } static void setUp() { stringstream s; s << "a:REAL" << std::endl; for (size_t i = 1; i <= 10; ++i) s << i << std::endl; FileHandle dh("TestAggregateFunctions3.odb"); dh.openForWrite(0); AutoClose close(dh); odc::api::odbFromCSV(s, dh); } static void tearDown() {} SIMPLE_TEST(AggregateFunctions3) odc-1.6.3/src/odc/tools/TestInt16_MissingCodec.cc0000664000175000017500000000640615146027420021654 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// \file UnitTest.h /// /// @author Piotr Kuchta, ECMWF, Jan 2010 #include "eckit/log/Log.h" #include "eckit/log/Timer.h" #include "odc/LibOdc.h" #include "odc/Reader.h" #include "odc/Writer.h" #include "odc/core/MetaData.h" #include "MockReader.h" #include "TestCase.h" using namespace std; using namespace eckit; using namespace odc; class MockReaderIterator3 { public: MockReaderIterator3() : noMore_(false), refCount_(0), columns_(1), nRows_(0), min_(23), data_(0) { core::Column* col = columns_[0] = new core::Column(columns_); ASSERT(col); col->name("column_name"); col->type(odc::api::INTEGER); col->hasMissing(true); next(); } core::MetaData& columns() { return columns_; } bool isNewDataset() { return false; } double* data() { return &data_; } double& data(size_t) { return data_; } // MockReaderIterator3& operator++() { next(); return *this; } bool next() { if (noMore_) return noMore_; switch (nRows_++) { case 0: data_ = min_ + 0; break; case 1: data_ = min_ + (0xffff - 1) / 2; break; case 2: data_ = min_ + (0xffff - 1); break; case 3: data_ = columns_[0]->coder().missingValue(); break; default: noMore_ = true; return false; } return true; } bool noMore_; int refCount_; private: core::MetaData columns_; unsigned int nRows_; double min_; double data_; }; static void setUp() { Timer t("Writing test_int16_missing.odb"); odc::Writer<> oda("test_int16_missing.odb"); typedef tool::MockReader M; M reader; M::iterator b = reader.begin(); const M::iterator e = reader.end(); odc::Writer<>::iterator outit = oda.begin(); outit->pass1(b, e); } static void test() { odc::Reader oda("test_int16_missing.odb"); odc::Reader::iterator it = oda.begin(); odc::Reader::iterator end = oda.end(); typedef tool::MockReader M; M reader; M::iterator originalIt = reader.begin(); const M::iterator originalItEnd = reader.end(); Log::info() << it->columns() << std::endl; for (; it != end; ++it, ++originalIt) { Log::info() << "it[0] = " << (*it)[0] << ", originalIt.data()[0]=" << (*originalIt)[0] << std::endl; ASSERT((*it)[0] == (*originalIt)[0]); } core::Codec& coder(it->columns()[0]->coder()); string name = coder.name(); LOG_DEBUG_LIB(LibOdc) << "test: codec name is '" << name << "'" << std::endl; ASSERT(name == "int16_missing"); LOG_DEBUG_LIB(LibOdc) << "test: OK" << std::endl; } static void tearDown() {} SIMPLE_TEST(Int16_MissingCodec) odc-1.6.3/src/odc/tools/TestSetvbuffer.cc0000664000175000017500000000347315146027420020440 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// \file TestSetvbuffer.h /// /// @author Piotr Kuchta, ECMWF, Feb 2009 #include #include "eckit/log/Timer.h" #include "odc/ODBAPISettings.h" #include "TestCase.h" #include "odc/Writer.h" using namespace std; using namespace eckit; using namespace odc; static void setUp() {} static void createFile(size_t numberOfColumns, long long numberOfRows, size_t setvbufferSize) { ODBAPISettings::instance().setvbufferSize(setvbufferSize); odc::Writer<> oda("TestSetvbuffer.odb"); odc::Writer<>::iterator row = oda.begin(); row->setNumberOfColumns(numberOfColumns); for (size_t i = 0; i < numberOfColumns; ++i) { stringstream name; name << "Column" << i; row->setColumn(i, name.str().c_str(), odc::api::REAL); } row->writeHeader(); for (long long i = 1; i <= numberOfRows; ++i, ++row) for (size_t c = 0; c < numberOfColumns; ++c) (*row)[c] = c; } static void tearDown() { int catStatus = std::system("ls -l TestSetvbuffer.odb"); ASSERT(WEXITSTATUS(catStatus) == 0); } static void test() { size_t cols = 400; long long rows = 1000; size_t buffSize = 8 * 1024 * 1024; for (size_t i = 0; i < 10; ++i) { stringstream s; s << "setUp(): createFile(" << cols << ", " << rows << ", " << buffSize << ")" << std::endl; Timer t(s.str()); createFile(cols, rows, buffSize); } } SIMPLE_TEST(Setvbuffer) odc-1.6.3/src/odc/tools/UnitTests.cc0000664000175000017500000006257615146027420017440 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// \file .h /// /// @author Piotr Kuchta, ECMWF, Feb 2009 #include #include "eckit/exception/Exceptions.h" #include "eckit/io/BufferedHandle.h" #include "eckit/io/FileHandle.h" #include "eckit/log/Number.h" #include "eckit/log/Timer.h" #include "eckit/sql/SQLParser.h" #include "eckit/sql/SQLSelectFactory.h" #include "odc/Comparator.h" #include "odc/DispatchingWriter.h" #include "odc/ODBAPISettings.h" #include "odc/Reader.h" #include "odc/Select.h" #include "odc/Writer.h" #include "odc/api/Odb.h" #include "odc/core/TablesReader.h" #include "odc/odccapi.h" #include "odc/tools/CountTool.h" #include "odc/tools/SplitTool.h" #include "odc/tools/TestCase.h" using namespace std; using namespace eckit; using namespace odc; using namespace odc::sql; typedef long long llong; static void foobar() { Reader in("concatenated.odb"); Reader::iterator it = in.begin(); Reader::iterator end = in.end(); Writer<> out("copy_of_concatenated.odb"); Writer<>::iterator o = out.begin(); o->pass1(it, end); Comparator().compare("concatenated.odb", "copy_of_concatenated.odb"); } // TESTCASE(foobar); static void createDataForMixedAggregated() { // See UnitTest.sql as well const char* data = "x:INTEGER,y:INTEGER,v:DOUBLE\n" "1,1,0.3\n" "1,1,0.2\n" "2,2,0.4\n" "2,2,0.1\n"; eckit::FileHandle dh("selectAggregatedAndNonAggregated.odb"); dh.openForWrite(0); AutoClose close(dh); odc::api::odbFromCSV(data, dh); } TEST(selectAggregatedAndNonAggregated) { createDataForMixedAggregated(); odc::Select oda("select x,min(v),max(v) from \"selectAggregatedAndNonAggregated.odb\";"); odc::Select::iterator it = oda.begin(); double r0 = (*it)[0], r1 = (*it)[1], r2 = (*it)[2]; Log::info() << "selectAggregatedAndNonAggregated: " << r0 << ", " << r1 << ", " << r2 << std::endl; ASSERT(Comparator::same(r0, 1)); ASSERT(Comparator::same(r1, 0.2)); ASSERT(Comparator::same(r2, 0.3)); ++it; r0 = (*it)[0]; r1 = (*it)[1]; r2 = (*it)[2]; Log::info() << "selectAggregatedAndNonAggregated: " << r0 << ", " << r1 << ", " << r2 << std::endl; // ASSERT((*it)[0] == 2 && (*it)[1] == 0.1); ASSERT(r0 == 2); ASSERT(r1 == 0.1); ASSERT(r2 == 0.4); ++it; ASSERT(!(it != oda.end())); } static void createDataForMixedAggregated2() { Writer<> out("selectAggregatedAndNonAggregated2.odb"); Writer<>::iterator o = out.begin(); MetaData md(o->columns()); md.addColumn /* */ ("x", "INTEGER"); //, true, .0); md.addColumn /* */ ("y", "INTEGER"); //, true, .0); md.addColumn /* */ ("v", "DOUBLE"); //, true, .0); o->columns(md); o->writeHeader(); for (size_t row = 0; row < 1000; ++row) for (size_t x = 0; x < 10; (*o)[0] = ++x) for (size_t y = 0; y < 10; (*o)[1] = ++y) for (double v = 0; v < 10; (*o)[2] = ++v) ++o; } TEST(selectAggregatedAndNonAggregated2) { createDataForMixedAggregated2(); odc::Select oda("select x,min(v),y,max(v) from \"selectAggregatedAndNonAggregated2.odb\";"); odc::Select::iterator it = oda.begin(); unsigned long counter = 0; for (; it != oda.end(); ++it, ++counter) { // double r0 = (*it)[0], r1 = (*it)[1], r2 = (*it)[2], r3 = (*it)[3]; // Log::info() << "selectAggregatedAndNonAggregated2: " << r0 << ", " << r1 << ", " << r2 << ", " << r3 << // std::endl; } Log::info() << "selectAggregatedAndNonAggregated2: counter= " << counter << std::endl; ASSERT(counter == 110); } static void createDataForMixedAggregated3() { // See UnitTest.sql as well const char* data = "x:STRING,y:INTEGER,v:DOUBLE\n" "'A',1,0.3\n" "'A',1,0.2\n" "'B',2,0.4\n" "'B',2,0.1\n"; FileHandle dh("selectAggregatedAndNonAggregated3.odb"); dh.openForWrite(0); AutoClose close(dh); odc::api::odbFromCSV(data, dh); } TEST(selectAggregatedAndNonAggregated3) { createDataForMixedAggregated3(); odc::Select oda("select x,count(*) from \"selectAggregatedAndNonAggregated3.odb\";"); odc::Select::iterator it = oda.begin(); unsigned long counter = 0; for (; it != oda.end(); ++it, ++counter) { double r0 = (*it)[0], r1 = (*it)[1]; Log::info() << "selectAggregatedAndNonAggregated3: " << r0 << ", " << r1 << std::endl; } Log::info() << "selectAggregatedAndNonAggregated3: counter= " << counter << std::endl; ASSERT(counter == 2); } static void createDataForMixedAggregatedNULL() { // See UnitTest.sql as well const char* data = "x:REAL,y:INTEGER,v:DOUBLE\n" "100,1,0.3\n" "100,1,0.2\n" "101,2,0.4\n" "101,2,0.1\n" "NULL,1,0.1\n" "NULL,2,0.2\n" "NULL,3,0.3\n"; FileHandle dh("selectAggregatedAndNonAggregatedNULL.odb"); dh.openForWrite(0); AutoClose close(dh); odc::api::odbFromCSV(data, dh); } TEST(selectAggregatedAndNonAggregatedNULL) { createDataForMixedAggregatedNULL(); odc::Select oda("select x,count(*) from \"selectAggregatedAndNonAggregatedNULL.odb\";"); odc::Select::iterator it = oda.begin(); unsigned long counter = 0; for (; it != oda.end(); ++it, ++counter) { double r0 = (*it)[0], r1 = (*it)[1]; Log::info() << "selectAggregatedAndNonAggregatedNULL: " << r0 << ", " << r1 << std::endl; } Log::info() << "selectAggregatedAndNonAggregatedNULL: counter= " << counter << std::endl; ASSERT(counter == 3); } ///////////////////////////////////////// // Regular expressions on the select list static void createDataForRegex1() { // See UnitTest.sql as well const char* data = "aa:INTEGER,ab:INTEGER,ba:INTEGER,bb:INTEGER\n" "1,2,3,4\n" "10,20,30,40\n" "11,22,33,44\n"; FileHandle dh("regex1.odb"); dh.openForWrite(0); AutoClose close(dh); odc::api::odbFromCSV(data, dh); } static void regex1() { // createDataForRegex1(); odc::Select oda("select \"/a.*/\" from \"regex1.odb\";"); odc::Select::iterator it = oda.begin(); Log::info() << "regex1: " << it->columns() << std::endl; ASSERT(it->columns().size() == 2); } // TESTCASE(regex1); TEST(vector_syntax) { const char* data = "a:INTEGER,b:INTEGER\n" "1,1\n" "2,2\n" "3,3\n" "4,4\n" "5,5\n" "6,6\n" "7,7\n" "8,8\n" "9,9\n" "10,10\n"; { FileHandle dh("vector_syntax.odb"); dh.openForWrite(0); AutoClose close(dh); odc::api::odbFromCSV(data, dh); } const char* sql = "set $X = [1,2,3,4,5];" "select * from \"vector_syntax.odb\" where a in $X;"; odc::Select oda(sql); odc::Select::iterator it = oda.begin(); odc::Select::iterator end = oda.end(); unsigned long counter = 0; for (; it != oda.end(); ++it, ++counter) ; ASSERT(counter == 5); } TEST(bitfieldsLength) { Log::info() << "sizeof(eckit::log::Number::W)" << sizeof(eckit::log::Number::W) << std::endl; Log::info() << "sizeof(double)" << sizeof(double) << std::endl; //>>> int('0b11100110011',2) // 1843 //>>> len('0b11100110011') // 13 //>>> // stringstream s; // eckit::log::Number::printBinary(s, 1843); // string r = s.str(); // Log::info() << "r: " << r << std::endl; ASSERT(eckit::log::Number::printBinary(1843).size() == 11); ASSERT(eckit::log::Number::printBinary(1843) == "11100110011"); ASSERT(eckit::log::Number::printBinary(0).size() == 1); ASSERT(eckit::log::Number::printBinary(0) == "0"); } /// ODB-85 TEST(bitfieldsPrintHexadecimal) { ASSERT(eckit::log::Number::printHexadecimal(1843) == std::string("733")); // eckit::Log::info() << eckit::log::Number::printHexadecimal(15) << std::endl; ASSERT(eckit::log::Number::printHexadecimal(10) == std::string("a")); ASSERT(eckit::log::Number::printHexadecimal(11) == std::string("b")); ASSERT(eckit::log::Number::printHexadecimal(15) == std::string("f")); ASSERT(eckit::log::Number::printHexadecimal(255) == std::string("ff")); } static void create_stringInWhere_file() { const char* data = "a:STRING,b:INTEGER\n" "'aaa',1\n" "'aaaa',2\n" "'bbb',2\n" "'bbbc',2\n"; FileHandle dh("stringInWhere.odb"); dh.openForWrite(0); AutoClose close(dh); odc::api::odbFromCSV(data, dh); } TEST(stringInWhere) { create_stringInWhere_file(); odc::Select oda("select * from 'stringInWhere.odb' where a = 'aaa';"); unsigned long counter = 0; for (odc::Select::iterator it = oda.begin(); it != oda.end(); ++it, ++counter) ; ASSERT(counter == 1); } TEST(vector_syntax2) { const char* sql = "set $y = 100; set $x = [$y, 'a', 'b', [1, 2]];"; eckit::sql::SQLSession session; eckit::sql::SQLParser::parseString(session, sql); } TEST(rownumber1) { const char* inputData = "a:INTEGER,b:INTEGER\n" "1,1\n" "2,2\n" "3,3\n" "4,4\n" "5,5\n" "6,6\n" "7,7\n" "8,8\n" "9,9\n" "10,10\n"; string path("Test_rownumber1.odb"); { FileHandle dh(path); dh.openForWrite(0); AutoClose close(dh); odc::api::odbFromCSV(inputData, dh); } string query("SELECT rownumber() from \"" + path + "\";"); odc::Select select(query); odc::Select::iterator it = select.begin(); odc::Select::iterator end = select.end(); llong i = 0; for (; it != end; ++it) { ASSERT((*it)[0] == ++i); } ASSERT(i == 10); } TEST(sqlOutputFormatting) { /* // See UnitTest.sql as well const char *data = "x:REAL,y:INTEGER,v:DOUBLE\n" "100,1,0.3\n" "100,1,0.2\n" "101,2,0.4\n" "101,2,0.1\n" "NULL,1,0.1\n" "NULL,2,0.2\n" "NULL,3,0.3\n" ; const char* testFile("sqlOutputFormatting.odb"); FileHandle dh(testFile); dh.openForWrite(0); AutoClose close(dh); odc::api::odbFromCSV(data, dh); bool doNotWriteColumnNames(false); // -T bool doNotWriteNULL(false); // -N string delimiter(" "); // -delimiter string inputFile(testFile); // -i string outputFile; // -o string outputFormat; // default is ascii odc::sql::SQLSelectFactory::instance() .config(odc::sql::SQLOutputConfig(doNotWriteColumnNames, doNotWriteNULL, delimiter, outputFile, outputFormat)); ostream& out = cout; odc::sql::SQLInteractiveSession session(out); odc::sql::SQLParser p; FileHandle fh(inputFile); fh.openForRead(); //p.parseString(StringTool::readFile(fileName), &fh, odc::sql::SQLSelectFactory::instance().config()); p.parseString("select x,y,v;", &fh, odc::sql::SQLSelectFactory::instance().config()); */ } static void createDataForWindSpeedWindDirection() { const char* data = "u:REAL,v:REAL\n" "11.7,-5.8\n" "0.0,0.0\n" "0,5.4\n" "5.4,0.0\n"; FileHandle dh("uv.odb"); dh.openForWrite(0); AutoClose close(dh); odc::api::odbFromCSV(data, dh); } TEST(windSpeedWindDirection) { createDataForWindSpeedWindDirection(); string path("uv.odb"); string query("SELECT ff(u,v), dd(u,v), speed(u,v),dir(u,v), sqrt(u*u+v*v), fmod(atan2(-u,-v)+360.,360.) from \"" + path + "\";"); odc::Select select(query); odc::Select::iterator it = select.begin(); odc::Select::iterator end = select.end(); llong i = 0; for (; it != end; ++it) { Log::info() << " ff = " << (*it)[0] << " speed sqrt= " << (*it)[4] << std::endl; Log::info() << " dd = " << (*it)[1] << " direction atan= " << (*it)[5] << std::endl; ASSERT((*it)[0] == (*it)[4]); ASSERT((*it)[1] == (*it)[5]); } } // TEST(odbcapi) //{ // odc::tool::test::test_odacapi_setup_in_C(0,0); // odc::tool::test::test_odacapi3(0,0); // } static void SplitTool_chunks() { const char* fn = "selectAggregatedAndNonAggregated.odb"; size_t n = odc::tool::CountTool::rowCount(fn); vector > chunks = odc::tool::SplitTool::getChunks(fn); Log::info() << "chunks.size():" << chunks.size() << std::endl; ASSERT(chunks.size() == 1 && chunks[0].first == Offset(0) && chunks[0].second == Length(357)); } // TESTCASE(SplitTool_chunks); static void FilePool1() { // FilePool pool(1); } // TESTCASE(FilePool1); static void copyVectorToArray() { const size_t size = 1024; const size_t n = 1000000; vector v(size); double a[size]; { Timer timer("std::copy"); for (size_t i = 0; i < n; ++i) std::copy(v.begin(), v.end(), a); } { Timer timer("for loop"); for (size_t i = 0; i < n; ++i) for (size_t j = 0; j < size; ++j) a[j] = v[j]; } } // TESTCASE(copyVectorToArray); static void count(void* counter, const double* data, size_t n) { ++*((llong*)counter); } static void* create_counter() { llong* r = new llong; *r = 0; return r; } static void destroy_counter(void* counter) { delete (llong*)counter; } static void* reduce_counter(void* left, void* right) { llong* result = new llong; *result = (*(llong*)left) + (*(llong*)right); return result; } class TemporaryPathName : public PathName { public: TemporaryPathName(const string& fn) : PathName(fn) {} ~TemporaryPathName() { unlink(); } }; typedef TemporaryPathName ScratchFile; TEST(hash_operator_on_select_list) { const char* data = "x:INTEGER,y:INTEGER\n" "1,1\n" "2,2\n" "3,3\n" "4,4\n" "5,5\n" "6,6\n" "7,7\n" "8,8\n" "9,9\n" "10,10\n"; ScratchFile f("hash_operator_on_select_list.odb"); { FileHandle dh(f); dh.openForWrite(0); AutoClose close(dh); odc::api::odbFromCSV(data, dh); } string sql("select x,x#-1,x#1 from \"" + f + "\";"); odc::Select select(sql); odc::Select::iterator it = select.begin(); odc::Select::iterator end = select.end(); for (; it != end; ++it) { Log::info() << it << std::endl; } } /// Shift or hash (#) operator doesn't work in the WHERE clause. /// This test doesn't test anything yet. TEST(hash_operator_in_where) { const char* data = "x:INTEGER,y:INTEGER\n" "1,1\n" "2,2\n" "3,3\n" "4,4\n" "5,5\n" "6,6\n" "7,7\n" "8,8\n" "9,9\n" "10,10\n"; ScratchFile f("hash_operator_in_where.odb"); { FileHandle dh(f); dh.openForWrite(0); AutoClose close(dh); odc::api::odbFromCSV(data, dh); } string sql("select x,x#-1,x#1 from \"" + f + "\" where x=2 and x#1=3;"); odc::Select select(sql); odc::Select::iterator it = select.begin(); odc::Select::iterator end = select.end(); for (; it != end; ++it) { Log::info() << it << std::endl; } } TEST(bitfields_hash_operator) { PathName f("2000010106-reduced.odb"); // odc::Select select("select lat,lat#1 from \"" + f + "\""); odc::Select select("select anflag@body,anflag.final@body,anflag.*@body from \"" + f + "\";"); odc::Select::iterator it = select.begin(); odc::Select::iterator end = select.end(); for (; it != end; ++it) { Log::info() << it << std::endl; } } TEST(select_constant_value) { const char* sql = "set $foo = 27;" "select $foo;"; odc::Select o(sql); odc::Select::iterator it = o.begin(); odc::Select::iterator end = o.end(); unsigned long counter = 0; for (; it != o.end(); ++it, ++counter) { Log::info() << it << std::endl; CHECK_EQUAL(it->data(0), 27); } CHECK_EQUAL(counter, 1); } TEST(include) { ofstream f("stuff.sql"); f //<< "select * from \"file1.odb\";" << endl << "set $foo = 10;" << endl << "set $bar = 20;" << std::endl; f.close(); const char* sql = "#include \"stuff.sql\"\n" "set $baz = $bar;" "select $foo * $bar;"; unsigned long counter = 0; odc::Select o(sql); for (odc::Select::iterator it(o.begin()), end(o.end()); it != o.end(); ++it, ++counter) { Log::info() << it << std::endl; } } TEST(log_error) { Log::error() << "Just a logger test" << std::endl; // TODO: test Log::error writes to stderr // TODO: test Log::error has a prefirx with timestamp and other things } /* TEST(create_table_using_variable) { const char *sql = "SET $c = { 2 : \"foo\" };\n" "SET $s = 2;\n" "CREATE TABLE t AS (c[$s]);" ; unsigned long counter = 0; odc::Select o(sql); for (odc::Select::iterator it(o.begin()), end(o.end()); it != o.end(); ++it, ++counter) { Log::info() << it << std::endl; } } */ TEST(meta_data_reader_checks_if_file_truncated) { ASSERT(0 == std::system("dd if=disp.7.1.odb of=disp.7.1.odb.truncated bs=30121 count=1")); core::TablesReader mdr("disp.7.1.odb.truncated"); try { for (auto it(mdr.begin()), end(mdr.end()); it != end; ++it) ; ASSERT(0 && "Scanning of truncated file did not fail"); } catch (odc::core::ODBIncomplete& ex) { Log::info() << "Scanning of truncated file disp.7.1.odb.truncated failed as expected." << std::endl; } } TEST(operator_ge) { const char* data = "a:INTEGER,b:INTEGER\n" "1,1\n" "2,2\n" "3,3\n" "4,4\n" "5,5\n" "6,6\n" "7,7\n" "8,8\n" "9,9\n" "10,10\n"; { FileHandle dh("1to10.odb"); dh.openForWrite(0); AutoClose close(dh); odc::api::odbFromCSV(data, dh); } odc::Select odb("select a,b from \"1to10.odb\" where a >= 3;"); unsigned long counter = 0; for (odc::Select::iterator it = odb.begin(), end = odb.end(); it != end; ++it, ++counter) ; ASSERT(counter == 8); } static void create_1to10() { const char* data = "a:INTEGER,b:INTEGER\n" "1,1\n" "2,2\n" "3,3\n" "4,4\n" "5,5\n" "6,6\n" "7,7\n" "8,8\n" "9,9\n" "10,10\n"; FileHandle dh("1to10.odb"); dh.openForWrite(0); AutoClose close(dh); odc::api::odbFromCSV(data, dh); } /* FIXME TEST(Select_isNewDataset) { create_1to10(); size_t blocks (0); odc::Select odb("select * from \"1to10.odb\";"); for (odc::Select::iterator it = odb.begin(), end = odb.end(); it != end; ++it) if (it->isNewDataset()) ++blocks; ASSERT(blocks == 1); } */ template static void test_isNewDataset() { create_1to10(); size_t blocks(0); blocks = 0; T odb("1to10.odb"); for (typename T::iterator it = odb.begin(), end = odb.end(); it != end; ++it) if (it->isNewDataset()) ++blocks; ASSERT(blocks == 1); } /* TEST(Gabor) { const char * cfg = "CLASS: class\n" "DATE: andate\n" "TIME: antime\n" "TYPE: type\n" "OBSGROUP: groupid\n" "REPORTYPE: reportype\n" "STREAM: stream\n" "EXPVER: expver\n" ; const string fileName("/tmp/gabor/massaged.odb"); odc::FastODA2Request o2r; o2r.parseConfig(cfg); eckit::OffsetList offsets; eckit::LengthList lengths; vector handles; bool rc = o2r.scanFile(fileName, offsets, lengths, handles); for (size_t i = 0; i < handles.size(); ++i) delete handles[i]; handles.clear(); ASSERT(rc); ASSERT(lengths.size()); ASSERT(lengths.size() == offsets.size()); for(size_t i = 1; i < offsets.size(); i++) ASSERT(offsets[i] > offsets[i-1]); size_t last = offsets.size()-1; ASSERT(PathName(fileName).size() == offsets[last] + lengths[last]); unsigned long long cnt = o2r.rowsNumber(); string filesRequest = "ARCHIVE,\n"; filesRequest += o2r.genRequest(); } // FIXME TEST(Reader_isNewDataset) { test_isNewDataset(); } // FIXME TEST(MetaDataReader_isNewDataset) { test_isNewDataset >(); } TEST(create_temporary_table) { const char* sql = "CREATE " " TEMPORARY " " TABLE foo AS (col1 pk9real, col2 pk9real,) INHERITS (bar,baz);"; cout << "Trying to execute: '" << sql << "'" << std::endl; odc::Select o(sql); odc::tool::SQLTool::execute(sql); } */ TEST(JULIAN_SECONDS) { ASSERT(1 == (*odc::Select("select julian_seconds(19750311,0) < julian_seconds(20140210,0) from dual;").begin())[0]); } TEST(CREATE_TABLE_and_SELECT_INTO) { const char* inputData = R"(a:INTEGER,b:INTEGER 1,1 2,2 3,3 4,4 5,5 6,6 7,7 8,8 9,9 10,10)"; { FileHandle dh("CREATE_TABLE_and_SELECT_INTO.odb"); dh.openForWrite(0); AutoClose close(dh); odc::api::odbFromCSV(inputData, dh); } const char* sql = R"( CREATE TYPE mybitfield AS ( codetype bit9, instype bit10, retrtype bit6, geoarea bit6, ); CREATE TABLE "foo.odb" AS ( lat real, lon real, status mybitfield, ); SELECT a,b,a*b INTO "foo.odb" FROM "CREATE_TABLE_and_SELECT_INTO.odb"; )"; { odc::Select o(sql); odc::Select::iterator it = o.begin(); unsigned long counter = 0; Log::info() << "Inside select" << std::endl; for (; it != o.end(); ++it, ++counter) { Log::info() << "Getting a line..." << std::endl; ; } Log::info() << "CREATE_TABLE_and_SELECT_INTO: counter=" << counter << endl; } std::system("ls -l foo.odb; "); std::system((eckit::PathName("~/bin/odc").asString() + " header foo.odb").c_str()); std::system((eckit::PathName("~/bin/odc").asString() + " ls foo.odb").c_str()); } /* TEST(SELECT_ALL) { ostream& L(eckit::Log::info()); odc::api::odbFromCSV("a:INTEGER,b:INTEGER\n1,2\n", "select_all_1.odb"); odc::api::odbFromCSV("a:INTEGER,b:INTEGER,c:INTEGER\n1,2,3\n", "select_all_2.odb"); std::system("cat select_all_1.odb select_all_2.odb >select_all.odb"); L << "--- Test_SELECT_ALL: open select_all.odb" << endl; odc::Select o("SELECT ALL * FROM \"select_all.odb\";"); odc::Select::iterator it (o.begin()), end (o.end()); L << "--- Test_SELECT_ALL: row #0" << endl; ++it; ASSERT(it->columns().size() == 2); L << "--- Test_SELECT_ALL: row #1" << endl; ++it; ASSERT(it->columns().size() == 3); } */ // ODB-106 TEST(SELECT_WHERE_0) { { FileHandle dh("select_where_0.odb"); dh.openForWrite(0); AutoClose close(dh); odc::api::odbFromCSV("a:INTEGER,b:INTEGER\n1,2\n3,4\n", dh); } odc::Select o("SELECT * FROM \"select_where_0.odb\" WHERE 0;"); odc::Select::iterator it(o.begin()), end(o.end()); ++it; } TEST(QuestionMarkHandlingWhenSplittingByStringColumn_ODB235) { const char* inFile("ODB_235.odb"); const char* data( "a:INTEGER,b:INTEGER,expver:STRING\n" "1,1,'?'\n" "2,2,'?'\n" "3,3,'?'\n"); const char* outFileTemplate("ODB_235_{a}_{expver}.odb"); { FileHandle dh(inFile); dh.openForWrite(0); AutoClose close(dh); odc::api::odbFromCSV(data, dh); } odc::Reader in(inFile); odc::DispatchingWriter out(outFileTemplate, /*maxOpenFiles*/ 3); odc::DispatchingWriter::iterator outIt(out.begin()); outIt->pass1(in.begin(), in.end()); ASSERT(PathName("ODB_235_1_?.odb").exists()); ASSERT(PathName("ODB_235_2_?.odb").exists()); ASSERT(PathName("ODB_235_3_?.odb").exists()); PathName("ODB_235_1_?.odb").unlink(); PathName("ODB_235_2_?.odb").unlink(); PathName("ODB_235_3_?.odb").unlink(); PathName("ODB_235.odb").unlink(); } TEST(LegacyAPIExecuteSelectTwice) { const std::string fn("legacy_execute_select_twice.odb"); { FileHandle dh(fn); dh.openForWrite(0); AutoClose close(dh); odc::api::odbFromCSV("a:INTEGER,b:INTEGER\n1,2\n3,4\n", dh); } odc::Select o(std::string("SELECT * FROM \"") + fn + "\";"); int i(0), j(0); for (odc::Select::iterator it(o.begin()), end(o.end()); it != end; ++it) ++i; ASSERT(i == 2); for (odc::Select::iterator it(o.begin()), end(o.end()); it != end; ++it) ++j; ASSERT(j == 2); } TEST(LegacyAPITraverseReaderTwice) { const std::string fn("legacy_traverse_reader_twice.odb"); { FileHandle dh(fn); dh.openForWrite(0); AutoClose close(dh); odc::api::odbFromCSV("a:INTEGER,b:INTEGER\n1,2\n3,4\n", dh); } odc::Reader o(fn); int i(0), j(0); for (odc::Reader::iterator it(o.begin()), end(o.end()); it != end; ++it) ++i; ASSERT(i == 2); for (odc::Reader::iterator it(o.begin()), end(o.end()); it != end; ++it) ++j; ASSERT(j == 2); } odc-1.6.3/src/odc/tools/SQLTool.cc0000664000175000017500000001251115146027420016753 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include #include #include "eckit/exception/Exceptions.h" #include "eckit/io/FileDescHandle.h" #include "eckit/io/FileHandle.h" #include "eckit/io/Length.h" #include "eckit/io/PartFileHandle.h" #include "eckit/utils/StringTools.h" #include "eckit/sql/SQLParser.h" #include "eckit/sql/SQLSelectFactory.h" #include "eckit/sql/SQLSession.h" #include "eckit/sql/SQLStatement.h" #include "eckit/types/Types.h" #include "odc/sql/SQLOutputConfig.h" #include "odc/sql/TODATable.h" #include "odc/tools/SQLTool.h" using namespace std; using namespace eckit; namespace odc { namespace tool { //---------------------------------------------------------------------------------------------------------------------- SQLTool::SQLTool(int argc, char** argv) : Tool(argc, argv) { registerOptionWithArgument("-o"); registerOptionWithArgument("-i"); registerOptionWithArgument("-I"); registerOptionWithArgument("-delimiter"); registerOptionWithArgument("-f"); // output format registerOptionWithArgument("-offset"); registerOptionWithArgument("-length"); if ((inputFile_ = optionArgument("-i", std::string(""))) == "-") inputFile_ = "/dev/stdin"; offset_ = optionArgument("-offset", (long)0); // FIXME@ optionArgument should accept unsigned long etc length_ = optionArgument("-length", (long)0); // Configure the output bool noColumnNames = optionIsSet("-T"); bool noNULL = optionIsSet("-N"); std::string fieldDelimiter = optionArgument("-delimiter", std::string("\t")); std::string outputFormat = optionArgument("-f", std::string(eckit::sql::SQLOutputConfig::defaultOutputFormat)); bool bitfieldsBinary = optionIsSet("--bin") || optionIsSet("--binary"); // bool bitfieldsHex = optionIsSet("--hex") || optionIsSet("--hexadecimal"); bool noColumnAlignment = optionIsSet("--no_alignment"); bool fullPrecision = optionIsSet("--full_precision") || optionIsSet("--full-precision"); sqlOutputConfig_.reset(new odc::sql::SQLOutputConfig(noColumnNames, noNULL, fieldDelimiter, outputFormat, bitfieldsBinary, noColumnAlignment, fullPrecision)); // Configure the output file std::string outputFile = optionArgument("-o", std::string("")); if (outputFile == "-") outputFile = "/dev/stdout"; if (!outputFile.empty()) { sqlOutputConfig_->setOutputFile(outputFile); } } SQLTool::~SQLTool() {} void SQLTool::run() { if (parameters().size() < 2) { Log::error() << "Usage: "; usage(parameters(0), Log::error()); Log::error() << std::endl; std::stringstream ss; ss << "Expected at least 2 command line parameters"; throw UserError(ss.str()); } if (sqlOutputConfig_->outputFormat() == "odb" && !optionIsSet("-o")) { throw UserError("Output file is required (option -o) for binary output format (option -f odb)"); } std::vector params(parameters()); params.erase(params.begin()); std::string sql(StringTool::isSelectStatement(params[0]) ? StringTools::join(" ", params) + ";" // FIXME: : StringTool::readFile(params[0] == "-" ? "/dev/tty" : params[0]) + ";"); std::unique_ptr outStream; if (optionIsSet("-o") && sqlOutputConfig_->outputFormat() != "odb") { outStream.reset(new std::ofstream(optionArgument("-o", std::string("")).c_str())); sqlOutputConfig_->setOutputStream(*outStream); } // Configure the session to include any specified ODB file eckit::sql::SQLSession session(std::move(sqlOutputConfig_)); // n.b. invalidates sqlOutputConfig_ std::unique_ptr implicitTableDH; std::unique_ptr implicitCloser; if (!inputFile_.empty()) { if (inputFile_ == "/dev/stdin" || inputFile_ == "stdin") { Log::info() << "Reading table from standard input" << std::endl; implicitTableDH.reset(new FileDescHandle(0)); NOTIMP; // Is this working? /// parser.parseString(session, sql, &fh, config); } else if (offset_ == eckit::Offset(0)) { implicitTableDH.reset(new FileHandle(inputFile_)); } else { implicitTableDH.reset(new PartFileHandle(inputFile_, offset_, length_)); } implicitTableDH->openForRead(); implicitCloser.reset(new AutoClose(*implicitTableDH)); eckit::sql::SQLDatabase& db(session.currentDatabase()); db.addImplicitTable(new odc::sql::ODATable(db, *implicitTableDH)); } // And actually do the SQL! eckit::sql::SQLParser parser; parser.parseString(session, sql); session.statement().execute(); } //---------------------------------------------------------------------------------------------------------------------- } // namespace tool } // namespace odc odc-1.6.3/src/odc/tools/TestCommandLineParsing.cc0000664000175000017500000000340115146027420022026 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// \file UnitTest.h /// /// @author Piotr Kuchta, ECMWF, July 2009 #include "eckit/exception/Exceptions.h" #include "TestCase.h" #include "Tool.h" using namespace std; using namespace eckit; using namespace odc; struct TestTool : public odc::tool::Tool { TestTool(int argc, char** argv) : odc::tool::Tool(argc, argv) { registerOptionWithArgument("-foo"); registerOptionWithArgument("-intOpt"); } static void help(std::ostream& o) { o << "No help available for this command yet." << std::endl; } void run() { Log::info() << "test: run" << std::endl; ASSERT(optionArgument("-foo", std::string("NONE")) == "bar"); ASSERT(optionArgument("-intOpt", 0) == 69); ASSERT(optionIsSet("-blah")); ASSERT(optionIsSet("-blahblah")); ASSERT(!optionIsSet("-blahblahblah")); ASSERT(optionIsSet("-lastOption")); ASSERT(parameters().size() == 3); ASSERT(parameters()[0] == "p1"); ASSERT(parameters()[1] == "p2"); ASSERT(parameters()[2] == "p3"); } }; static void test() { const char* args[] = {"-foo", "bar", "-intOpt", "69", "-blah", "-blahblah", "p1", "p2", "-lastOption", "p3", 0}; TestTool testTool(sizeof(args) / sizeof(char*) - 1, const_cast(args)); testTool.run(); } static void setUp() {} static void tearDown() {} SIMPLE_TEST(CommandLineParsing) odc-1.6.3/src/odc/tools/TestOdaCAPI.cc0000664000175000017500000001510715146027420017462 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "TestCase.h" #include "eckit/eckit.h" #include "eckit/exception/Exceptions.h" #include "eckit/log/Log.h" #include "eckit/log/Timer.h" #include "odc/Writer.h" #include "odc/odccapi.h" #include "TestOdaCAPI.h" using namespace std; using namespace eckit; using namespace odc; namespace odc { namespace tool { namespace test { int test_odacapi_setup_in_C(int argc, char* argv[]) { const char* filename = "test.odb"; int err = 0; oda_writer* writer = odb_writer_create("", &err); ASSERT(writer); oda_write_iterator* wi = odb_create_write_iterator(writer, filename, &err); ASSERT(wi); ASSERT(0 == odb_write_iterator_set_no_of_columns(wi, 2)); ASSERT(0 == odb_write_iterator_set_column(wi, 0, odc::api::INTEGER, "ifoo")); ASSERT(0 == odb_write_iterator_set_column(wi, 1, odc::api::REAL, "nbar")); ASSERT(0 == odb_write_iterator_write_header(wi)); double data[2]; for (int i = 1; i <= 10; i++) { data[0] = i; data[1] = i; ASSERT(0 == odb_write_iterator_set_next_row(wi, data, 2)); } ASSERT(0 == odb_write_iterator_destroy(wi)); ASSERT(0 == odb_writer_destroy(writer)); return 0; } int test_odacapi_setup(int argc, char* argv[]) { Timer t("Writing test.odb"); odc::Writer<> oda("test.odb"); odc::Writer<>::iterator writer = oda.begin(); writer->setNumberOfColumns(2); writer->setColumn(0, "ifoo", odc::api::INTEGER); writer->setColumn(1, "nbar", odc::api::REAL); writer->writeHeader(); for (int i = 1; i <= 10; i++) { writer->data()[0] = i; // col 0 writer->data()[1] = i; // col 1 ++writer; } // writer->close(); return 0; } int test_odacapi1(int argc, char* argv[]) { std::cout << "UnitTest odacapi..." << std::endl; int err; std::cout << "Calling oda_create..." << std::endl; oda_ptr oh = odb_read_create("", &err); oda_read_iterator* it = odb_create_read_iterator(oh, "test.odb", &err); ASSERT(0 == err); ASSERT(0 != it); int nCols; ASSERT(0 == odb_read_iterator_get_no_of_columns(it, &nCols)); ASSERT(nCols == 2); int type0; ASSERT(0 == odb_read_iterator_get_column_type(it, 0, &type0)); ASSERT(type0 == 1 /*INTEGER*/); int type1; ASSERT(0 == odb_read_iterator_get_column_type(it, 1, &type1)); ASSERT(type1 == 2 /*REAL*/); char* name0; int name0size; ASSERT(0 == odb_read_iterator_get_column_name(it, 0, &name0, &name0size)); char* name1; int name1size; ASSERT(0 == odb_read_iterator_get_column_name(it, 1, &name1, &name1size)); double buffer[2]; double* data = buffer; int newDataset = 0; int nRows = 0; while (0 == odb_read_iterator_get_next_row(it, 2, data, &newDataset)) { ++nRows; int v0 = int(data[0]); std::cout << "Read row " << nRows << std::endl; ASSERT(v0 == nRows); } ASSERT(0 == odb_read_iterator_destroy(it)); ASSERT(0 == odb_read_destroy(oh)); std::cout << "OK" << std::endl; return 0; } int test_odacapi2(int argc, char* argv[]) { std::cout << "UnitTest odacapi 2..." << std::endl; int err; std::cout << "Calling odb_start_with_args..." << std::endl; odb_start_with_args(argc, argv); std::cout << "Calling odb_create..." << std::endl; oda_ptr oh = odb_select_create("", &err); Log::info() << "Log::info initialised properly." << std::endl; oda_select_iterator* it = odb_create_select_iterator(oh, "select * from \"test.odb\";", &err); ASSERT(0 == err); ASSERT(0 != it); int nCols; ASSERT(0 == odb_select_iterator_get_no_of_columns(it, &nCols)); ASSERT(nCols == 2); int type0; ASSERT(0 == odb_select_iterator_get_column_type(it, 0, &type0)); ASSERT(type0 == 1 /*INTEGER*/); int type1; ASSERT(0 == odb_select_iterator_get_column_type(it, 1, &type1)); ASSERT(type1 == 2 /*REAL*/); char* name0; int name0size; ASSERT(0 == odb_select_iterator_get_column_name(it, 0, &name0, &name0size)); char* name1; int name1size; ASSERT(0 == odb_select_iterator_get_column_name(it, 1, &name1, &name1size)); double buffer[2]; double* data = buffer; int newDataset = 0; int nRows = 0; while (0 == odb_select_iterator_get_next_row(it, 2, data, &newDataset)) { ++nRows; int v0 = int(data[0]); std::cout << "Read row " << nRows << std::endl; ASSERT(v0 == nRows); } ASSERT(0 == odb_select_iterator_destroy(it)); ASSERT(0 == odb_read_destroy(oh)); std::cout << "OK" << std::endl; return 0; } int test_odacapi3(int argc, char* argv[]) { std::cout << "UnitTest ODB C API append to file functionality..." << std::endl; const char* filename = "test.odb"; int err = 0; double n = odb_count(filename); std::cout << "test_odacapi3: number of rows = " << n << std::endl; ASSERT(n == 10); oda_writer* writer = odb_writer_create("", &err); ASSERT(writer); oda_write_iterator* wi = odb_create_append_iterator(writer, filename, &err); ASSERT(wi); ASSERT(0 == err); ASSERT(0 != wi); ASSERT(0 == odb_write_iterator_set_no_of_columns(wi, 2)); ASSERT(0 == odb_write_iterator_set_column(wi, 0, odc::api::INTEGER, "ifoo")); ASSERT(0 == odb_write_iterator_set_column(wi, 1, odc::api::REAL, "nbar")); ASSERT(0 == odb_write_iterator_write_header(wi)); double data[2]; for (int i = 1; i <= 10; i++) { data[0] = i; data[1] = i; ASSERT(0 == odb_write_iterator_set_next_row(wi, data, 2)); } ASSERT(0 == odb_write_iterator_destroy(wi)); ASSERT(0 == odb_writer_destroy(writer)); n = odb_count(filename); std::cout << "test_odacapi3: number of rows = " << n << std::endl; ASSERT(n == 20); return 0; } int test_odacapi(int argc, char* argv[]) { std::cout << "Calling odb_init..." << std::endl; odb_start_with_args(argc, argv); Log::info() << "Log::info initialised properly." << std::endl; // return test_odacapi_setup() return test_odacapi_setup_in_C(argc, argv) || test_odacapi1(argc, argv) || test_odacapi2(argc, argv) || test_odacapi3(argc, argv); } } // namespace test } // namespace tool } // namespace odc odc-1.6.3/src/odc/tools/ToolRunnerApplication.cc0000664000175000017500000000371415146027420021756 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// \file ToolRunnerApplication.h /// /// @author Piotr Kuchta, ECMWF, Feb 2009 #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/PathName.h" #include "odc/tools/Tool.h" #include "odc/tools/ToolFactory.h" #include "odc/tools/ToolRunnerApplication.h" using namespace eckit; namespace odc { namespace tool { ToolRunnerApplication::ToolRunnerApplication(int argc, char** argv, bool createCommandLineTool, bool deleteTool) : ODBApplication(argc, argv), tool_(!createCommandLineTool ? 0 : AbstractToolFactory::createTool(PathName(argv[1]).baseName(), argc - 1, argv + 1)), deleteTool_(deleteTool) {} ToolRunnerApplication::ToolRunnerApplication(odc::tool::Tool& tool, int argc, char** argv) : ODBApplication(argc, argv), tool_(&tool), deleteTool_(false) {} ToolRunnerApplication::~ToolRunnerApplication() { if (deleteTool_) delete tool_; } void ToolRunnerApplication::tool(odc::tool::Tool* tool) { tool_ = tool; } void ToolRunnerApplication::run() { if (tool_ == 0) { std::cerr << name() << ": Unknown command '" << argv(1) << "'" << std::endl; throw eckit::UserError(std::string("Unknown command: ") + argv(1), Here()); } tool_->run(); if (deleteTool_) { delete tool_; tool_ = 0; } } int ToolRunnerApplication::printHelp(std::ostream& out) { if (tool_ == 0) { std::cerr << name() << ": Unknown command '" << argv(1) << "'" << std::endl; return 1; } // tool_->help(out); return 0; } } // namespace tool } // namespace odc odc-1.6.3/src/odc/tools/Examples.cc0000664000175000017500000000474515146027420017246 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// This file contains examples of usage of public APIs. #include #include #include #include "eckit/filesystem/PathName.h" #include "eckit/io/FileHandle.h" #include "odc/api/Odb.h" #include "odc/Reader.h" #include "odc/Select.h" #include "odc/Writer.h" #include "TestCase.h" using namespace eckit; namespace { TEST(example_select_data_read_results) { // Prepare input data const std::string data = R"(x:INTEGER,y:INTEGER,v:DOUBLE 1,1,0.3 1,1,0.2 2,2,0.4 2,2,0.1)"; FileHandle out("example_select_data_read_results.odb"); out.openForWrite(0); AutoClose close(out); odc::api::odbFromCSV(data, out); odc::Select select("select x,min(v),max(v);", "example_select_data_read_results.odb"); for (odc::Select::iterator it(select.begin()), end(select.end()); it != end; ++it) { double r0 = (*it)[0], r1 = (*it)[1], r2 = (*it)[2]; std::cout << r0 << ", " << r1 << ", " << r2 << std::endl; } } TEST(example_read_data) { // Prepare input data const std::string data = "x:INTEGER,y:INTEGER,v:DOUBLE\n" "1,1,0.3\n" "1,1,0.2\n" "2,2,0.4\n" "2,2,0.1\n"; FileHandle out("example_read_data.odb"); out.openForWrite(0); AutoClose close(out); odc::api::odbFromCSV(data, out); odc::Reader o("example_read_data.odb"); for (odc::Reader::iterator it(o.begin()), end(o.end()); it != end; ++it) { double r0 = (*it)[0], r1 = (*it)[1], r2 = (*it)[2]; std::cout << r0 << ", " << r1 << ", " << r2 << std::endl; } } TEST(example_write_data) { odc::core::MetaData metaData; metaData.addColumn("x", "INTEGER").addColumn("y", "INTEGER").addColumn("v", "DOUBLE"); odc::Writer<> writer("example_write_data.odb"); odc::Writer<>::iterator it(writer.begin()); it->columns(metaData); it->writeHeader(); for (size_t i(1); i <= 1000; ++i) { (*it)[0] = i; (*it)[1] = i * 2; (*it)[2] = i * 3; // Incrementing iterator moves coursor to the next row. ++it; } } } // namespace odc-1.6.3/src/odc/tools/ODAHeaderTool.h0000664000175000017500000000206515146027420017675 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef HeaderTool_H #define HeaderTool_H #include "Tool.h" namespace odc { namespace tool { class HeaderTool : public Tool { public: HeaderTool(int argc, char* argv[]); void run(); static void help(std::ostream& o) { o << "Shows header(s) and metadata(s) of file"; } static void usage(const std::string& name, std::ostream& o) { o << name << " [-offsets] [-ddl] [-table ] "; } private: // No copy allowed HeaderTool(const HeaderTool&); HeaderTool& operator=(const HeaderTool&); std::string readFile(const std::string& fileName); }; } // namespace tool } // namespace odc #endif odc-1.6.3/src/odc/tools/Makefile0000664000175000017500000000001015146027420016576 0ustar alastairalastairall: m odc-1.6.3/src/odc/tools/SetTool.h0000664000175000017500000000207215146027420016712 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef SetTool_H #define SetTool_H #include "Tool.h" namespace odc { namespace tool { class SetTool : public Tool { public: SetTool(int argc, char* argv[]); void run(); static void help(std::ostream& o) { o << "Creates a new file setting columns to given values"; } static void usage(const std::string& name, std::ostream& o) { o << name << " "; } private: // No copy allowed SetTool(const SetTool&); SetTool& operator=(const SetTool&); void parseUpdateList(std::string s, std::vector& columns, std::vector& values); }; } // namespace tool } // namespace odc #endif odc-1.6.3/src/odc/tools/FixedSizeRowTool.h0000664000175000017500000000246415146027420020546 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include "odc/tools/Tool.h" #include namespace odc::tool { //---------------------------------------------------------------------------------------------------------------------- class FixedSizeRowTool : public Tool { public: FixedSizeRowTool(int argc, char* argv[]); void run() override; static void help(std::ostream& o) { o << "Converts file to a format with fixed size rows"; } static void usage(const std::string& name, std::ostream& o) { o << name << " "; } private: // No copy allowed FixedSizeRowTool(const FixedSizeRowTool&); FixedSizeRowTool& operator=(const FixedSizeRowTool&); }; template <> struct ExperimentalTool { enum { experimental = true }; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace odc::tool odc-1.6.3/src/odc/tools/TestAggregateFunctions2.cc0000664000175000017500000000337415146027420022166 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// \file TestAggregateFunctions2.h /// /// @author Piotr Kuchta, ECMWF, September 2010 #include "eckit/io/FileHandle.h" #include "odc/Select.h" #include "odc/Writer.h" #include "odc/api/Odb.h" #include "TestCase.h" using namespace std; using namespace eckit; using namespace odc; static void test() { string sql = "select count(*) from \"TestAggregateFunctions2.odb\";"; Log::info() << "Executing: '" << sql << "'" << std::endl; odc::Select oda(sql); odc::Select::iterator it = oda.begin(); ASSERT(it->columns().size() == 1); ASSERT((*it)[0] == 10); odc::Select sel(sql); odc::Select::iterator it2 = sel.begin(); odc::Select::iterator end2 = sel.end(); FileHandle fhout("TestAggregateFunctions2.odb"); fhout.openForWrite(0); AutoClose closer(fhout); odc::Writer<> writer(fhout); odc::Writer<>::iterator outit = writer.begin(); // outit->pass1(it2, end2); size_t i = 0; for (; it2 != end2; ++it2) { ++i; ASSERT((*it2)[0] == 10); } ASSERT(i == 1); } static void setUp() { stringstream s; s << "a:REAL" << std::endl; for (size_t i = 1; i <= 10; ++i) s << i << std::endl; FileHandle dh("TestAggregateFunctions2.odb"); dh.openForWrite(0); AutoClose close(dh); odc::api::odbFromCSV(s, dh); } static void tearDown() {} SIMPLE_TEST(AggregateFunctions2) odc-1.6.3/src/odc/tools/CompactTool.h0000664000175000017500000000237315146027420017551 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include "odc/tools/Tool.h" #include namespace odc::tool { //---------------------------------------------------------------------------------------------------------------------- class CompactTool : public Tool { public: CompactTool(int argc, char* argv[]); void run() override; static void help(std::ostream& o) { o << "Tries to compress a file"; } static void usage(const std::string& name, std::ostream& o) { o << name << " "; } private: // No copy allowed CompactTool(const CompactTool&); CompactTool& operator=(const CompactTool&); }; template <> struct ExperimentalTool { enum { experimental = true }; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace odc::tool odc-1.6.3/src/odc/tools/TestFunctionDotp.cc0000664000175000017500000000272615146027420020741 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// \file UnitTest.cc /// /// @author ECMWF, July 2010 const double EPS = 4e-5; #include #include "eckit/log/Timer.h" #include "odc/Select.h" #include "TestCase.h" #include "odc/Writer.h" using namespace std; using namespace eckit; using namespace odc; static void setUp() { Timer t("Test Dotp function"); odc::Writer<> oda("test_dotp.odb"); odc::Writer<>::iterator row = oda.begin(); row->setNumberOfColumns(2); row->setColumn(0, "x", odc::api::REAL); row->setColumn(1, "y", odc::api::REAL); row->writeHeader(); (*row)[0] = 3.0; (*row)[1] = 2.0; ++row; (*row)[0] = 7.5; (*row)[1] = 112.0; ++row; (*row)[0] = 93.7; (*row)[1] = 12.3; ++row; } static void tearDown() { PathName("test_dotp.odb").unlink(); } static void test() { const string sql = "select dotp(x,y) from \"test_dotp.odb\";"; Log::info() << "Executing: '" << sql << "'" << std::endl; odc::Select oda(sql); odc::Select::iterator it = oda.begin(); ASSERT(fabs((*it)[0] - 1998.51e0) < EPS); // } SIMPLE_TEST(FunctionDotp) odc-1.6.3/src/odc/tools/TestOrderBy.cc0000664000175000017500000000630015146027420017663 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// \file UnitTest.h /// /// @author Piotr Kuchta, ECMWF, September 2010 #include "eckit/filesystem/PathName.h" #include "eckit/io/FileHandle.h" #include "eckit/utils/StringTools.h" #include "odc/Select.h" #include "odc/api/Odb.h" #include "odc/core/MetaData.h" #include "TestCase.h" using namespace std; using namespace eckit; using namespace odc; /// static void test() { { string sql = "select distinct a from \"TestOrderBy_a1to10twice.odb\" order by a;"; odc::Select sel(sql); odc::Select::iterator it = sel.begin(); odc::Select::iterator end = sel.end(); int i = 0; for (; it != end; ++it) { int v = (*it)[0]; ASSERT(v == ++i); } ASSERT(i == 10); } { string sql = "select a from \"TestOrderBy_a1to10twice.odb\" order by a;"; odc::Select sel(sql); odc::Select::iterator it = sel.begin(); odc::Select::iterator end = sel.end(); int i = 0, j = 0; for (; it != end; ++it, ++j) { int v = (*it)[0]; ASSERT(i <= v); i = v; } ASSERT(i == 10); ASSERT(j == 20); } { string sql = "select distinct a from \"TestOrderBy_a1to10twice.odb\" order by a desc;"; odc::Select sel(sql); odc::Select::iterator it = sel.begin(); odc::Select::iterator end = sel.end(); int i = 10, j = 0; for (; it != end; ++it, ++j) { int v = (*it)[0]; ASSERT(i-- == v); } ASSERT(i == 0); ASSERT(j == 10); } { const char* in = "a:REAL,b:REAL,c:STRING\n" "1,10,'one'\n" "1,20,'two'\n" "2,30,'three'\n" "2,40,'four'\n"; { FileHandle dh("TestOrderBy.odb"); dh.openForWrite(0); AutoClose close(dh); odc::api::odbFromCSV(in, dh); } string sql = "select distinct a,b,c from \"TestOrderBy.odb\" order by a desc, b asc;"; odc::Select sel(sql); odc::Select::iterator it = sel.begin(); odc::Select::iterator end = sel.end(); int i = 0, v1 = 0, v2 = 0; string s; for (; it != end; ++it, ++i) { v1 = (*it)[0]; v2 = (*it)[1]; s = (*it).string(2); } ASSERT(i == 4); ASSERT(v1 == 1 && v2 == 20 && StringTools::trim(s) == "two"); } } static void setUp() { stringstream s; s << "a:REAL" << std::endl; for (size_t i = 1; i <= 10; ++i) s << i << std::endl; for (size_t i = 1; i <= 10; ++i) s << i << std::endl; FileHandle dh("TestOrderBy_a1to10twice.odb"); dh.openForWrite(0); AutoClose close(dh); odc::api::odbFromCSV(s, dh); } static void tearDown() {} SIMPLE_TEST(OrderBy) odc-1.6.3/src/odc/tools/IndexTool.h0000664000175000017500000000214015146027420017222 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Piotr Kuchta, ECMWF, Oct 2015 #ifndef IndexTool_H #define IndexTool_H #include "eckit/io/Length.h" #include "eckit/io/Offset.h" #include "odc/tools/Tool.h" #include #include namespace odc { namespace tool { typedef std::vector > BlockOffsets; typedef unsigned long long ullong; class IndexTool : public Tool { public: IndexTool(int argc, char* argv[]); void run(); static void help(std::ostream& o); static void usage(const std::string& name, std::ostream& o); private: // No copy allowed IndexTool(const IndexTool&); IndexTool& operator=(const IndexTool&); }; } // namespace tool } // namespace odc #endif odc-1.6.3/src/odc/tools/TestAggregateFunctions.cc0000664000175000017500000000502115146027420022073 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// \file UnitTest.h /// /// @author Piotr Kuchta, ECMWF, Feb 2009 #include "eckit/exception/Exceptions.h" #include "eckit/log/Log.h" #include "odc/Select.h" #include "odc/core/MetaData.h" #include "TestCase.h" using namespace std; using namespace eckit; using namespace odc; static void test() { string sql = "SELECT " " count(*)," " count(lat)," " sum(blacklist.fg_depar@body) as sumfg_depar," " sum((blacklist.fg_depar@body) * (blacklist.fg_depar@body)) as s2umfg_depar," " min(blacklist.fg_depar@body) as minfg_depar," " max(blacklist.fg_depar@body) as maxfg_depar, " " sum(biascorr@body) as sumbiascorr," " sum((biascorr@body) * (biascorr@body)) as s2umbiascorr," " min(biascorr@body) as minbiascorr," " max(biascorr@body) as maxbiascorr," " sum(blacklist.fg_depar@body + biascorr@body) as sumfgdp_unc," " sum((blacklist.fg_depar@body + biascorr@body) * (blacklist.fg_depar@body + biascorr@body)) as s2umfgdp_unc," " min(blacklist.fg_depar@body + biascorr@body) as minfgdp_unc," " max(blacklist.fg_depar@body + biascorr@body) as maxfgdp_unc,\n" "--obstype@hdr as obstype, varno@body as varno, status@body as status, lldegrees(lat@hdr)<=-20 as latbin0, " "lldegrees(lat@hdr)<=20 AND lldegrees(lat@hdr)>-20 as latbin1, lldegrees(lat@hdr)>20 as latbin2 \n" "FROM \"2000010106-reduced.odb\" " "WHERE (biascorr@body is not NULL and biascorr@body <> 0)" " AND not((obstype@hdr == 10 and obschar.codetype@hdr == 250))" " AND (obstype@hdr in (1,4,8,9) or (obstype@hdr == 7 and (obschar.codetype@hdr == 215 or " "obschar.codetype@hdr == 206)));"; Log::info() << "Executing: '" << sql << "'" << std::endl; odc::Select oda(sql); odc::Select::iterator it = oda.begin(); // Log::info() << "it->columns().size() => " << it->columns().size() << std::endl; ASSERT(it->columns().size() == 14); ASSERT((*it)[0] == 805); // COUNT(*) == 805 ASSERT((*it)[1] == 805); // COUNT(lat) == 805 } static void setUp() {} static void tearDown() {} SIMPLE_TEST(AggregateFunctions) odc-1.6.3/src/odc/tools/CAPIExamples.cc0000664000175000017500000001440615146027420017676 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// \file CAPIExamples.cc /// /// This file contains examples of usage of public APIs. /// /// @author Piotr Kuchta, ECMWF, September 2015 #include #include #include #include #include #include "eckit/exception/Exceptions.h" #include "eckit/io/FileHandle.h" #include "odc/api/odc.h" #include "odc/odccapi.h" #include "TestCase.h" namespace { // TODO: Migrate this test over!!! #if 0 // DISABLED TEST(c_api_example_select_data_read_results) { const int maxCols = 4; int err; double buffer[maxCols]; double* data = buffer; int newDataset = 0; int nRows = 0; oda_ptr oh; oda_select_iterator* it; size_t i; // Prepare input data long size = 0; void* buf = 0; ASSERT(false); //odc_import_encode_text( // "x:INTEGER,y:INTEGER,v:DOUBLE,f:BITFIELD[a:1;b:2]]\n" // "1,1,0.3,0\n" // "1,1,0.2,1\n" // "2,2,0.4,2\n" // "2,2,0.1,3\n", ",", 0, &size); eckit::FileHandle dh("c_example_select_data_read_results.odb"); dh.openForWrite(size); ASSERT(dh.write(buf, size) == size); dh.close(); free(buf); oh = odb_select_create("", &err); ASSERT(0 == err); it = odb_create_select_iterator(oh, "select x, min(v), max(v)" " from \"c_example_select_data_read_results.odb\"" " order by x;", &err); ASSERT(0 == err); for (i = 0; 0 == odb_select_iterator_get_next_row(it, 3, data, &newDataset); ++i) { switch (i) { case 0: ASSERT(data[0] == 1 && fabs(data[1] - 0.2) < 0.0000001); break; case 1: ASSERT(data[0] == 2 && fabs(data[1] - 0.1) < 0.0000001); break; } } ASSERT(i == 2); ASSERT(0 == odb_select_iterator_destroy(it)); // Select some bitfields it = odb_create_select_iterator(oh, "select f, f.a, f.b from \"c_example_select_data_read_results.odb\";", &err); ASSERT(0 == err); for (i = 0; 0 == odb_select_iterator_get_next_row(it, 3, data, &newDataset); ++i) { double f = data[0], a = data[1], b = data[2]; switch (i) { case 0: ASSERT(f == 0 && a == 0 && b == 0); break; case 1: ASSERT(f == 1 && a == 1 && b == 0); break; case 2: ASSERT(f == 2 && a == 0 && b == 1); break; case 3: ASSERT(f == 3 && a == 1 && b == 1); break; } } ASSERT(i == 4); ASSERT(0 == odb_select_iterator_destroy(it)); ASSERT(0 == odb_select_destroy(oh)); } TEST(c_api_example_read_data) { const int numberOfColumns = 4; int err, nCols, type, nameLength; char *name; oda_ptr oh; oda_read_iterator* it; double buffer[numberOfColumns]; double* data; int newDataset; int nRows; // Prepare input data long size = 0; void* buf = 0; ASSERT(false); //odc_import_encode_text( // "x:INTEGER,y:REAL,v:DOUBLE,f:BITFIELD[a:1;b:2]\n" // "1,1,0.3,0\n" // "2,1,0.2,1\n" // "3,2,0.4,2\n" // "4,2,0.1,3\n", ",", 0, &size); eckit::FileHandle dh("c_api_example_read_data.odb"); dh.openForWrite(size); ASSERT(dh.write(buf, size) == size); dh.close(); free(buf); oh = odb_read_create("", &err); it = odb_create_read_iterator(oh, "c_api_example_read_data.odb", &err); ASSERT(0 == err); ASSERT(0 != it); odb_read_iterator_get_no_of_columns(it, &nCols); ASSERT(nCols == numberOfColumns); ASSERT(0 == odb_read_iterator_get_column_type(it, 0, &type)); ASSERT(type == 1 /*INTEGER*/); ASSERT(0 == odb_read_iterator_get_column_type(it, 1, &type)); ASSERT(type == 2 /*REAL*/); ASSERT(0 == odb_read_iterator_get_column_type(it, 2, &type)); ASSERT(type == 5 /*DOUBLE*/); ASSERT(0 == odb_read_iterator_get_column_type(it, 3, &type)); ASSERT(type == 4 /*BITFIELD*/); ASSERT(0 == odb_read_iterator_get_column_name(it, 0, &name, &nameLength)); ASSERT(0 == strncmp("x", name, strlen("x"))); ASSERT(0 == odb_read_iterator_get_column_name(it, 1, &name, &nameLength)); ASSERT(0 == strncmp("y", name, strlen("y"))); ASSERT(0 == odb_read_iterator_get_column_name(it, 2, &name, &nameLength)); ASSERT(0 == strncmp("v", name, strlen("v"))); ASSERT(0 == odb_read_iterator_get_column_name(it, 3, &name, &nameLength)); ASSERT(0 == strncmp("f", name, strlen("f"))); data = buffer; newDataset = 0; for (nRows = 0; 0 == odb_read_iterator_get_next_row(it, numberOfColumns, data, &newDataset); ++nRows) { int x = int(data[0]); int f = int(data[3]); std::cout << "Read row " << nRows << ", x=" << x << std::endl; ASSERT(x == nRows + 1 && f == nRows); } ASSERT(nRows == 4); ASSERT(0 == odb_read_iterator_destroy(it)); ASSERT(0 == odb_read_destroy(oh)); } #endif TEST(c_api_example_write_data) { int err; oda_writer* writer = odb_writer_create("", &err); oda_write_iterator* wi = odb_create_write_iterator(writer, "c_api_example_write_data.odb", &err); ASSERT(0 == odb_write_iterator_set_no_of_columns(wi, 4)); ASSERT(0 == odb_write_iterator_set_column(wi, 0, odc::api::INTEGER, "x")); ASSERT(0 == odb_write_iterator_set_column(wi, 1, odc::api::REAL, "y")); ASSERT(0 == odb_write_iterator_set_column(wi, 2, odc::api::DOUBLE, "v")); // Define three fields: a (1 bit only), b (2 bits), c (1 bit) ASSERT(0 == odb_write_iterator_set_bitfield(wi, 3, odc::api::BITFIELD, "bf", "a:b:c", "1:2:1")); ASSERT(0 == odb_write_iterator_write_header(wi)); double data[4]; for (int i = 1; i <= 10; ++i) { data[0] = i; data[1] = i * 10; data[2] = i * 100; data[3] = i; ASSERT(0 == odb_write_iterator_set_next_row(wi, data, 4)); } ASSERT(0 == odb_write_iterator_destroy(wi)); ASSERT(0 == odb_writer_destroy(writer)); } } // namespace odc-1.6.3/src/odc/tools/MockReader.h0000664000175000017500000000150215146027420017332 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef MockReader_H #define MockReader_H #include "odc/IteratorProxy.h" namespace odc { namespace tool { template class MockReader { public: typedef T iterator_class; typedef odc::IteratorProxy iterator; iterator begin() { return iterator(new T); } const iterator end() { return iterator(0); } }; #include "MockReader.cc" } // namespace tool } // namespace odc #endif odc-1.6.3/src/odc/tools/TestCase.h0000664000175000017500000001126315146027420017036 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Piotr Kuchta /// @date Feb 2009 #pragma once #include "Tool.h" #include "ToolFactory.h" #include namespace odc::tool::test { //---------------------------------------------------------------------------------------------------------------------- class TestCase : public Tool { public: static void help(std::ostream& o) { o << "No help available for this command yet"; } static void usage(const std::string& name, std::ostream& o) { o << name << ": Not implemented yet"; } virtual void run(); virtual void setUp(); virtual void test(); virtual void tearDown(); virtual ~TestCase(); protected: TestCase(int argc, char** argv); }; typedef std::vector TestCases; #define TESTCASE(F) \ struct Test_##F : public TestCase { \ Test_##F(int argc, char** argv) : TestCase(argc, argv) {} \ void test() { \ F(); \ } \ }; \ ToolFactory test_##F(std::string("Test_") + #F); #define TEST_FIXTURE(F, T) \ struct Test_##F##_##T : public F, public odc::tool::test::TestCase { \ Test_##F##_##T(int argc, char** argv) : F(), odc::tool::test::TestCase(argc, argv) {} \ void test(); \ }; \ odc::tool::ToolFactory test_##F##_##T(std::string("Test_") + #F + std::string("_") + #T); \ void Test_##F##_##T::test() #define TEST(T) \ struct Test_##T : public odc::tool::test::TestCase { \ Test_##T(int argc, char** argv) : odc::tool::test::TestCase(argc, argv) {} \ void test(); \ }; \ odc::tool::ToolFactory test_##T(std::string("Test_") + #T); \ void Test_##T::test() #define SIMPLE_TEST(name) \ struct Test_##name : public odc::tool::test::TestCase { \ Test_##name(int argc, char** argv) : odc::tool::test::TestCase(argc, argv) {} \ void setUp() { \ ::setUp(); \ } \ void tearDown() { \ ::tearDown(); \ } \ void test() { \ ::test(); \ } \ }; \ odc::tool::ToolFactory test_##name(std::string("Test_") + #name); #define CHECK(expected) ASSERT(expected) #define CHECK_EQUAL(expected, actual) ASSERT((expected) == (actual)) template bool CheckArrayEqual(const Expected& expected, const Actual& actual, const int count) { bool equal = true; for (int i = 0; i < count; ++i) equal &= (expected[i] == actual[i]); return equal; } #define CHECK_ARRAY_EQUAL(expected, actual, count) ASSERT(odc::tool::test::CheckArrayEqual(expected, actual, count)) //---------------------------------------------------------------------------------------------------------------------- } // namespace odc::tool::test odc-1.6.3/src/odc/tools/MergeTool.cc0000664000175000017500000001232415146027420017355 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "odc/tools/MergeTool.h" #include "eckit/exception/Exceptions.h" #include "eckit/io/FileHandle.h" #include "eckit/log/Timer.h" #include "odc/Reader.h" #include "odc/Select.h" #include "odc/Writer.h" using namespace eckit; using namespace odc::core; namespace odc { namespace tool { void MergeTool::help(std::ostream& o) { o << "Merges rows from files"; } void MergeTool::usage(const std::string& name, std::ostream& o) { o << name << " -o ..." << std::endl << "" << std::endl << "\t or " << std::endl << "" << std::endl << name << "\t -S -o ..." << std::endl; } MergeTool::MergeTool(int ac, char* av[]) : Tool(ac, av), inputFiles_(), sql_(), outputFile_(), sqlFiltering_(false) { registerOptionWithArgument("-o"); if (parameters().size() < 3) { Log::error() << "Usage: "; usage(parameters(0), Log::error()); Log::error() << std::endl; std::stringstream ss; ss << "Expected at least 3 command line parameters"; throw UserError(ss.str()); } sqlFiltering_ = optionIsSet("-S"); std::string o(optionArgument("-o", std::string(""))); if (o == "") UserError("Output file is obligatory (option -o)"); outputFile_ = o; for (size_t i = 1; i < parameters().size(); ++i) { inputFiles_.push_back(PathName(parameters()[i])); if (sqlFiltering_) { std::string s(parameters()[++i]); sql_.push_back(StringTool::isSelectStatement(s) ? s : StringTool::readFile(s)); } } } void MergeTool::run() { if (inputFiles_.size() == 0) return; std::stringstream s; for (size_t i = 0; i < inputFiles_.size(); ++i) s << inputFiles_[i] << ","; Timer t(std::string("Merging files '") + s.str() + "' into '" + outputFile_ + "'"); if (!sqlFiltering_) merge(inputFiles_, outputFile_); else merge(inputFiles_, sql_, outputFile_); } template void doMerge(std::vector>& iterators, const PathName& outputFile) { odc::Writer<> writer(outputFile); odc::Writer<>::iterator out(writer.begin()); for (size_t i = 0; i < iterators.size(); ++i) { MetaData columns(iterators[i].first->columns()); for (size_t i = 0; i < columns.size(); ++i) if (out->columns().hasColumn(columns[i]->name())) throw eckit::UserError(std::string("Column '") + columns[i]->name() + "' occurs in more than one input file of merge."); MetaData md(out->columns()); md += columns; out->columns(md); // out->columns() += columns; } out->writeHeader(); Log::info() << "MergeTool::merge: output metadata: " << out->columns() << std::endl; for (;;) { for (size_t i = 0, ii = 0; ii < iterators.size(); ++ii) { I& in(iterators[ii].first); I& inEnd(iterators[ii].second); if (!(in != inEnd)) return (void)(Log::info() << "Input file number " << ii << " ended." << std::endl); for (size_t cn = 0; cn < in->columns().size(); ++cn) { ASSERT(i < out->columns().size()); out->data(i++) = (*in)[cn]; } ++in; } ++out; } } void MergeTool::merge(const std::vector& inputFiles, const PathName& outputFile) { typedef odc::Reader R; typedef R::iterator I; std::vector> readers; std::vector> iterators; for (size_t i = 0; i < inputFiles.size(); ++i) { readers.push_back(std::unique_ptr(new odc::Reader(inputFiles[i]))); iterators.push_back(std::make_pair(readers[i]->begin(), readers[i]->end())); } doMerge(iterators, outputFile); } void MergeTool::merge(const std::vector& inputFiles, const std::vector& sqls, const PathName& outputFile) { typedef odc::Select S; std::vector> fhs; std::vector> closers; std::vector> readers; std::vector> iterators; for (size_t i = 0; i < inputFiles.size(); ++i) { fhs.push_back(std::unique_ptr(new FileHandle(inputFiles[i]))); fhs.back()->openForRead(); closers.push_back(std::unique_ptr(new AutoClose(*fhs.back()))); readers.push_back(std::unique_ptr(new S(sqls[i], *fhs[i]))); iterators.push_back(std::make_pair(readers[i]->begin(), readers[i]->end())); } doMerge(iterators, outputFile); } } // namespace tool } // namespace odc odc-1.6.3/src/odc/ODAHandle.cc0000664000175000017500000000233315146027420016036 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/log/Log.h" #include "eckit/types/Types.h" #include "odc/LibOdc.h" #include "odc/ODAHandle.h" using namespace eckit; namespace odc { ODAHandle::ODAHandle(Offset start, Offset end) : start_(start), end_(end) { LOG_DEBUG_LIB(LibOdc) << "ODAHandle::ODAHandle(" << start << ", " << end << ")" << std::endl; } void ODAHandle::print(std::ostream& o) const { o << "[start:" << start_ << ", end_:" << end_ << ", values_:" /*<< values_ <<*/ "]"; } ODAHandle::~ODAHandle() { LOG_DEBUG_LIB(LibOdc) << "ODAHandle::~ODAHandle()" << std::endl; } void ODAHandle::addValue(const std::string& columnName, double v) { LOG_DEBUG_LIB(LibOdc) << "ODAHandle::addValue('" << columnName << "', '" << v << "')" << std::endl; ASSERT(values_.find(columnName) == values_.end()); values_[columnName] = v; } } // namespace odc odc-1.6.3/src/odc/Select.h0000664000175000017500000000453515146027420015406 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Piotr Kuchta /// @author /// @date April 2010 #ifndef odc_Select_H #define odc_Select_H #ifdef SWIGPYTHON #include #endif #include "eckit/sql/SQLSession.h" #include "odc/IteratorProxy.h" #include "odc/SelectIterator.h" namespace eckit { class PathName; } namespace eckit { class DataHandle; } namespace odc { //---------------------------------------------------------------------------------------------------------------------- class Select { public: typedef IteratorProxy iterator; typedef iterator::Row row; Select(const std::string& selectStatement = "", bool manageOwnBuffer = true); Select(const std::string& selectStatement, eckit::DataHandle& dh, bool manageOwnBuffer = true); Select(const std::string& selectStatement, const eckit::PathName& path, bool manageOwnBuffer = true); // This only exists to disambiguate const char* --> std::string rather than to bool. Select(const std::string& selectStatement, const char* path, bool manageOwnBuffer = true); ~Select() noexcept(false); #ifdef SWIGPYTHON iterator __iter__() { return iterator(createSelectIterator(selectStatement_)); } #endif eckit::sql::SQLDatabase& database(); iterator begin(); const iterator end(); SelectIterator* createSelectIterator(const std::string&); private: friend class odc::IteratorProxy; std::unique_ptr ownDH_; // std::istream* istream_; std::string selectStatement_; std::string delimiter_; eckit::sql::SQLSession session_; // This is horrible, but the TextReader, and any stream based iteraton, can only // iterate once, so we MUST NOT create two iterators if begin() is called twice. bool initted_; iterator it_; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace odc #endif odc-1.6.3/src/odc/WriterDispatchingIterator.cc0000664000175000017500000004660015146027420021470 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/log/Log.h" #include "eckit/log/Timer.h" #include "eckit/utils/Translator.h" #include "odc/Comparator.h" #include "odc/LibOdc.h" #include "odc/Reader.h" #include "odc/WriterDispatchingIterator.h" using namespace odc::core; namespace odc { //---------------------------------------------------------------------------------------------------------------------- template WriterDispatchingIterator::WriterDispatchingIterator(OWNER& owner, int maxOpenFiles, bool append) : owner_(owner), iteratorsOwner_(), columns_(0), lastValues_(0), nextRow_(0), columnOffsets_(0), nrows_(0), outputFileTemplate_(owner_.outputFileTemplate()), properties_(), dispatchedIndexes_(), values2iteratorIndex_(), lastDispatch_(maxOpenFiles, -1), iteratorIndex2fileName_(maxOpenFiles), lastDispatchedValues_(), lastIndex_(), initialized_(false), append_(append), refCount_(0), iterators_(), files_(), templateParameters_(), maxOpenFiles_(maxOpenFiles), filesCreated_() {} template void WriterDispatchingIterator::setNumberOfColumns(size_t n) { columns_.setSize(n); } template int WriterDispatchingIterator::setColumn(size_t index, std::string name, api::ColumnType type) { ASSERT(index < columns().size()); Column* col = columns_[index]; ASSERT(col); col->name(name); col->type(type); return 0; } template int WriterDispatchingIterator::setBitfieldColumn(size_t index, std::string name, api::ColumnType type, eckit::sql::BitfieldDef b) { ASSERT(index < columns().size()); Column* col = columns_[index]; ASSERT(col); col->name(name); col->type(type); col->bitfieldDef(b); return 0; } template std::string WriterDispatchingIterator::generateFileName(const double* values, unsigned long count) { // n.b. Duplicated from eckit::sql::expression::function::FunctionEQ::trimStringInDouble. // Provided as static lambda because this .cc file is included in the header and even when the // function is declared in an annonymos workspace the declaration is marked as unneeded and emits // a warning. // TODO: Put somewhere better. static const auto trimStringInDouble = [](char*& p, size_t& len) { len = 0; for (; len < sizeof(double) && isprint(p[len]); ++len) ; for (; len > 0 && isspace(p[len - 1]); --len) ; size_t plen = len; for (char* pp = p; isspace(*p) && p < pp + plen;) { ++p; --len; } }; std::string fileName(outputFileTemplate_); int diff(0); for (TemplateParameters::iterator it(templateParameters_.begin()); it != templateParameters_.end(); ++it) { TemplateParameter& p(*(*it)); // TODO: if values collected can be of different type then integer, // then below code must be updated: // code updated for std::string [19/07/2011] AF double d(values[p.columnIndex]); std::string s; if (columns_[p.columnIndex]->type() == api::STRING) { char* sp(reinterpret_cast(&d)); size_t len(0); trimStringInDouble(sp, len); s = std::string(sp, len); while (s.find("/") != std::string::npos) { std::string old(s); size_t pos(s.find("/")); s.replace(pos, pos + 1, std::string("__SLASH__")); // eckit::Log::info() << "WriterDispatchingIterator::generateFileName: '" << old << "' => '" << s << "'" // << std::endl; } } else { s = eckit::Translator()(int(d)); } fileName.replace(p.startPos - diff, p.endPos - p.startPos + 1, s); diff = outputFileTemplate_.size() - fileName.size(); } // LOG_DEBUG_LIB(LibOdc) << "WriterDispatchingIterator::generateFileName: fileName = " << fileName << std::endl; return fileName; } template WRITE_ITERATOR& WriterDispatchingIterator::dispatch(const double* values, unsigned long count) { return *iterators_[this->dispatchIndex(values, count)]; } template int WriterDispatchingIterator::dispatchIndex(const double* values, unsigned long count) { Values dispatchedValues; for (size_t i(0); i < dispatchedIndexes_.size(); ++i) dispatchedValues.push_back(values[dispatchedIndexes_[i]]); if (dispatchedValues == lastDispatchedValues_) return lastIndex_; Values2IteratorIndex::iterator p(values2iteratorIndex_.find(dispatchedValues)); size_t iteratorIndex((p != values2iteratorIndex_.end()) ? p->second : createIterator(dispatchedValues, generateFileName(values, count))); lastDispatchedValues_ = dispatchedValues; lastIndex_ = iteratorIndex; lastDispatch_[iteratorIndex] = nrows_; return iteratorIndex; } template int WriterDispatchingIterator::createIterator(const Values& dispatchedValues, const std::string& fileName) { int iteratorIndex(iterators_.size()); if (iterators_.size() >= maxOpenFiles_) { ASSERT(iterators_.size()); size_t oldest(0); unsigned long long oldestRow(lastDispatch_[oldest]); for (size_t i = oldest; i < lastDispatch_.size(); ++i) { if (lastDispatch_[i] < oldestRow) { oldestRow = lastDispatch_[i]; oldest = i; } } iteratorIndex = oldest; LOG_DEBUG_LIB(LibOdc) << "split writer: evicted iterator " << iteratorIndex << "' " << iteratorIndex2fileName_[iteratorIndex] << "' " << " (oldest row: " << oldestRow << "), nrows_=" << nrows_ << std::endl; delete iterators_[iteratorIndex]; iterators_[iteratorIndex] = 0; Values2IteratorIndex::iterator vit(values2iteratorIndex_.begin()); for (; vit != values2iteratorIndex_.end(); ++vit) if (vit->second == iteratorIndex) break; values2iteratorIndex_.erase(vit); } std::string operation; // bool append = false; if (append_ || !eckit::PathName(fileName).exists()) { filesCreated_[fileName] = 1; operation = "creating"; } else { if (filesCreated_.find(fileName) == filesCreated_.end()) { filesCreated_[fileName] = 1; operation = "overwriting"; } else { append_ = true; filesCreated_[fileName]++; operation = "appending"; } } LOG_DEBUG_LIB(LibOdc) << iteratorIndex << ": " << operation << " '" << fileName << "'" << std::endl; if (iteratorIndex == iterators_.size()) { iterators_.push_back(iteratorsOwner_.createWriteIterator(fileName, append_)); files_.push_back(fileName); } else { iterators_[iteratorIndex] = iteratorsOwner_.createWriteIterator(fileName, append_); files_[iteratorIndex] = fileName; // ASSERT(files_[iteratorIndex] == fileName); } values2iteratorIndex_[dispatchedValues] = iteratorIndex; iteratorIndex2fileName_[iteratorIndex] = fileName; // Prop. metadata iterators_[iteratorIndex]->columns(columns()); iterators_[iteratorIndex]->writeHeader(); return iteratorIndex; } template std::vector WriterDispatchingIterator::outputFiles() { std::vector paths; for (std::map::iterator it(filesCreated_.begin()); it != filesCreated_.end(); ++it) paths.push_back(it->first); return paths; } template WriterDispatchingIterator::~WriterDispatchingIterator() { // LOG_DEBUG_LIB(LibOdc) << "WriterDispatchingIterator::~WriterDispatchingIterator()" << std::endl; delete[] lastValues_; delete[] nextRow_; delete[] columnOffsets_; for (size_t i = 0; i < iterators_.size(); ++i) delete iterators_[i]; } template unsigned long WriterDispatchingIterator::gatherStats(const double* values, unsigned long count) { return dispatch(values, count).gatherStats(values, count); } template void WriterDispatchingIterator::writeHeader() { // LOG_DEBUG_LIB(LibOdc) << "WriterDispatchingIterator::writeHeader" << std::endl; delete[] lastValues_; delete[] nextRow_; delete[] columnOffsets_; int32_t numDoubles = rowDataSizeDoubles(); int32_t count = columns().size(); lastValues_ = new double[numDoubles]; nextRow_ = new double[numDoubles]; columnOffsets_ = new size_t[count]; ASSERT(lastValues_); size_t offset = 0; for (int i(0); i < count; i++) { nextRow_[i] = lastValues_[i] = columns_[i]->missingValue(); columnOffsets_[i] = offset; offset += columns_[i]->dataSizeDoubles(); } nrows_ = 0; } template bool WriterDispatchingIterator::next() { return writeRow(nextRow_, columns().size()) == 0; } template void WriterDispatchingIterator::parseTemplateParameters() { templateParameters_.reset(); TemplateParameters::parse(outputFileTemplate_, templateParameters_, columns()); if (templateParameters_.size() == 0) { std::stringstream ss; ss << "No parameters in output file template '" << outputFileTemplate_ << "'" << std::endl; throw eckit::UserError(ss.str()); } dispatchedIndexes_.clear(); for (size_t i(0); i < templateParameters_.size(); ++i) dispatchedIndexes_.push_back(templateParameters_[i]->columnIndex); initialized_ = true; } template double* WriterDispatchingIterator::data() { return nextRow_; } template double& WriterDispatchingIterator::data(size_t i) { return nextRow_[columnOffsets_[i]]; } template size_t WriterDispatchingIterator::rowDataSizeDoubles() const { size_t total = 0; for (const auto& column : columns()) { total += column->dataSizeDoubles(); } return total; } template int WriterDispatchingIterator::writeRow(const double* values, unsigned long count) { if (!initialized_) parseTemplateParameters(); WRITE_ITERATOR& wi = dispatch(values, count); int rc = wi.writeRow(values, count); if (rc == 0) nrows_++; return rc; } template int WriterDispatchingIterator::open() { return 0; } /* * / template int WriterDispatchingIterator::setColumn(size_t index, std::string name, api::ColumnType type) { ASSERT(index < columns().size()); Column* col = columns_[index]; ASSERT(col); typedef DataStream DS; col->name(name); col->type(type, false); //col->hasMissing(hasMissing); //col->missingValue(missingValue); for (typename Iterators::iterator it = iterators_.begin(); it != iterators_.end(); ++it) //(*it)->setColumn(index, name, type, hasMissing, missingValue); (*it)->setColumn(index, name, type); return 0; } template int WriterDispatchingIterator::setBitfieldColumn(size_t index, std::string name, api::ColumnType type, eckit::sql::BitfieldDef b) { //eckit::Log::info() << "WriterDispatchingIterator::setBitfieldColumn: " << std::endl; ASSERT(index < columns().size()); Column* col = columns_[index]; ASSERT(col); typedef DataStream DS; col->name(name); col->type(type, false); //col->hasMissing(hasMissing); //col->missingValue(missingValue); col->bitfieldDef(b); for (typename Iterators::iterator it = iterators_.begin(); it != iterators_.end(); ++it) //(*it)->setBitfieldColumn(index, name, type, b, hasMissing, missingValue); (*it)->setBitfieldColumn(index, name, type, b); return 0; } / * */ template const MetaData& WriterDispatchingIterator::columns(const MetaData& md) { columns_ = md; return md; } template void WriterDispatchingIterator::missingValue(size_t i, double missingValue) { ASSERT(i < columns().size()); Column* col(columns_[i]); ASSERT(col); col->missingValue(missingValue); } template <> template unsigned long WriterDispatchingIterator::pass1(T& it, const T& end) { if (!(it != end)) { eckit::Log::warning() << "Split: No input data." << std::endl; return 0; } // Copy columns from the input iterator. columns(it->columns()); if (!initialized_) parseTemplateParameters(); size_t maxcols = columns().size(); ASSERT(maxcols > 0); LOG_DEBUG_LIB(LibOdc) << "WriterDispatchingIterator::pass1: columns().size() => " << maxcols << std::endl; nrows_ = 0; for (; it != end; ++it) { if (it->isNewDataset() && columns() != it->columns()) { columns(it->columns()); parseTemplateParameters(); for (size_t i = 0; i < iterators_.size(); ++i) { iterators_[i]->flush(); iterators_[i]->columns(columns()); iterators_[i]->writeHeader(); } } const double* data(it->data()); size_t size(it->columns().size()); int rc(writeRow(data, size)); ASSERT(rc == 0); } LOG_DEBUG_LIB(LibOdc) << "Split: processed " << nrows_ << " row(s)." << std::endl; return nrows_; } template void WriterDispatchingIterator::flushAndResetColumnSizes( const std::map& resetColumnSizeDoubles) { for (size_t i = 0; i < iterators_.size(); ++i) { iterators_[i]->flushAndResetColumnSizes(resetColumnSizeDoubles); } } template <> template void WriterDispatchingIterator::verify(T& it, const T& end) { using namespace eckit; using namespace std; Log::info() << "Verifying split..." << endl; Timer timer("Split verification"); // Copy columns from the input iterator. columns(it->columns()); vector readers; vector > iterators; for (size_t i(0); i < files_.size(); ++i) { Reader* reader(new Reader(files_[i])); readers.push_back(reader); iterators.push_back(make_pair(reader->begin(), reader->end())); } vector rowsRead(files_.size()); // Frames in the source/dispatched cases will have different sizes, so the haveMissing() // value may differ. Only compare the values bool skipTestingHaveMissing = true; Comparator comparator(skipTestingHaveMissing); unsigned long numberOfDifferences(0); long long i(0); for (; it != end; ++i) { if (it->isNewDataset() && columns() != it->columns()) { columns(it->columns()); parseTemplateParameters(); } size_t fileIndex(dispatchIndex(it->data(), it->columns().size())); const std::string& outFileName(files_[fileIndex]); size_t n(columns().size()); typedef Reader::iterator I; std::pair& its(iterators[fileIndex]); I &sIt(its.first), sEnd(its.second); const MetaData& sMetaData(sIt->columns()); try { ASSERT(sIt != sEnd); // In the parent codec, we will always have the largest data size (e.g. for // the longest string). This will not be true in the dispatched iterators // which will only be large enough for the strings dispatched to them. bool compareDataSizes = false; ASSERT(sMetaData.equals(columns(), compareDataSizes)); ++rowsRead[fileIndex]; const double* const originalData(it->data()); const double* const outputData(sIt->data()); comparator.compare(n, originalData, outputData, columns(), sMetaData); } catch (...) { ++numberOfDifferences; Log::info() << "Row " << i << " of input (" << rowsRead[fileIndex] << " of " << outFileName << ") not correct." << endl << endl; } ++it; ++sIt; } Log::info() << "Number of rows: " << i << ". Total number of differences: " << numberOfDifferences << std::endl; ASSERT(!(it != end)); for (size_t j = 0; j < readers.size(); ++j) delete readers[j]; } template void WriterDispatchingIterator::property(std::string key, std::string value) { // TODO: save property, make sure they are propagated to iterators as they are created properties_[key] = value; } template std::string WriterDispatchingIterator::property(std::string key) { return properties_[key]; } template int WriterDispatchingIterator::close() { // LOG_DEBUG_LIB(LibOdc) << "WriterDispatchingIterator::close()" << std::endl; int rc = 0; for (typename Iterators::iterator it = iterators_.begin(); it != iterators_.end(); ++it) { rc |= (*it)->close(); delete *it; } iterators_.clear(); return rc; } //---------------------------------------------------------------------------------------------------------------------- } // namespace odc odc-1.6.3/src/odc/ODBAPIVersion.h0000664000175000017500000000142215146027420016463 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// /// This file will be updated autmatically by the make.sms script /// namespace odc { class ODBAPIVersion { public: static const char* version(); static const char* gitsha1(); static unsigned int formatVersionMajor(); static unsigned int formatVersionMinor(); static const char* installPrefix(); static const char* buildDirectory(); }; } // namespace odc odc-1.6.3/src/odc/Comparator.h0000664000175000017500000001140215146027420016265 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef Comparator_H #define Comparator_H #include #include #include #include "eckit/exception/Exceptions.h" #include "eckit/log/Log.h" namespace eckit { class PathName; class DataHandle; } // namespace eckit namespace odc { namespace core { class MetaData; class Column; } // namespace core const double maxAbsoluteError = 1e-9; const double maxRelativeError = 1e-9; class Comparator { public: Comparator(bool skipTestingHaveMissing = false); void operator()() { run(); } void run(); template bool compare(T1& it1, const T1& end1, T2& it2, const T2& end2, const std::string& desc1, const std::string& desc2); template bool compare(T1& it1, const T1& end1, T2& it2, const T2& end2, const std::string& desc1, const std::string& desc2, const std::vector& excludedColumnsTypes, const std::vector& excludedColumns); void compare(eckit::DataHandle&, eckit::DataHandle&); void compare(const eckit::PathName&, const eckit::PathName&); void compare(const eckit::PathName& pathName1, const eckit::PathName& pathName2, const std::vector& excludedColumnsTypes, const std::vector& excludedcolumns); void compare(const core::MetaData& metaData1, const core::MetaData& metaData2, const std::set& excludedColumnsTypes, const std::set& excludedColumns, std::vector& skipCols); void compare(int nCols, const double* data1, const double* data2, const core::MetaData& metaData1, const core::MetaData& metaData2); void compare(int nCols, const double* data1, const double* data2, const core::MetaData& metaData1, const core::MetaData& metaData2, const std::vector& skipCols); inline static double err(double A, double B) { double relativeError; if (fabs(A) <= maxAbsoluteError || fabs(B) <= maxAbsoluteError) relativeError = fabs(A - B); else if (fabs(B) > fabs(A)) relativeError = fabs((A - B) / B); else relativeError = fabs((A - B) / A); return relativeError; } inline static int same(double A, double B) { return err(A, B) < maxRelativeError; } void raiseNotEqual(const core::Column&, double, double); private: bool skipTestingHaveMissing_; long nRow_; bool NaN_isOK_; }; template bool Comparator::compare(T1& it1, const T1& end1, T2& it2, const T2& end2, const std::string& desc1, const std::string& desc2) { std::vector noExcludedColumnsTypes; std::vector noExcludedColumns; return compare(it1, end1, it2, end2, desc1, desc2, noExcludedColumnsTypes, noExcludedColumns); } template bool Comparator::compare(T1& it1, const T1& end1, T2& it2, const T2& end2, const std::string& desc1, const std::string& desc2, const std::vector& excludedColumnsTypes, const std::vector& excludedColumns) { eckit::Log::info() << "Comparator::compare: (1) " << desc1 << " to (2) " << desc2 << std::endl; nRow_ = 0; // The columns to skip are filled in by the column comparator function std::vector skipCols; // Convert the excluded columns/types into sets, for more efficient lookups std::set excludedColumnsTypesSet(excludedColumnsTypes.begin(), excludedColumnsTypes.end()); std::set excludedColumnsSet(excludedColumns.begin(), excludedColumns.end()); compare(it1->columns(), it2->columns(), excludedColumnsTypesSet, excludedColumnsSet, skipCols); for (; it1 != end1 && it2 != end2; ++it1, ++it2) { ++nRow_; if (it1->isNewDataset()) compare(it1->columns(), it2->columns(), excludedColumnsTypesSet, excludedColumnsSet, skipCols); if (it2->isNewDataset()) compare(it1->columns(), it2->columns(), excludedColumnsTypesSet, excludedColumnsSet, skipCols); compare(it1->columns().size(), it1->data(), it2->data(), it1->columns(), it2->columns(), skipCols); } ASSERT("First file has more rows!" && !(it1 != end1)); ASSERT("Second file has more rows!" && !(it2 != end2)); return true; // ? } } // namespace odc #endif odc-1.6.3/src/odc/WriterBufferingIterator.h0000664000175000017500000001402215146027420020775 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// /// \file WriterBufferingIterator.h /// /// @author Piotr Kuchta, August 2009 #ifndef odc_WriterBufferingIterator_H #define odc_WriterBufferingIterator_H #include "eckit/filesystem/PathName.h" #include "eckit/io/HandleHolder.h" #include "eckit/log/Log.h" #include "odc/IteratorProxy.h" #include "odc/LibOdc.h" #include "odc/codec/CodecOptimizer.h" namespace eckit { class PathName; } namespace eckit { class DataHandle; } namespace odc { //---------------------------------------------------------------------------------------------------------------------- template class Writer; namespace sql { class TableDef; } class WriterBufferingIterator : public eckit::HandleHolder { public: typedef Writer Owner; // WriterBufferingIterator (Owner &owner, eckit::DataHandle *, bool openDataHandle=true); WriterBufferingIterator(Owner& owner, eckit::DataHandle*, bool openDataHandle, const odc::sql::TableDef* tableDef = 0); WriterBufferingIterator(Owner& owner, eckit::DataHandle&, bool openDataHandle, const odc::sql::TableDef* tableDef = 0); ~WriterBufferingIterator(); int open(); double* data(); double& data(size_t i); int setColumn(size_t index, std::string name, api::ColumnType type); int setBitfieldColumn(size_t index, std::string name, api::ColumnType type, eckit::sql::BitfieldDef b); void missingValue(size_t i, double); template unsigned long pass1(T&, const T&); unsigned long gatherStats(const double* values, unsigned long count); int close(); const core::MetaData& columns() const { return columns_; } const core::MetaData& columns(const core::MetaData& md) { columns_ = md; for (auto& col : columns_) col->resetCodec(); // We want to use the default codecs for encoding initialisedColumns_ = columns_.allColumnsInitialised(); return columns_; } void setNumberOfColumns(size_t n) { columns_.setSize(n); } Owner& owner() { return owner_; } eckit::DataHandle& dataHandle() { return handle(); } void property(std::string key, std::string value) { properties_[key] = value; } /// The offset of a given column in the doubles[] data array size_t dataOffset(size_t i) const { ASSERT(columnOffsets_); return columnOffsets_[i]; } // protected: int setOptimalCodecs(); void writeHeader(); int writeRow(const double* values, unsigned long count); // Get the number of doubles per row. size_t rowDataSizeDoubles() const { return rowDataSizeDoubles_; } size_t rowsBufferSize() { return rowsBufferSize_; } void rowsBufferSize(size_t n) { rowsBufferSize_ = n; } void flush(); std::vector outputFiles(); bool next(); // If we are encoding strings, and the relevant string column size changes, we need // to restart the encoding process void flushAndResetColumnSizes(const std::map& resetColumnSizeDoubles); private: size_t rowDataSizeDoublesInternal() const; public: int refCount_; protected: Owner& owner_; core::MetaData columns_; double* lastValues_; double* nextRow_; size_t* columnOffsets_; // in doubles size_t* columnByteSizes_; unsigned long long nrows_; eckit::PathName path_; private: // No copy allowed. WriterBufferingIterator(const WriterBufferingIterator&); WriterBufferingIterator& operator=(const WriterBufferingIterator&); template void pass1init(T&, const T&); std::pair serializeHeader(size_t dataSize, size_t rowsNumber); void allocBuffers(); void allocRowsBuffer(); void resetColumnsBuffer(); int doWriteRow(core::DataStream& stream, const double* values); bool initialisedColumns_; core::Properties properties_; eckit::Buffer rowsBuffer_; unsigned char* nextRowInBuffer_; size_t rowsBufferSize_; size_t rowDataSizeDoubles_; size_t rowByteSize_; codec::CodecOptimizer codecOptimizer_; const odc::sql::TableDef* tableDef_; private: bool openDataHandle_; friend class IteratorProxy; }; template void WriterBufferingIterator::pass1init(T& it, const T& end) { LOG_DEBUG_LIB(LibOdc) << "WriterBufferingIterator::pass1init" << std::endl; // Copy columns from the input iterator. columns(it->columns()); columns_.resetCodecs(); columns_.resetStats(); size_t nCols = it->columns().size(); ASSERT(nCols > 0); allocRowsBuffer(); } template unsigned long WriterBufferingIterator::pass1(T& it, const T& end) { LOG_DEBUG_LIB(LibOdc) << "WriterBufferingIterator::pass1" << std::endl; pass1init(it, end); writeHeader(); unsigned long nrows = 0; for (; it != end; ++it, ++nrows) { if (it->isNewDataset() && it->columns() != columns()) { LOG_DEBUG_LIB(LibOdc) << "WriterBufferingIterator::pass1: Change of input metadata." << std::endl; flush(); pass1init(it, end); writeHeader(); } writeRow(it->data(), it->columns().size()); } LOG_DEBUG_LIB(LibOdc) << "Flushing rest of the buffer..." << std::endl; flush(); LOG_DEBUG_LIB(LibOdc) << "WriterBufferingIterator::pass1: processed " << nrows << " row(s)." << std::endl; ASSERT(close() == 0); return nrows; } //---------------------------------------------------------------------------------------------------------------------- } // namespace odc #endif odc-1.6.3/src/odc/csv/0000775000175000017500000000000015146027420014602 5ustar alastairalastairodc-1.6.3/src/odc/csv/TextReaderIterator.h0000664000175000017500000000535415146027420020543 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// /// \file TextReaderIterator.h /// /// @author Piotr Kuchta, Oct 2010 #ifndef TextReaderIterator_H #define TextReaderIterator_H #include "odc/IteratorProxy.h" #include "odc/core/MetaData.h" namespace eckit { class PathName; } namespace eckit { class DataHandle; } namespace odc { namespace sql { class ODATableIterator; } } // namespace odc namespace odc { //---------------------------------------------------------------------------------------------------------------------- class TextReader; class TextReaderIterator { public: TextReaderIterator(TextReader& owner); TextReaderIterator(TextReader& owner, const eckit::PathName&); TextReaderIterator(const TextReaderIterator&) = delete; TextReaderIterator& operator=(const TextReaderIterator&) = delete; ~TextReaderIterator(); bool isNewDataset(); const double* data() const { return lastValues_; } double* data() { return lastValues_; } double& data(size_t); // long integer(int i); bool operator!=(const TextReaderIterator& other); core::MetaData& columns() { return columns_; } static eckit::sql::BitfieldDef parseBitfields(const std::string&); // protected: int close(); // next() is public cause it needs to be used by the C API functions - normally client code should not use it bool next(); /// The offset of a given column in the doubles[] data array size_t dataOffset(size_t i) const { ASSERT(columnOffsets_); return columnOffsets_[i]; } // Get the number of doubles per row. size_t rowDataSizeDoubles() const { return rowDataSizeDoubles_; } private: void initRowBuffer(); void parseHeader(); core::MetaData columns_; double* lastValues_; size_t* columnOffsets_; size_t rowDataSizeDoubles_; unsigned long long nrows_; std::string delimiter_; std::istream* in_; // eckit::DataHandle *f; // Properties properties_; bool newDataset_; public: bool noMore_; bool ownsF_; int refCount_; protected: // FIXME: TextReaderIterator() : columns_(0) {} friend class odc::TextReader; friend class odc::IteratorProxy; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace odc #endif odc-1.6.3/src/odc/csv/TextReaderIterator.cc0000664000175000017500000002207515146027420020700 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// /// \file TextReaderIterator.cc /// /// @author Piotr Kuchta, Oct 2010 #include #include #include "eckit/filesystem/PathName.h" #include "eckit/log/Log.h" #include "eckit/types/Types.h" #include "eckit/utils/StringTools.h" #include "eckit/utils/Translator.h" #include "odc/LibOdc.h" #include "odc/api/ColumnType.h" #include "odc/csv/TextReader.h" #include "odc/csv/TextReaderIterator.h" using namespace eckit; typedef StringTools S; namespace odc { TextReaderIterator::TextReaderIterator(TextReader& owner) : columns_(0), lastValues_(0), columnOffsets_(0), nrows_(0), delimiter_(owner.delimiter()), in_(0), newDataset_(false), noMore_(false), ownsF_(false), refCount_(0) { in_ = &owner.stream(); ASSERT(in_); parseHeader(); next(); } TextReaderIterator::TextReaderIterator(TextReader& owner, const PathName& pathName) : columns_(0), lastValues_(0), columnOffsets_(0), nrows_(0), delimiter_(owner.delimiter()), in_(0), newDataset_(false), noMore_(false), ownsF_(false), refCount_(0) { in_ = new std::ifstream(pathName.localPath()); ASSERT(in_); ownsF_ = true; parseHeader(); next(); } eckit::sql::BitfieldDef TextReaderIterator::parseBitfields(const std::string& c) { size_t leftBracket(c.find('[')); size_t rightBracket(c.find(']')); if (!(leftBracket != std::string::npos && rightBracket != std::string::npos)) throw UserError( std::string( "Error parsing bitfield definition. Should be like: bitfield_column_name:BITFIELD[a:1;b:3] was: '") + c + "'"); std::string s(c.substr(leftBracket + 1, rightBracket - leftBracket - 1)); // LOG_DEBUG_LIB(LibOdc) << "TextReaderIterator::parseBitfields: s='" << s << "'" << std::endl; eckit::sql::FieldNames names; eckit::sql::Sizes sizes; size_t numberOfBits = 0; std::vector bs(S::split(";", s)); // LOG_DEBUG_LIB(LibOdc) << "TextReaderIterator::parseBitfields: bs=" << bs << std::endl; for (size_t i = 0; i < bs.size(); ++i) { std::vector v(S::split(":", bs[i])); // LOG_DEBUG_LIB(LibOdc) << "TextReaderIterator::parseBitfields: bs[" << i << "] = " << bs[i] << " " << v << " // : " << v.size() << std::endl; if (v.size() != 2) throw UserError("Bitfields definition parse error"); if (std::find(names.begin(), names.end(), v[0]) != names.end()) throw UserError("Names of fields must be unique within one bitfield"); names.push_back(v[0]); int size = atoi(v[1].c_str()); if (!(v.size() > 0)) throw UserError("Size of a bitfield must be positive and larger than zero"); numberOfBits += size; sizes.push_back(size); } // LOG_DEBUG_LIB(LibOdc) << "TextReaderIterator::parseBitfields: numberOfbits=" << numberOfBits << std::endl; if (numberOfBits > 31) { throw UserError("Bitfields can have up to 31 bits only currently"); } return eckit::sql::BitfieldDef(make_pair(names, sizes)); } void TextReaderIterator::parseHeader() { std::string header; std::getline(*in_, header); std::vector columns(S::split(delimiter_, header)); // c->missingValue(missingValue); std::ostream& L(Log::info()); L << "TextReaderIterator::parseHeader: columns: " << columns << std::endl; L << "TextReaderIterator::parseHeader: delimiter: '" << delimiter_ << "'" << std::endl; L << "TextReaderIterator::parseHeader: header: '" << header << "'" << std::endl; for (size_t i = 0; i < columns.size(); ++i) { LOG_DEBUG_LIB(LibOdc) << "TextReaderIterator::parseHeader: column " << i << " '" << columns[i] << "'" << std::endl; std::vector column(S::split(":", columns[i])); if (column.size() < 2) throw UserError(std::string("Column '") + columns[i] + "': format should be NAME \":\" TYPE"); const std::string columnName(S::trim(column[0])); const std::string columnType( S::upper(S::join(":", std::vector(column.begin() + 1, column.end())))); if (!S::startsWith(columnType, "BITFIELD")) { LOG_DEBUG_LIB(LibOdc) << "TextReaderIterator::parseHeader: adding column " << columns_.size() << " '" << columnName << "' : " << columnType << std::endl; columns_.addColumn(columnName, columnType); } else { LOG_DEBUG_LIB(LibOdc) << "TextReaderIterator::parseHeader: adding BITFIELD " << columns_.size() << " '" << columns[i] << std::endl; columns_.addBitfield(columnName, parseBitfields(columns[i])); } } initRowBuffer(); } TextReaderIterator::~TextReaderIterator() { close(); delete[] lastValues_; delete[] columnOffsets_; } bool TextReaderIterator::operator!=(const TextReaderIterator& other) { return noMore_; } void TextReaderIterator::initRowBuffer() { delete[] lastValues_; delete[] columnOffsets_; rowDataSizeDoubles_ = 0; columnOffsets_ = new size_t[columns().size()]; for (size_t i = 0; i < columns().size(); i++) { columnOffsets_[i] = rowDataSizeDoubles_; rowDataSizeDoubles_ += columns()[i]->dataSizeDoubles(); } lastValues_ = new double[rowDataSizeDoubles_]; for (size_t i = 0; i < columns().size(); i++) lastValues_[columnOffsets_[i]] = columns()[i]->missingValue(); } bool TextReaderIterator::next() { newDataset_ = false; if (noMore_) return false; std::string line; std::getline(*in_, line); line = S::trim(line); std::vector values(S::split(delimiter_, line)); size_t nCols = values.size(); if (nCols == 0) { noMore_ = true; return false; } ASSERT(nCols == columns().size()); for (size_t i = 0; i < nCols; ++i) { const std::string& v(S::trim(values[i])); if (S::upper(v) == "NULL") { lastValues_[columnOffsets_[i]] = columns_[i]->missingValue(); } else { api::ColumnType typ(columns()[i]->type()); switch (typ) { case api::STRING: { std::string unquoted = S::unQuote(v); size_t charlen = unquoted.length(); size_t lenDoubles = charlen > 0 ? (((charlen - 1) / 8) + 1) : 1; // If the string is bigger than any we have come across before, we need to // resize the buffers to cope for this // TODO: Adjust the writer to be able to easily continue if all we have changed is a column size. if (lenDoubles > columns_[i]->dataSizeDoubles()) { newDataset_ = true; columns_[i]->dataSizeDoubles(lenDoubles); // Allocate a new buffer, but keep the old data around double* oldData = lastValues_; lastValues_ = 0; initRowBuffer(); ASSERT(oldData); ::memcpy(lastValues_, oldData, columnOffsets_[i] * sizeof(double)); delete[] oldData; } char* buf = reinterpret_cast(&lastValues_[columnOffsets_[i]]); lenDoubles = columns_[i]->dataSizeDoubles(); ::memcpy(buf, &unquoted[0], charlen); ::memset(buf + charlen, 0, (lenDoubles * sizeof(double)) - charlen); break; } case api::REAL: lastValues_[columnOffsets_[i]] = static_cast(Translator()(v)); break; case api::DOUBLE: lastValues_[columnOffsets_[i]] = Translator()(v); break; case api::INTEGER: case api::BITFIELD: lastValues_[columnOffsets_[i]] = static_cast(Translator()(v)); break; default: throw SeriousBug("Unexpected type in column", Here()); } } } return nCols; } bool TextReaderIterator::isNewDataset() { return newDataset_; } double& TextReaderIterator::data(size_t i) { ASSERT(i >= 0 && i < columns().size()); return lastValues_[columnOffsets_[i]]; } int TextReaderIterator::close() { // if (ownsF_ && f) { f->close(); delete f; f = 0; } if (ownsF_ && in_) { delete in_; in_ = 0; } return 0; } } // namespace odc odc-1.6.3/src/odc/csv/TextReader.cc0000664000175000017500000000443215146027420017163 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "odc/csv/TextReader.h" #include #include "odc/csv/TextReaderIterator.h" using namespace eckit; namespace odc { //---------------------------------------------------------------------------------------------------------------------- TextReader::TextReader(std::istream& input, const std::string& delimiter) : in_(&input), deleteDataHandle_(false), delimiter_(delimiter), iteratorSingleton_(new TextReaderIterator(*this)) {} TextReader::TextReader(const std::string& path, const std::string& delimiter) : in_(new std::ifstream(path.c_str())), deleteDataHandle_(true), delimiter_(delimiter), iteratorSingleton_(new TextReaderIterator(*this)) {} TextReader::TextReader(TextReader&& rhs) : in_(rhs.in_), deleteDataHandle_(rhs.deleteDataHandle_), delimiter_(rhs.delimiter_), iteratorSingleton_(rhs.iteratorSingleton_) { rhs.in_ = 0; rhs.deleteDataHandle_ = false; } TextReader& TextReader::operator=(TextReader&& rhs) { std::swap(in_, rhs.in_); std::swap(deleteDataHandle_, rhs.deleteDataHandle_); std::swap(delimiter_, rhs.delimiter_); std::swap(iteratorSingleton_, rhs.iteratorSingleton_); return *this; } TextReader::~TextReader() { /// if (dataHandle_ && deleteDataHandle_) if (in_ && deleteDataHandle_) { // dataHandle_->close(); // delete dataHandle_; delete in_; } } // TextReaderIterator* TextReader::createReadIterator(const PathName& pathName) //{ // return new TextReaderIterator(*this, pathName); // } TextReader::iterator TextReader::begin() { /// @note YUCK. THis is not a good idom. begin() and end() don't really work... return iteratorSingleton_; } TextReader::iterator TextReader::end() const { return iterator(0); } //---------------------------------------------------------------------------------------------------------------------- } // namespace odc odc-1.6.3/src/odc/csv/TextReader.h0000664000175000017500000000432615146027420017027 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// /// @author Piotr Kuchta /// @author Simon Smart /// @date Oct 2010 #ifndef odc_TextReader_H #define odc_TextReader_H #ifdef SWIGPYTHON #include #endif #include "odc/IteratorProxy.h" namespace eckit { class PathName; } namespace odc { //---------------------------------------------------------------------------------------------------------------------- class TextReaderIterator; class TextReader { public: typedef IteratorProxy iterator; typedef iterator::Row row; TextReader(std::istream&, const std::string& delimiter); TextReader(const std::string& path, const std::string& delimiter); TextReader(const TextReader&) = delete; TextReader& operator=(const TextReader&) = delete; TextReader(TextReader&&); TextReader& operator=(TextReader&&); virtual ~TextReader(); iterator begin(); iterator end() const; std::istream& stream() { return *in_; } // For C API // TextReaderIterator* createReadIterator(const eckit::PathName&); #ifdef SWIGPYTHON iterator __iter__() { return begin(); } #endif const std::string& delimiter() { return delimiter_; } private: std::istream* in_; bool deleteDataHandle_; std::string delimiter_; // This is a bit nasty. The TextReader currently assumes that the data will only be // iterated _once_ (hence initiated by an istream). Therefore only create the iterator // once so that the MetaData doesn't get read from the stream multiple times. iterator iteratorSingleton_; friend class odc::IteratorProxy; friend class odc::TextReaderIterator; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace odc #endif odc-1.6.3/src/odc/Block.h0000664000175000017500000000325215146027420015214 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Piotr Kuchta, ECMWF, Oct 2015 #ifndef Block_H #define Block_H #include "eckit/filesystem/PathName.h" #include "eckit/io/Length.h" #include "eckit/io/MultiHandle.h" #include "eckit/io/Offset.h" #include "odc/tools/Tool.h" #include namespace odc { typedef unsigned long long ullong; class Block { public: Block() : fileName(), start(), end(), firstRow(), lastRow() {} Block(const eckit::PathName& fileName, const eckit::Offset& start, const eckit::Offset& end, const ullong firstRow, const ullong lastRow) : fileName(fileName), start(start), end(end), firstRow(firstRow), lastRow(lastRow) {} Block(const Block& other) : fileName(other.fileName), start(other.start), end(other.end), firstRow(other.firstRow), lastRow(other.lastRow) {} Block& operator=(const Block& other) { fileName = other.fileName; start = other.start; end = other.end; firstRow = other.firstRow; lastRow = other.lastRow; return *this; } eckit::PathName fileName; eckit::Offset start; eckit::Offset end; ullong firstRow; ullong lastRow; std::string str() const; }; std::ostream& operator<<(std::ostream&, const Block&); } // namespace odc #endif odc-1.6.3/src/odc/MDI.h0000664000175000017500000000203615146027420014572 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @file MDI.h /// @author Piotr Kuchta - ECMWF Nov 13 #ifndef MDI_H #define MDI_H namespace odc { class MDI { public: static double realMDI() { return realMDI_; } static double integerMDI() { return integerMDI_; } // Historically this used 0 as the missing value. This seems ... wrong. Really wrong. No bits // set is not the same as a missing value (see ODB-493) static double bitfieldMDI() { return integerMDI(); } static void realMDI(double v) { realMDI_ = v; } static void integerMDI(double v) { integerMDI_ = v; } private: static double realMDI_; static double integerMDI_; }; } // namespace odc #endif odc-1.6.3/src/odc/RowsCounter.h0000664000175000017500000000130415146027420016450 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #pragma once #include namespace odc { class RowsCounter { public: static unsigned long long fastRowCount(const eckit::PathName&); private: // No copy allowed RowsCounter(const RowsCounter&); RowsCounter& operator=(const RowsCounter&); }; } // namespace odc odc-1.6.3/src/odc/ShiftedBitColumnExpression.h0000664000175000017500000000150315146027420021442 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ // File ShiftedBitColumnExpression.h // Piotr Kuchta - ECMWF Dec 2012 #ifndef ShiftedBitColumnExpression_H #define ShiftedBitColumnExpression_H #include "odc/BitColumnExpression.h" #include "odc/ShiftedColumnExpression.h" namespace odc { namespace sql { namespace expression { typedef ShiftedColumnExpression ShiftedBitColumnExpression; } // namespace expression } // namespace sql } // namespace odc #endif odc-1.6.3/src/odc/RequestUtils.h0000664000175000017500000000205315146027420016631 0ustar alastairalastair/* * (C) Copyright 1996-2018 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Simon Smart /// @date Aug 2018 #ifndef odc_RequestUtils_h #define odc_RequestUtils_h #include #include #include namespace odc { //---------------------------------------------------------------------------------------------------------------------- typedef std::map> RequestDict; RequestDict unquoteRequestValues(const RequestDict& r); void checkKeywordsHaveValues(const RequestDict& request, const std::vector& keywords); //---------------------------------------------------------------------------------------------------------------------- } // namespace odc #endif odc-1.6.3/src/odc/CommandLineParser.cc0000664000175000017500000001227515146027420017670 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// \file CommandLineParser.h /// /// @author Piotr Kuchta, ECMWF, July 2009 #include "odc/CommandLineParser.h" #include "odc/LibOdc.h" #include "eckit/exception/Exceptions.h" #include "eckit/log/Log.h" #include "eckit/utils/Translator.h" namespace odc { namespace tool { CommandLineParser::~CommandLineParser() {} CommandLineParser::CommandLineParser(int argc, char** argv) : commandLineParsed_(false), argc_(argc), argv_(argv), registeredOptionsWithArguments_(), optionsWithArguments_(), optionsNoArguments_(), parameters_() {} CommandLineParser::CommandLineParser(const CommandLineParser& other) { CommandLineParser& self = *this; self.registeredOptionsWithArguments_ = other.registeredOptionsWithArguments_; self.parameters_ = const_cast(other).parameters(); self.optionsWithArguments_ = other.optionsWithArguments_; self.optionsNoArguments_ = other.optionsNoArguments_; self.commandLineParsed_ = true; ASSERT(self.commandLineParsed_ == other.commandLineParsed_); } CommandLineParser& CommandLineParser::operator=(const CommandLineParser& other) { if (this == &other) return *this; CommandLineParser& self = *this; self.registeredOptionsWithArguments_ = other.registeredOptionsWithArguments_; self.parameters_ = const_cast(other).parameters(); self.optionsWithArguments_ = other.optionsWithArguments_; self.optionsNoArguments_ = other.optionsNoArguments_; self.commandLineParsed_ = true; ASSERT(self.commandLineParsed_ == other.commandLineParsed_); return *this; } int CommandLineParser::argc() { return argc_; } std::string CommandLineParser::argv(int i) { if (i >= argc_) { std::stringstream ss; ss << "Expected at least " << i << " command line parameters"; throw eckit::UserError(ss.str()); } return argv_[i]; } void CommandLineParser::registerOptionWithArgument(const std::string& option) { registeredOptionsWithArguments_.insert(std::string(option)); } const std::vector CommandLineParser::parameters() { if (!commandLineParsed_) parseCommandLine(); return parameters_; } bool CommandLineParser::optionIsSet(const std::string& option) { if (!commandLineParsed_) parseCommandLine(); if (optionsNoArguments_.find(option) != optionsNoArguments_.end()) return true; return optionsWithArguments_.find(option) != optionsWithArguments_.end(); } template T CommandLineParser::optionArgument(const std::string& option, T defaultValue) { if (!commandLineParsed_) parseCommandLine(); std::map::iterator it = optionsWithArguments_.find(option); if (it == optionsWithArguments_.end()) return defaultValue; eckit::Translator translator; return translator(it->second); } void CommandLineParser::parseCommandLine() { for (int i = 0; i < argc(); ++i) { std::string s = argv(i); if (s[0] != '-' || s.size() == 1) { parameters_.push_back(s); LOG_DEBUG_LIB(LibOdc) << "CommandLineParser::parseCommandLine: parameter: " << s << std::endl; } else { if (registeredOptionsWithArguments_.find(s) != registeredOptionsWithArguments_.end()) { optionsWithArguments_[s] = argv(++i); LOG_DEBUG_LIB(LibOdc) << "CommandLineParser::parseCommandLine: option with argument: " << s << " = " << optionsWithArguments_[s] << std::endl; } else { optionsNoArguments_.insert(s); LOG_DEBUG_LIB(LibOdc) << "CommandLineParser::parseCommandLine: option with no argument: " << s << std::endl; } } } commandLineParsed_ = true; } void CommandLineParser::print(std::ostream& s) const { for (std::set::const_iterator i = optionsNoArguments_.begin(); i != optionsNoArguments_.end(); ++i) s << *i << " "; for (std::map::const_iterator i = optionsWithArguments_.begin(); i != optionsWithArguments_.end(); ++i) s << i->first << " " << i->second << " "; for (size_t i = 0; i < parameters_.size(); ++i) s << parameters_[i] << " "; } // Template function's explicit instantiations. template std::string CommandLineParser::optionArgument(const std::string&, std::string); template int CommandLineParser::optionArgument(const std::string&, int); template long CommandLineParser::optionArgument(const std::string&, long); template double CommandLineParser::optionArgument(const std::string&, double); } // namespace tool } // namespace odc odc-1.6.3/src/odc/ConstantSetter.h0000664000175000017500000000244415146027420017144 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef ConstantSetter_H #define ConstantSetter_H #include "odc/IteratorProxy.h" #include "odc/ODAUpdatingIterator.h" namespace odc { template class ConstantSetter { public: typedef typename odc::ODAUpdatingIterator iterator_class; typedef typename odc::IteratorProxy iterator; ConstantSetter(const T& b, const T& e, const std::vector& columns, const std::vector& values) : ii_(b), end_(e), columns_(columns), values_(values) {} ~ConstantSetter() {} iterator begin() { return iterator(new iterator_class(ii_, end_, columns_, values_)); } const iterator end() { return iterator(new iterator_class(end_)); } private: T ii_; const T& end_; const std::vector columns_; const std::vector values_; }; } // namespace odc #include "odc/ConstantSetter.cc" #endif odc-1.6.3/src/odc/ODBApplication.h0000664000175000017500000000215215146027420016750 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// \file ODBApplication.h /// /// @author Piotr Kuchta, ECMWF, Feb 2009 #ifndef ODBApplication_H #define ODBApplication_H #include "eckit/runtime/Tool.h" #include "odc/CommandLineParser.h" namespace odc { namespace tool { class ODBApplication : public eckit::Tool { public: ODBApplication(int argc, char** argv); virtual ~ODBApplication(); CommandLineParser& commandLineParser(); private: virtual eckit::LogTarget* createInfoLogTarget() const; virtual eckit::LogTarget* createWarningLogTarget() const; virtual eckit::LogTarget* createErrorLogTarget() const; virtual eckit::LogTarget* createDebugLogTarget() const; CommandLineParser clp_; }; } // namespace tool } // namespace odc #endif odc-1.6.3/src/odc/ODBTarget.cc0000664000175000017500000000222415146027420016071 0ustar alastairalastair/* * (C) Copyright 1996-2016 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "odc/ODBTarget.h" #include #include "eckit/log/TimeStamp.h" #include "eckit/runtime/Monitor.h" namespace odc { ODBTarget::ODBTarget(const char* tag, eckit::LogTarget* target) : eckit::WrapperTarget(target), tag_(tag) {} void ODBTarget::writePrefix() { std::ostringstream oss; oss //<< std::setw(3) //<< std::setfill('0') //<< Monitor::instance().self() << "000" << std::setfill(' ') << ' ' << eckit::TimeStamp() << ' '; if (tag_ && *tag_) { oss << tag_ << ' '; } const std::string& s(oss.str()); const char* p(s.c_str()); target_->write(p, p + s.size()); } void ODBTarget::print(std::ostream& s) const { s << "ODBTarget"; } void ODBTarget::writeSuffix() {} } // namespace odc odc-1.6.3/src/odc/fwrap.py0000775000175000017500000003402615146027420015510 0ustar alastairalastair#!/usr/bin/env python import re PARAM_TYPE_COLUMN = 43 CONSTANTS = {} def formatParameter(typ, name): global PARAM_TYPE_COLUMN space = ' ' * (PARAM_TYPE_COLUMN - len(typ)) return typ + space + ':: ' + name def declarations(source_cc = 'odbql.cc'): decls = [line for line in [l.strip() for l in open(source_cc).read().splitlines()] if line.find('odbql_') <> -1 and not line.startswith('//') and line.find('return') == -1 and line.find('virtual') == -1 and line.find('::') == -1 and line.find('typedef') == -1] return decls def constants(source_h = 'odbql.h'): lines = [line.strip() for line in open(source_h).read().splitlines() if line.startswith('#define ODBQL_') and len(line.split()) > 2] defs = [line.split(None, 2)[1:] for line in lines] return defs def evaluated_expression(s): try: e = s for c in CONSTANTS: e = e.replace(c, CONSTANTS[c]) return ' [' + str(eval(e)) + ']' except: return '' def translate_value_and_comment(value_and_possibly_comment): value = value_and_possibly_comment.split('/*')[0].strip() comment = '' if len(value_and_possibly_comment.split('/*')) > 1: comment = value_and_possibly_comment.split('/*')[1].split('*/')[0] if value.find('|') <> -1: comment = value + ' ' + comment + evaluated_expression(value_and_possibly_comment) l,r = [x.strip(' ()') for x in value.split('|')] i, shift = [x.strip() for x in r.split('<<')] value = 'IOR(%s, LSHIFT(%s,%s))' % (l, i, shift) if value.startswith('0x'): comment = value + ' ' + comment value = eval(value) return value, '! ' + comment def generateParameter(define): global CONSTANTS name, value_and_possibly_comment = define typ = 'integer' if value_and_possibly_comment.find('"') <> -1: typ = 'character(len=*)' if name == 'ODBQL_TRANSIENT': value, comment = '-1', " ! ((odbql_destructor_type)-1)" elif name == 'ODBQL_STATIC': value, comment = '0', ' ! ((odbql_destructor_type)0)' else: value, comment = translate_value_and_comment(value_and_possibly_comment) CONSTANTS[name] = value return '%s, parameter :: %s = %s %s' % (typ, name, value, comment) def generateParameters(defs): return """ module odbql_constants implicit none """ + '\n'.join(generateParameter(d) for d in defs) + """ end module odbql_constants """ def normalize_type(t): return re.sub(' [*]', '*', t) def parseParam(p): p = p.strip() # Hack for the destructor type, used in sqlite3 to let user pass # a function that will release memory occupied by strings and blobs' # No idea how to handle this in Fortran at this point, not sure # if we immediately need this functionality anyway. ## if p == 'void(*d)(void*)': return 'void(*)(void*)', 'd' param_name = re.findall(r"[\w']+", p)[-1] param_type = p[:len(p) - len(param_name)].strip() param_type = normalize_type(param_type) return param_type, param_name def parseDeclaration(decl): ret_and_fun_name = decl.split('(', 1)[0].strip() fun_name = re.findall(r"[\w']+", ret_and_fun_name)[-1] return_type = ret_and_fun_name[:-len(fun_name)].strip() return_type = normalize_type(return_type) params = decl.split('(',1)[1].strip().rsplit(')', 1)[0].strip() params = list([parseParam(p) for p in params.split(',')]) # filter out void from parameterless functions e.g: const char * odbql_libversion(void) if len(params) == 1 and (len(params[0][0]) == 0 and params[0][1] == 'void'): params = [] return (decl, (return_type, fun_name, params)) def translate_type_for_binding(t): #print 'translate_type_for_binding:', t if t == 'const char*': return 'character(kind=C_CHAR), dimension(*)' if t == 'const char**': return 'character(kind=C_CHAR), dimension(*)' # TODO if t == 'double': return 'real(kind=C_DOUBLE), value' if t == 'int': return 'integer(kind=C_INT), value' if t == 'odbql*': return 'type(C_PTR), VALUE' if t == 'odbql**': return 'type(C_PTR)' if t == 'odbql_stmt*': return 'type(C_PTR), VALUE' if t == 'odbql_stmt**': return 'type(C_PTR)' if t == 'odbql_value*': return 'type(C_PTR), VALUE' if t == 'odbql_value**': return 'type(C_PTR)' if t == 'void(*)(void*)': return 'type(C_PTR), VALUE' raise Exception("Don't know how to translate '" + t + "'") def translate_type_for_binding_return(t): #print 'translate_type_for_binding_return:', t if t == 'const char*': return 'type(C_PTR)' if t == 'const unsigned char*': return 'type(C_PTR)' if t == 'int': return 'integer(kind=C_INT)' if t == 'double': return 'real(kind=C_DOUBLE)' if t == 'odbql_value*': return 'type(C_PTR)' if t == 'error_code_t': return 'integer(kind=C_INT)' raise Exception("Don't know how to translate '" + t + "'") def translate_type_for_fortran(t): #print 'translate_type_for_fortran:', t if t == 'const char*': return 'character(len=*), intent(in)' if t == 'const char**': return 'character(len=*), intent(out)' if t == 'double': return 'real(kind=C_DOUBLE), value' if t == 'int': return 'integer(kind=C_INT), value' if t == 'odbql*': return 'type(odbql), value' if t == 'odbql**': return 'type(odbql)' if t == 'odbql_stmt*': return 'type(odbql_stmt), value' if t == 'odbql_stmt**': return 'type(odbql_stmt)' if t == 'odbql_value*': return 'type(odbql_value), value' if t == 'odbql_value**': return 'type(odbql_value)' if t == 'void(*)(void*)': return 'type(C_PTR), value' raise Exception("Don't know how to translate '" + t + "'") def translate_type_for_fortran_return(t): #print 'translate_type_for_fortran_return:', t if t == 'const char*': return 'character(len=*), intent(out)' if t == 'const unsigned char*': return 'character(len=*), intent(out)' # TODO: think about it if t == 'int': return 'integer(kind=C_INT)' if t == 'double': return 'real(kind=C_DOUBLE)' if t == 'odbql_value*': return 'type(odbql_value)' if t == 'error_code_t': return 'integer(kind=C_INT), intent(out), optional' raise Exception("Don't know how to translate '" + t + "'") def fortranParamTypeDeclaration(p, translate_type = translate_type_for_binding): typ, parameter_name = p fortran_type = translate_type(typ) return formatParameter(fortran_type, parameter_name) nl_indent = '\n ' helper_functions = """ !> Helper routine to convert C '\\0' terminated strings to Fortran strings subroutine C_to_F_string(c_string_pointer, out_string) use, intrinsic :: iso_c_binding, only: c_ptr,c_f_pointer,c_char,c_null_char type(c_ptr), intent(in) :: c_string_pointer character(len=*), intent(out) :: out_string character(kind=c_char), dimension(:), pointer :: char_array_pointer integer :: i,length char_array_pointer => null() call c_f_pointer(c_string_pointer,char_array_pointer,[255]) if (.not.associated(char_array_pointer)) then out_string = "NULL" return end if out_string = " " do i = 1, len(out_string) if (char_array_pointer(i) == c_null_char) exit out_string(i:i) = char_array_pointer(i) end do end subroutine """ status_handling_code = """ if (present(status)) then status = rc ! let user handle the error else if (rc /= ODBQL_OK .and. rc /= ODBQL_ROW .and. rc /= ODBQL_DONE .and. rc /= ODBQL_METADATA_CHANGED) then write (0,*) 'Error in %(function_name)s' stop end if end if """ status_handling_code = nl_indent.join (status_handling_code.splitlines()) def parameter_type(p): return p[0] def parameter_name(p): return p[1] def actual_parameter(p): if p[1] == 'iCol': return p[1] + '-1' if p[0] == 'const char*': return p[1] + '_tmp' if p[0] == 'odbql*': return p[1] + '%this' if p[0] == 'odbql**': return p[1] + '%this' if p[0] == 'odbql_stmt*': return p[1] + '%this' if p[0] == 'odbql_stmt**': return p[1] + '%this' if p[0] == 'odbql_value*': return p[1] + '%this' if p[0] == 'odbql_value**': return p[1] + '%this' return p[1] def generateWrapper(signature, comment, template): print 'generateWrapper:', signature, comment, template global status_handling_code return_type, function_name, params = signature procedure_keyword = 'function' output_parameter = function_name # filter out user supplied destructor functions (not supported now) fortran_params = [p for p in params if not parameter_type(p) == 'void(*)(void*)'] fortran_params_excluding_return_parameter = fortran_params[:] binding_parameter_list = '(' + ','.join([p[1] for p in params]) + ')' actual_binding_parameter_list = '(' + ','.join([actual_parameter(p) for p in params]) + ')' temporary_variables_declarations = nl_indent.join( [formatParameter('character(len=len_trim('+p[1]+')+1)', p[1] + '_tmp') for p in params if p[0] == 'const char*'] + [formatParameter('type(C_PTR)', p[1]) for p in params if p[0] == 'void(*)(void*)']) temporary_variables_assignments = nl_indent.join(p[1] + '_tmp = ' + p[1] + '//achar(0)' for p in params if p[0] == 'const char*') call_binding = function_name + '_c' + actual_binding_parameter_list error_handling = '' return_value_tmp = None if return_type == 'odbql_value*': return_value_tmp = output_parameter + "%this" if return_type == 'error_code_t': procedure_keyword = 'subroutine' output_parameter = 'status' return_value_tmp = 'rc' temporary_variables_declarations = nl_indent.join( [temporary_variables_declarations, formatParameter('integer(kind=c_int)', 'rc')]) fortran_params.append( (return_type, output_parameter) ) error_handling = status_handling_code % locals() if return_type == 'const char*' or return_type == 'const unsigned char*': procedure_keyword = 'subroutine' output_parameter = 'return_value' fortran_params.append( (return_type, output_parameter) ) # For C to F string, we call a subroutine rather than make an assignment return_value_assignment = "call C_to_F_string(%s, %s)" % (call_binding, (return_value_tmp or output_parameter)) else: # Normal assignment return_value_assignment = (return_value_tmp or output_parameter) + ' = ' + call_binding binding_parameters_declarations = nl_indent.join([fortranParamTypeDeclaration(p) for p in params]) binding_return_type_declaration = formatParameter(translate_type_for_binding_return(return_type), function_name + '_c') fortran_parameter_list = '(' + ','.join([p[1] for p in fortran_params]) + ')' fortran_parameters_declarations = nl_indent.join( [fortranParamTypeDeclaration(p, translate_type = translate_type_for_fortran) for p in fortran_params_excluding_return_parameter]) fortran_return_type_declaration = formatParameter(translate_type_for_fortran_return(return_type), output_parameter) use_intrinsic = formatParameter('use, intrinsic', 'iso_c_binding') return template % locals() def generateWrappers(decls, header, footer, template): s = header for original, ast in [parseDeclaration(decl) for decl in decls]: s += generateWrapper(ast, original, template) + '\n' s += footer return s def generateBindings(source_cc = '../odb_api/odbql.cc', source_h = '../odb_api/odbql.h', binding_f90 = '../fortran/odbql_binding.f90', wrappers_f90 = '../fortran/odbql_wrappers.f90', constants_f90 = '../fortran/odbql_constants.f90'): decls = declarations(source_cc) defs = constants(source_h) with open(constants_f90, 'w') as f: f.write(generateParameters(defs)) ########## odbql_binding.f90: declarations of ISO C bindings for the new ODB API with open(binding_f90, 'w') as f: f.write(generateWrappers(decls, header = """ !!!!! THIS FILE WAS AUTOMATICALLY GENERATED. DO NOT EDIT MANUALLY !!!!! !! See fwrap.py module odbql_binding use iso_c_binding use, intrinsic :: iso_c_binding implicit none interface """, footer = """ end interface end module odbql_binding """, template = """ !> %(comment)s function %(function_name)s_c %(binding_parameter_list)s bind(C, name="%(function_name)s") %(use_intrinsic)s %(binding_parameters_declarations)s %(binding_return_type_declaration)s end function %(function_name)s_c """)) ############ odbql_wrappers.f90 The Fortran user interface with open(wrappers_f90, 'w') as f: f.write(generateWrappers(decls, header = """ !!!!! THIS FILE WAS AUTOMATICALLY GENERATED. DO NOT EDIT MANUALLY !!!!! !! See fwrap.py module odbql_wrappers use odbql_binding use odbql_constants implicit none type odbql type(c_ptr) :: this end type type odbql_stmt type(c_ptr) :: this end type type odbql_value type(c_ptr) :: this end type contains %(helper_functions)s """ % globals(), footer = """ end module odbql_wrappers """, template = """ !> %(comment)s %(procedure_keyword)s %(function_name)s %(fortran_parameter_list)s use odbql_binding %(use_intrinsic)s %(fortran_parameters_declarations)s %(fortran_return_type_declaration)s %(temporary_variables_declarations)s %(temporary_variables_assignments)s %(return_value_assignment)s %(error_handling)s end %(procedure_keyword)s %(function_name)s """ )) if __name__ == '__main__': generateBindings() odc-1.6.3/src/odc/StringTool.cc0000664000175000017500000001262615146027420016431 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// \file StringTool.h /// /// @author Piotr Kuchta, ECMWF, Feb 2009 #include "eckit/utils/StringTools.h" #include #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/PathName.h" #include "eckit/io/FileHandle.h" #include "eckit/log/CodeLocation.h" #include "eckit/log/Log.h" #include "eckit/utils/Regex.h" #include "eckit/utils/Translator.h" #include "odc/LibOdc.h" #include "odc/StringTool.h" using namespace std; using namespace eckit; namespace odc { std::vector StringTool::readLines(const PathName fileName, bool logging) { std::string s = readFile(fileName, logging); return StringTools::split("\n", s); } std::string StringTool::readFile(const PathName fileName, bool logging) { const size_t CHUNK_SIZE = 1024; char buffer[CHUNK_SIZE]; FileHandle f(fileName); f.openForRead(); AutoClose close(f); std::string ret; size_t read, totalRead = 0; while ((read = f.read(buffer, sizeof(buffer) / sizeof(char))) > 0) { totalRead += read; ret.append(std::string(static_cast(buffer), read)); } if (logging) Log::info() << "Read " << totalRead << " bytes from file " << fileName << "[" << ret << "]" << std::endl; return ret; } int StringTool::shell(std::string cmd, const CodeLocation& where, bool assertSuccess) { std::string c = "/bin/sh -c \"" + cmd + "\""; Log::info() << "Executing '" + c + "' "; Log::info() << " " << where.file() << " +" << where.line(); Log::info() << std::endl; int rc = std::system(c.c_str()); if (assertSuccess && rc != 0) { throw eckit::SeriousBug(std::string(" \"") + cmd + "\" failed. " + where.file() + " +" + Translator()(where.line())); ASSERT(rc == 0); } return rc; } bool StringTool::check(const std::string& s, ctypeFun fun) { for (size_t i = 0; i < s.size(); ++i) if (!fun(s[i])) return false; return true; } bool StringTool::isInQuotes(const std::string& value) { return value.size() > 1 && ((value[0] == '"' && value[value.size() - 1] == '"') || (value[0] == '\'' && value[value.size() - 1] == '\'')); } std::string StringTool::unQuote(const std::string& value) { if (!value.size()) return value; if (isInQuotes(value)) return value.substr(1, value.size() - 2); return value; } std::string StringTool::double_as_string(double m) { char buf[sizeof(double) + 1]; memset(buf, 0, sizeof(buf)); memcpy(buf, reinterpret_cast(&m), sizeof(double)); return std::string(buf, sizeof(double)); } double StringTool::cast_as_double(const std::string& value) { char buf[sizeof(double)]; memset(buf, ' ', sizeof(double)); ASSERT(value.size() <= sizeof(double)); strncpy(buf + sizeof(double) - value.size(), value.c_str(), value.size()); return *reinterpret_cast(buf); } std::string StringTool::int_as_double2string(double v) { std::stringstream s; s.precision(0); s << fixed << v; return s.str(); } double StringTool::translate(const std::string& v) { return isInQuotes(v) ? cast_as_double(unQuote(v)) : Translator()(v); } void StringTool::trimInPlace(std::string& str) { str = StringTools::trim(str); } bool StringTool::match(const std::string& regex, const std::string& s) { return Regex(regex).match(s); } bool StringTool::matchAny(const std::vector& regs, const std::string& s) { for (size_t i = 0; i < regs.size(); ++i) if (match(regs[i], s)) return true; return false; } ostream& operator<<(std::ostream& s, const std::vector& st) { s << '['; for (std::vector::const_iterator it = st.begin(); it != st.end(); ++it) s << *it << ","; s << ']'; return s; } std::string StringTool::valueAsString(double d, api::ColumnType t) { using namespace api; stringstream s; switch (t) { case INTEGER: return int_as_double2string(d); case BITFIELD: return int_as_double2string(d); // TODO: have something to print bitfields in StringTool case STRING: return double_as_string(d); case DOUBLE: case REAL: s << d; return s.str(); case IGNORE: default: ASSERT(0 && "Type not known."); } s << d; return s.str(); } std::string StringTool::patchTimeForMars(const std::string& ss) { std::string v = ss; if (v.size() == 5) v = std::string("0") + v; if (v.size() == 6) { std::string s = v; v = v.substr(0, 4); LOG_DEBUG_LIB(odc::LibOdc) << "StringTool::patchTimeForMars: stripping seconds from TIME: '" << s << "' => '" << v << "'" << std::endl; } return v; } bool StringTool::isSelectStatement(const std::string& s) { return StringTool::match("select", eckit::StringTools::lower(eckit::StringTools::trim(s))); } } // namespace odc odc-1.6.3/src/odc/pyodbapi.h0000664000175000017500000000071015146027420015765 0ustar alastairalastair// Classes user by swig #include "eckit/eckit.h" #include "eckit/sql/SQLParser.h" #include "odc/DispatchingWriter.h" #include "odc/IteratorProxy.h" #include "odc/ODBAPISettings.h" #include "odc/ODBAPIVersion.h" #include "odc/Reader.h" #include "odc/Select.h" #include "odc/WriterBufferingIterator.h" #include "odc/core/Column.h" #include "odc/core/MetaData.h" #include "odc/csv/TextReader.h" #include "odc/csv/TextReaderIterator.h" using namespace odc; odc-1.6.3/src/odc/odccapi.cc0000664000175000017500000004006115146027420015721 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// /// \file odbcapi.cc /// /// @author Piotr Kuchta, March 2009 #include "eckit/runtime/Main.h" #include "odc/ODAHandle.h" #include "odc/ODBAPISettings.h" #include "odc/ODBAPIVersion.h" #include "odc/Reader.h" #include "odc/Select.h" #include "odc/SelectIterator.h" #include "odc/Writer.h" #include "odc/api/odc.h" #include "odc/core/MetaData.h" #include "odc/core/TablesReader.h" #include "odc/odccapi.h" using namespace eckit; using namespace odc; using namespace odc::api; using namespace odc::core; char* dummyCommandLineArgs[] = {const_cast("odbcapi"), 0}; // #include "odbcapi.h" template int get_bitfield(T it, int index, char** bitfield_names, char** bitfield_sizes, int* nSize, int* sSize) { I* iter = reinterpret_cast(it); const eckit::sql::BitfieldDef& bitfieldDef(iter->columns()[index]->bitfieldDef()); eckit::sql::FieldNames fieldNames(bitfieldDef.first); eckit::sql::Sizes sizes(bitfieldDef.second); std::stringstream ns, ss; for (size_t i = 0; i < fieldNames.size(); ++i) { ns << fieldNames[i] << ":"; ss << sizes[i] << ":"; } std::string names(ns.str()); std::string ssizes(ss.str()); // FIXME: the memory allocated by strdup should be freed on Fortran side // TODO: This is a memory leak!!! *bitfield_names = strdup(names.c_str()); *bitfield_sizes = strdup(ssizes.c_str()); *nSize = names.size(); *sSize = ssizes.size(); return 0; } extern "C" { void odb_start() { static const char* argv[2] = {"odb_start", 0}; odb_start_with_args(1, const_cast(argv)); } int odc_git_sha1(const char** o) { return odc_vcs_version(o); } void odb_start_with_args(int argc, char* argv[]) { eckit::Main::initialise(argc, argv); } double odb_count(const char* filename) { double n = 0; PathName path = filename; core::TablesReader mdReader(path); for (auto it(mdReader.begin()), end(mdReader.end()); it != end; ++it) { n += it->rowCount(); } return n; } int get_blocks_offsets(const char* fileName, size_t* numberOfBlocks, off_t** offsets, size_t** sizes) { core::TablesReader reader(fileName); OffsetList offs; LengthList lengths; for (const auto& table : reader) { offs.push_back(table.startPosition()); lengths.push_back(table.nextPosition() - table.startPosition()); } ASSERT(offs.size() == lengths.size()); size_t n = offs.size(); *numberOfBlocks = n; *offsets = new off_t[n]; *sizes = new size_t[n]; for (size_t i = 0; i < n; ++i) { (*offsets)[i] = offs[i]; (*sizes)[i] = lengths[i]; } return 0; } int release_blocks_offsets(off_t** offsets) { delete[] *offsets; *offsets = 0; return 0; } int release_blocks_sizes(size_t** sizes) { delete[] *sizes; *sizes = 0; return 0; } unsigned int odb_get_headerBufferSize() { return ODBAPISettings::instance().headerBufferSize(); } void odb_set_headerBufferSize(unsigned int n) { ODBAPISettings::instance().headerBufferSize(n); } unsigned int odb_get_setvbufferSize() { return ODBAPISettings::instance().setvbufferSize(); } void odb_set_setvbufferSize(unsigned int n) { ODBAPISettings::instance().setvbufferSize(n); } unsigned int odc_format_version_major() { return odc::ODBAPIVersion::formatVersionMajor(); } unsigned int odc_format_version_minor() { return odc::ODBAPIVersion::formatVersionMinor(); } /// @param config ignored for now. oda_ptr odb_read_create(const char* config, int* err) { Reader* o = new Reader; *err = !o; return oda_ptr(o); } oda_ptr odb_create(const char* config, int* err) { return odb_read_create(config, err); } /// @param config ignored for now. oda_ptr odb_select_create(const char* config, int* err) { // n.b. We will provide the output buffer --> don't get the Select to // allocate its own internally Select* o = new Select("", /* manageOwnBuffer */ false); *err = !o; return oda_ptr(o); } /// @param config ignored for now. oda_writer_ptr odb_writer_create(const char* config, int* err) { // PathName path = filename; Writer<>* o = new Writer<>; //(path); *err = !o; return oda_writer_ptr(o); } int odb_read_destroy(oda_ptr o) { delete reinterpret_cast(o); return 0; } int odb_destroy(oda_ptr o) { return odb_read_destroy(o); } int odb_select_destroy(oda_ptr o) { delete reinterpret_cast(o); return 0; } int odb_writer_destroy(oda_writer_ptr o) { delete reinterpret_cast*>(o); return 0; } oda_read_iterator_ptr odb_create_read_iterator(oda_ptr co, const char* filename, int* err) { Reader* o(reinterpret_cast(co)); std::string fileName(filename); PathName fn(fileName); if (!fn.exists()) { *err = 2; // TODO: define error codes return 0; } ReaderIterator* iter(o->createReadIterator(fn)); *err = !iter; return oda_read_iterator_ptr(iter); } oda_select_iterator_ptr odb_create_select_iterator(oda_ptr co, const char* sql, int* err) { Select* o(reinterpret_cast(co)); try { SelectIterator* iter(o->createSelectIterator(sql)); *err = !iter; return oda_select_iterator_ptr(iter); } catch (eckit::CantOpenFile& e) { *err = 1; return 0; } catch (eckit::ReadError& e) { *err = 2; return 0; } catch (eckit::Exception& e) { *err = 3; eckit::Log::error() << "Caught exception: " << e << std::endl; return 0; } } oda_select_iterator_ptr odb_create_select_iterator_from_file(oda_ptr co, const char* sql, const char* filename, int* err) { Select* o(reinterpret_cast(co)); std::string full_sql(std::string(sql) + " from \"" + std::string(filename) + "\""); SelectIterator* iter(o->createSelectIterator(full_sql)); *err = !iter; return oda_select_iterator_ptr(iter); } int odb_read_iterator_destroy(oda_read_iterator_ptr it) { delete reinterpret_cast(it); return 0; } int odb_select_iterator_destroy(oda_select_iterator_ptr it) { delete reinterpret_cast(it); return 0; } int odb_read_iterator_get_no_of_columns(oda_read_iterator_ptr it, int* numberOfColumns) { ReaderIterator* iter(reinterpret_cast(it)); *numberOfColumns = iter->columns().size(); return 0; } int odb_read_iterator_get_column_size_doubles(oda_read_iterator_ptr it, int n, int* size) { ReaderIterator* iter(reinterpret_cast(it)); *size = iter->columns()[n]->dataSizeDoubles(); return 0; } int odb_read_iterator_get_column_offset(oda_read_iterator_ptr it, int n, int* offset) { ReaderIterator* iter(reinterpret_cast(it)); *offset = iter->dataOffset(n); return 0; } int odb_select_iterator_get_no_of_columns(oda_select_iterator_ptr it, int* numberOfColumns) { SelectIterator* iter(reinterpret_cast(it)); *numberOfColumns = iter->columns().size(); return 0; } int odb_select_iterator_get_column_size_doubles(oda_select_iterator_ptr it, int n, int* size) { SelectIterator* iter(reinterpret_cast(it)); *size = iter->columns()[n]->dataSizeDoubles(); return 0; } int odb_select_iterator_get_column_offset(oda_select_iterator_ptr it, int n, int* offset) { SelectIterator* iter(reinterpret_cast(it)); *offset = iter->dataOffset(n); return 0; } int odb_read_iterator_get_column_type(oda_select_iterator_ptr it, int n, int* type) { ReaderIterator* iter(reinterpret_cast(it)); *type = iter->columns()[n]->type(); return 0; } int odb_select_iterator_get_column_type(oda_select_iterator_ptr it, int n, int* type) { SelectIterator* iter(reinterpret_cast(it)); *type = iter->columns()[n]->type(); return 0; } int odb_read_iterator_get_column_name(oda_read_iterator_ptr it, int n, char** name, int* size_name) { ReaderIterator* iter(reinterpret_cast(it)); *name = const_cast(iter->columns()[n]->name().c_str()); *size_name = iter->columns()[n]->name().length(); return 0; } int odb_select_iterator_get_column_name(oda_select_iterator_ptr it, int n, char** name, int* size_name) { SelectIterator* iter(reinterpret_cast(it)); *name = const_cast(iter->columns()[n]->name().c_str()); *size_name = iter->columns()[n]->name().length(); return 0; } int odb_read_iterator_get_next_row(oda_read_iterator_ptr it, int count, double* data, int* new_dataset) { ReaderIterator* iter(reinterpret_cast(it)); if (!iter->next()) return 1; if (iter->isNewDataset()) *new_dataset = 1; else *new_dataset = 0; ASSERT(count >= 0); if (count != static_cast(iter->columns().size())) return 2; // TDOO: define error codes // ::memcpy(data, iter->data(), count * sizeof(data[0])); ::memcpy(data, iter->data(), iter->rowDataSizeDoubles() * sizeof(double)); return 0; } int odb_select_iterator_get_next_row(oda_select_iterator_ptr it, int count, double* data, int* new_dataset) { SelectIterator* iter(reinterpret_cast(it)); iter->setOutputRowBuffer(data); if (!iter->next()) return 1; if (iter->isNewDataset()) { *new_dataset = 1; } else { *new_dataset = 0; } return 0; } oda_write_iterator_ptr odb_create_append_iterator(oda_ptr co, const char* filename, int* err) { Writer<>* o(reinterpret_cast*>(co)); eckit::Length estimatedLength(0); DataHandle* fh = ODBAPISettings::instance().appendToFile(PathName(std::string(filename)), estimatedLength, true); // TODO: make sure there's no leaks (FileHandle) Writer<>::iterator_class* w(new Writer<>::iterator_class(*o, fh, false)); *err = !w; return oda_write_iterator_ptr(w); } oda_write_iterator_ptr odb_create_write_iterator(oda_ptr co, const char* filename, int* err) { Writer<>* o(reinterpret_cast*>(co)); eckit::Length estimatedLength(0); DataHandle* fh = ODBAPISettings::instance().writeToFile(PathName(std::string(filename)), estimatedLength, true); // TODO: make sure there's no leaks (FileHandle) Writer<>::iterator_class* w(new Writer<>::iterator_class(*o, fh, true)); *err = !w; return oda_write_iterator_ptr(w); } int odb_write_iterator_destroy(oda_write_iterator_ptr wi) { delete reinterpret_cast::iterator_class*>(wi); return 0; } int odb_write_iterator_set_no_of_columns(oda_write_iterator_ptr wi, int n) { Writer<>::iterator_class* w(reinterpret_cast::iterator_class*>(wi)); w->setNumberOfColumns(n); return 0; } int odb_write_iterator_set_column(oda_write_iterator_ptr wi, int index, int type, const char* name) { Writer<>::iterator_class* w(reinterpret_cast::iterator_class*>(wi)); return w->setColumn(index, std::string(name), ColumnType(type)); } int odb_write_iterator_set_bitfield(oda_write_iterator_ptr wi, int index, int type, const char* name, const char* bitfieldNames, const char* bitfieldSizes) { std::string bnames(bitfieldNames); std::string bsizes(bitfieldSizes); eckit::sql::FieldNames(bitfield_names); eckit::sql::Sizes(bitfield_sizes); // std::cout << " columnName = " << name << " " << bnames << " " << bsizes << std::endl; size_t iprev(0); for (size_t i(0); i < bnames.size(); i++) { if (bnames[i] == ':') { std::string name(bnames.substr(iprev, i - iprev)); iprev = i + 1; bitfield_names.push_back(name); } } iprev = 0; for (size_t i(0); i < bsizes.size(); i++) { if (bsizes[i] == ':') { std::string name(bsizes.substr(iprev, i - iprev)); size_t size(atof(name.c_str())); // bit[0-9]+ iprev = i + 1; bitfield_sizes.push_back(size); } } eckit::sql::BitfieldDef bitfieldType(make_pair(bitfield_names, bitfield_sizes)); Writer<>::iterator_class* w(reinterpret_cast::iterator_class*>(wi)); std::string columnName(name); int rc(w->setBitfieldColumn(index, columnName, ColumnType(type), bitfieldType)); return rc; } int odb_write_iterator_set_column_size_doubles(oda_write_iterator_ptr wi, int n, int size) { Writer<>::iterator_class* w(reinterpret_cast::iterator_class*>(wi)); w->columns()[n]->dataSizeDoubles(size); return 0; } int odb_write_iterator_get_row_buffer_size_doubles(oda_write_iterator_ptr wi, int* size) { Writer<>::iterator_class* w(reinterpret_cast::iterator_class*>(wi)); *size = w->rowDataSizeDoubles(); return 0; } int odb_write_iterator_get_column_offset(oda_write_iterator_ptr wi, int n, int* offset) { Writer<>::iterator_class* w(reinterpret_cast::iterator_class*>(wi)); *offset = w->dataOffset(n); return 0; } int odb_read_iterator_get_missing_value(oda_read_iterator_ptr ri, int index, double* value) { ReaderIterator* r(reinterpret_cast(ri)); if (index < 0 || long(r->columns().size()) < index) { std::stringstream ss; ss << "odb_read_iterator_get_missing_value: index " << index << " out of range, should be between 0 and " << r->columns().size(); throw UserError(ss.str()); } *value = r->columns()[index]->missingValue(); return 0; } int odb_read_iterator_get_row_buffer_size_doubles(oda_read_iterator_ptr ri, int* size) { ReaderIterator* r = reinterpret_cast(ri); *size = r->rowDataSizeDoubles(); return 0; } int odb_select_iterator_get_row_buffer_size_doubles(oda_select_iterator_ptr ri, int* size) { SelectIterator* r = reinterpret_cast(ri); *size = r->rowDataSizeDoubles(); return 0; } int odb_write_iterator_set_missing_value(oda_write_iterator_ptr wi, int index, double value) { Writer<>::iterator_class* w(reinterpret_cast::iterator_class*>(wi)); w->missingValue(index, value); return 0; } int odb_write_iterator_write_header(oda_write_iterator_ptr wi) { Writer<>::iterator_class* w(reinterpret_cast::iterator_class*>(wi)); w->writeHeader(); return 0; } int odb_write_iterator_set_next_row(oda_write_iterator_ptr wi, double* data, int count) { Writer<>::iterator_class* w(reinterpret_cast::iterator_class*>(wi)); return w->writeRow(data, count); } int odb_read_iterator_get_bitfield(oda_read_iterator_ptr it, int index, char** bitfield_names, char** bitfield_sizes, int* nSize, int* sSize) { return get_bitfield(it, index, bitfield_names, bitfield_sizes, nSize, sSize); } int odb_select_iterator_get_bitfield(oda_select_iterator_ptr it, int index, char** bitfield_names, char** bitfield_sizes, int* nSize, int* sSize) { return get_bitfield(it, index, bitfield_names, bitfield_sizes, nSize, sSize); } } // extern "C" int odb_select_iterator_get_missing_value(oda_select_iterator_ptr ri, int index, double* value) { SelectIterator* r(reinterpret_cast(ri)); if (index < 0 || long(r->columns().size()) < index) { std::stringstream ss; ss << "odb_select_iterator_get_missing_value: index " << index << " out of range, should be between 0 and " << r->columns().size(); throw UserError(ss.str()); } *value = r->columns()[index]->missingValue(); return 0; } odc-1.6.3/src/odc/MDSetter.cc0000664000175000017500000000066015146027420016007 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ namespace odc {} // namespace odc odc-1.6.3/src/odc/Block.cc0000664000175000017500000000202015146027420015342 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include #include "eckit/eckit.h" #include "eckit/io/Length.h" #include "eckit/io/Offset.h" #include "odc/Block.h" #include "odc/Reader.h" #include "odc/Select.h" #include "odc/core/MetaData.h" using namespace eckit; using namespace std; namespace odc { std::string Block::str() const { stringstream ss; ss << *this; return ss.str(); } ostream& operator<<(ostream& o, const Block& b) { o << "block,file=\"" << b.fileName << "\"," << "start=" << b.start << "," << "end=" << b.end << "," << "firstRow=" << b.firstRow << "," << "lastRow=" << b.lastRow; return o; } } // namespace odc odc-1.6.3/src/odc/RowsCounter.cc0000664000175000017500000000155015146027420016611 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "odc/RowsCounter.h" #include "eckit/eckit.h" #include "odc/Reader.h" #include "odc/core/MetaData.h" #include "odc/core/TablesReader.h" using namespace eckit; namespace odc { unsigned long long RowsCounter::fastRowCount(const PathName& db) { unsigned long long n = 0; core::TablesReader reader(db); auto it = reader.begin(); auto end = reader.end(); for (; it != end; ++it) n += it->rowCount(); return n; } } // namespace odc odc-1.6.3/src/odc/CMakeLists.txt0000664000175000017500000000677015146027420016561 0ustar alastairalastairconfigure_file( ODBAPIVersionSHA1.cc.in ODBAPIVersionSHA1.cc @ONLY ) configure_file( odc_config.f90.in odc_config.f90 ) list( APPEND odclib_src_files ${CMAKE_CURRENT_BINARY_DIR}/ODBAPIVersionSHA1.cc api/odc.h api/odc.cc api/Odb.h api/Odb.cc api/ColumnType.h api/ColumnInfo.h api/StridedData.h pyodbapi.h StringTool.cc StringTool.h CommandLineParser.cc CommandLineParser.h Comparator.cc Comparator.h ConstantSetter.cc ConstantSetter.h DispatchingWriter.cc DispatchingWriter.h MDI.cc MDI.h IteratorProxy.h LibOdc.cc LibOdc.h MDSetter.cc MDSetter.h MDUpdatingIterator.cc MDUpdatingIterator.h ODAHandle.cc ODAHandle.h ODATranslator.h ODAUpdatingIterator.cc ODAUpdatingIterator.h ODBAPISettings.cc ODBAPISettings.h ODBAPIVersion.cc ODBAPIVersion.h ODBApplication.cc ODBApplication.h Select.cc Select.h Reader.cc Reader.h ReaderIterator.cc ReaderIterator.h SelectIterator.cc SelectIterator.h TemplateParameters.cc TemplateParameters.h Writer.cc Writer.h WriterBufferingIterator.cc WriterBufferingIterator.h WriterDispatchingIterator.cc WriterDispatchingIterator.h odccapi.cc odccapi.h RowsCounter.h RowsCounter.cc Block.h Block.cc Indexer.h Indexer.cc RequestUtils.h RequestUtils.cc ODBTarget.cc ODBTarget.h core/Column.cc core/Column.h core/DataStream.h core/DecodeTarget.cc core/DecodeTarget.h core/Encoder.cc core/Encoder.h core/Exceptions.cc core/Exceptions.h core/Header.cc core/Header.h core/MetaData.cc core/MetaData.h core/Span.cc core/Span.h core/Table.cc core/Table.h core/TablesReader.cc core/TablesReader.h core/ThreadSharedDataHandle.cc core/ThreadSharedDataHandle.h core/Codec.cc core/Codec.h core/CodecFactory.h core/CodecFactory.cc codec/Constant.cc codec/Constant.h codec/Integer.cc codec/Integer.h codec/IntegerMissing.cc codec/IntegerMissing.h codec/String.cc codec/String.h codec/Real.cc codec/Real.h codec/CodecOptimizer.cc codec/CodecOptimizer.h csv/TextReader.cc csv/TextReader.h csv/TextReaderIterator.cc csv/TextReaderIterator.h sql/SQLOutputConfig.cc sql/SQLOutputConfig.h sql/SQLSelectOutput.cc sql/SQLSelectOutput.h sql/ODAOutput.cc sql/ODAOutput.h sql/TODATable.cc sql/TODATable.h sql/TODATableIterator.cc sql/TODATableIterator.h sql/Types.cc sql/Types.h utility/Tracer.cc utility/Tracer.h #iterator-api/BaseReader.cc #iterator-api/BaseReader.h ) # templates list( APPEND odclib_templates ConstantSetter.cc ODAUpdatingIterator.cc WriterDispatchingIterator.cc MDUpdatingIterator.cc ) # define odc library ecbuild_add_library( TARGET odccore INSTALL_HEADERS LISTED HEADER_DESTINATION ${INSTALL_INCLUDE_DIR}/odc SOURCES ${odclib_src_files} TEMPLATES ${odclib_templates} PUBLIC_INCLUDES $ $ PUBLIC_LIBS eckit_sql eckit ) ecbuild_add_library( TARGET fodc CONDITION HAVE_FORTRAN SOURCES api/odc.f90 ${CMAKE_CURRENT_BINARY_DIR}/odc_config.f90 PUBLIC_INCLUDES $ $ PUBLIC_LIBS odccore ) if ( HAVE_FORTRAN AND ECBUILD_INSTALL_FORTRAN_MODULES ) install( FILES ${CMAKE_Fortran_MODULE_DIRECTORY}/${CMAKE_CFG_INTDIR}/odc.mod DESTINATION module/odc COMPONENT modules ) endif() odc-1.6.3/src/odc/WriterDispatchingIterator.h0000664000175000017500000001060415146027420021325 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// /// \file WriterDispatchingIterator.h /// /// @author Piotr Kuchta, June 2009 #ifndef odc_WriterDispatchingIterator_H #define odc_WriterDispatchingIterator_H #include #include #include "eckit/sql/SQLTypedefs.h" #include "odc/TemplateParameters.h" #include "odc/Writer.h" #include "odc/api/ColumnType.h" namespace eckit { class PathName; } namespace eckit { class DataHandle; } namespace odc { class TemplateParameters; template class WriterDispatchingIterator { typedef std::vector Values; typedef std::map Values2IteratorIndex; typedef std::vector Iterators; public: WriterDispatchingIterator(OWNER& owner, int maxOpenFiles, bool append = false); ~WriterDispatchingIterator(); int open(); double* data(); double& data(size_t i); void setNumberOfColumns(size_t n); int setColumn(size_t index, std::string name, api::ColumnType type); int setBitfieldColumn(size_t index, std::string name, api::ColumnType type, eckit::sql::BitfieldDef b); void missingValue(size_t i, double); template unsigned long pass1(T&, const T&); template void verify(T&, const T&); unsigned long gatherStats(const double* values, unsigned long count); int close(); api::ColumnType columnType(size_t index); const std::string& columnName(size_t index) const; const std::string& codecName(size_t index) const; double columnMissingValue(size_t index); const core::MetaData& columns() const { return columns_; } const core::MetaData& columns(const core::MetaData& md); OWNER& owner() { return owner_; } void property(std::string key, std::string value); std::string property(std::string); // std::vector getFiles(); std::vector outputFiles(); TemplateParameters& templateParameters() { return templateParameters_; } size_t rowDataSizeDoubles() const; /// The offset of a given column in the doubles[] data array size_t dataOffset(size_t i) const { ASSERT(columnOffsets_); return columnOffsets_[i]; } // protected: void writeHeader(); int writeRow(const double* values, unsigned long count); // If we are encoding strings, and the relevant string column size changes, we need // to restart the encoding process void flushAndResetColumnSizes(const std::map& resetColumnSizeDoubles); protected: bool next(); /// Find iterator data should be dispatched to. WRITE_ITERATOR& dispatch(const double* values, unsigned long count); int dispatchIndex(const double* values, unsigned long count); int createIterator(const Values& dispatchedValues, const std::string& fileName); std::string generateFileName(const double* values, unsigned long count); OWNER& owner_; Writer iteratorsOwner_; core::MetaData columns_; double* lastValues_; double* nextRow_; size_t* columnOffsets_; unsigned long long nrows_; std::string outputFileTemplate_; core::Properties properties_; std::vector dispatchedIndexes_; Values2IteratorIndex values2iteratorIndex_; std::vector lastDispatch_; std::vector iteratorIndex2fileName_; Values lastDispatchedValues_; int lastIndex_; bool initialized_; bool append_; private: // No copy allowed. WriterDispatchingIterator(const WriterDispatchingIterator&); WriterDispatchingIterator& operator=(const WriterDispatchingIterator&); void parseTemplateParameters(); int refCount_; Iterators iterators_; std::vector files_; TemplateParameters templateParameters_; int maxOpenFiles_; std::map filesCreated_; friend class IteratorProxy, DispatchingWriter>; }; } // namespace odc #include "odc/WriterDispatchingIterator.cc" #endif odc-1.6.3/src/odc/Comparator.cc0000664000175000017500000002470215146027420016432 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "eckit/config/Resource.h" #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/PathName.h" #include "eckit/log/Log.h" #include "odc/Comparator.h" #include "odc/LibOdc.h" #include "odc/Reader.h" #include "odc/StringTool.h" #include "odc/core/Column.h" #include "odc/core/MetaData.h" #include "odc/utility/Tracer.h" #include #if __cplusplus >= 199711L #define isnan(x) std::isnan(x) #endif using namespace std; using namespace eckit; using namespace odc::api; using namespace odc::utility; using namespace odc::core; class ValuesDifferent : public Exception { public: ValuesDifferent(const std::string& what) : Exception(what) {} }; namespace odc { Comparator::Comparator(bool skipTestingHaveMissing) : skipTestingHaveMissing_(skipTestingHaveMissing), nRow_(0), NaN_isOK_(Resource("$odc_NAN_IS_OK", false)) {} void Comparator::compare(const PathName& p1, const PathName& p2) { std::vector noExcludedColumnTypes; std::vector noExcludedColumns; compare(p1, p2, noExcludedColumnTypes, noExcludedColumns); } void Comparator::compare(eckit::DataHandle& l, eckit::DataHandle& r) { std::vector noExcludedColumnTypes; std::vector noExcludedColumns; odc::Reader oda1(l); odc::Reader oda2(r); odc::Reader::iterator it1(oda1.begin()); odc::Reader::iterator end1(oda1.end()); odc::Reader::iterator it2(oda2.begin()); odc::Reader::iterator end2(oda2.end()); compare(it1, end1, it2, end2, "left", "right", noExcludedColumnTypes, noExcludedColumns); } void Comparator::compare(const PathName& p1, const PathName& p2, const std::vector& excludedColumnsTypes, const std::vector& excludedColumns) { Tracer t(Log::debug(), std::string("Comparator::compare: ") + p1 + ", " + p2); odc::Reader oda1(p1); odc::Reader oda2(p2); odc::Reader::iterator it1(oda1.begin()); odc::Reader::iterator end1(oda1.end()); odc::Reader::iterator it2(oda2.begin()); odc::Reader::iterator end2(oda2.end()); compare(it1, end1, it2, end2, p1, p2, excludedColumnsTypes, excludedColumns); } void Comparator::raiseNotEqual(const Column& column, double d1, double d2) { ColumnType type(column.type()); stringstream ss; ss << "Values different in column " << column.name() << ": " << StringTool::valueAsString(d1, type) << " is not equal " << StringTool::valueAsString(d2, type) << endl; throw ValuesDifferent(ss.str()); } void Comparator::compare(int nCols, const double* data1, const double* data2, const MetaData& md1, const MetaData& md2) { std::vector skipColsUnused; compare(nCols, data1, data2, md1, md2, skipColsUnused); } void Comparator::compare(int nCols, const double* data1, const double* data2, const MetaData& md1, const MetaData& md2, const std::vector& skipCols) { std::vector::const_iterator nextSkipCol = skipCols.begin(); const double* pdata1 = data1; const double* pdata2 = data2; unsigned long long numberOfDifferences(0); for (int i = 0; i < nCols; i++) { // Skip the specified columns if (nextSkipCol != skipCols.end() && (*nextSkipCol) == i) { ++nextSkipCol; pdata1 += md1[i]->dataSizeDoubles(); pdata2 += md2[i]->dataSizeDoubles(); continue; } try { const Column& column(*md1[i]); const Column& column2(*md2[i]); ColumnType type(column.type()); // Is this a missing value in both columns? bool isMissing1 = false; if (column.hasMissing() || skipTestingHaveMissing_) { double missing1 = column.missingValue(); const char* punnable_val1 = reinterpret_cast(pdata1); const char* punnable_missing1 = reinterpret_cast(&missing1); const uint64_t* punned_val1 = reinterpret_cast(punnable_val1); const uint64_t* punned_missing1 = reinterpret_cast(punnable_missing1); isMissing1 = (*punned_val1 == *punned_missing1); } bool isMissing2 = false; if (column2.hasMissing() || skipTestingHaveMissing_) { double missing2 = column2.missingValue(); const char* punnable_val2 = reinterpret_cast(pdata2); const char* punnable_missing2 = reinterpret_cast(&missing2); const uint64_t* punned_val2 = reinterpret_cast(punnable_val2); const uint64_t* punned_missing2 = reinterpret_cast(punnable_missing2); isMissing2 = (*punned_val2 == *punned_missing2); } if (isMissing1 != isMissing2) { raiseNotEqual(column, *pdata1, *pdata2); } if (!isMissing1 && !isMissing2) { switch (type) { case STRING: { size_t width1 = column.dataSizeDoubles() * sizeof(double); size_t width2 = column2.dataSizeDoubles() * sizeof(double); size_t len1 = ::strnlen(reinterpret_cast(pdata1), width1); size_t len2 = ::strnlen(reinterpret_cast(pdata2), width2); if (len1 != len2 || ::strncmp(reinterpret_cast(pdata1), reinterpret_cast(pdata2), len1)) { std::ostringstream ss; ss << "String values differ in column " << column.name() << ": " << std::string(reinterpret_cast(pdata1), len1) << " is not equal to " << std::string(reinterpret_cast(pdata2), len2) << std::endl; throw ValuesDifferent(ss.str()); } break; } case INTEGER: case BITFIELD: case DOUBLE: if (!(same(*pdata1, *pdata2) || (NaN_isOK_ && (::isnan(*pdata1) && ::isnan(*pdata2))))) raiseNotEqual(column, *pdata1, *pdata2); break; case REAL: if (!(same(float(*pdata1), float(*pdata2)) || (NaN_isOK_ && (::isnan(*pdata1) && ::isnan(*pdata2))))) raiseNotEqual(column, *pdata1, *pdata2); break; case IGNORE: default: ASSERT(!"Unknown type"); break; } } } catch (Exception& e) { ++numberOfDifferences; Log::info() << "While comparing rows number " << nRow_ << ", columns " << i << " found different." << std::endl; Log::info() << " " << e.what() << std::endl; Log::info() << " data1[" << i << "] = " << std::scientific << *pdata1 << std::endl; Log::info() << " data2[" << i << "] = " << std::scientific << *pdata2 << std::endl; Log::info() << " md1[" << i << "] = " << *md1[i] << std::endl; Log::info() << " md2[" << i << "] = " << *md2[i] << std::endl; // TODO: make it an option to stop when an error found // throw; } pdata1 += md1[i]->dataSizeDoubles(); pdata2 += md2[i]->dataSizeDoubles(); } if (numberOfDifferences) { stringstream ss; ss << "Files differ. "; // << numberOfDifferences << " difference" << ((numberOfDifferences == 1) ? "" : "s") // << " found."; throw Exception(ss.str()); } } void Comparator::compare(const MetaData& metaData1, const MetaData& metaData2, const std::set& excludedColumnsTypes, const std::set& excludedColumns, std::vector& skipCols) { ASSERT("Number of columns must be the same" && (metaData1.size() == metaData2.size())); // We keep track of which columns are skipped in this routine. skipCols.clear(); size_t size = metaData1.size(); for (size_t i = 0; i < size; i++) { Column& column1 = *metaData1[i]; Column& column2 = *metaData2[i]; try { ASSERT(column1.name() == column2.name()); // If we are skipping a column, then we should check nothing for it. if (excludedColumns.find(column1.name()) != excludedColumns.end()) { skipCols.push_back(i); continue; } if (excludedColumnsTypes.find(column1.name()) == excludedColumnsTypes.end()) { ASSERT(column1.type() == column2.type()); if (column1.type() == BITFIELD) if (!(column1.bitfieldDef() == column2.bitfieldDef())) { Log::error() << "Comparator::compare: bitfield definitions for column " << i << " '" << column1.name() << "' differ." << std::endl; ASSERT(column1.bitfieldDef() == column2.bitfieldDef()); } } if (!skipTestingHaveMissing_) { if (column1.hasMissing() != column2.hasMissing()) { Log::error() << "column1.hasMissing()=" << (column1.hasMissing() ? "true" : "false") << ", " << "column2.hasMissing()=" << (column2.hasMissing() ? "true" : "false") << std::endl; ASSERT(column1.hasMissing() == column2.hasMissing()); } } } catch (...) { Log::info() << "While comparing column " << i << ": " << column1.name() << std::endl; throw; } } } } // namespace odc odc-1.6.3/src/odc/MDI.cc0000664000175000017500000000102615146027420014726 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "odc/MDI.h" namespace odc { double MDI::realMDI_ = -2147483647; double MDI::integerMDI_ = 2147483647; } // namespace odc odc-1.6.3/src/odc/odccapi.h0000664000175000017500000001143215146027420015563 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef ODB_C_API_H /** * \file odbcapi.h * * @author Piotr Kuchta, March 2009 * */ #include #include // For off_t on Cray: #include #if defined(__cplusplus) || defined(c_plusplus) || defined(SWIGPYTHON) extern "C" { #endif typedef void* oda_ptr; typedef void* oda_read_iterator_ptr; typedef void* oda_select_iterator_ptr; typedef void* oda_writer_ptr; typedef void* oda_write_iterator_ptr; // typedef void oda; typedef void oda_read_iterator; typedef void oda_select_iterator; typedef void oda_writer; typedef void oda_write_iterator; void odb_start(); void odb_start_with_args(int argc, char* argv[]); unsigned int odb_get_headerBufferSize(); void odb_set_headerBufferSize(unsigned int); unsigned int odb_get_setvbufferSize(); void odb_set_setvbufferSize(unsigned int); int odc_version(const char**); int odc_git_sha1(const char**); unsigned int odc_format_version_major(); unsigned int odc_format_version_minor(); oda_ptr odb_read_create(const char*, int*); oda_ptr odb_create(const char*, int*); // Compatibility int odb_read_destroy(oda_ptr); int odb_destroy(oda_ptr); // Compatibility oda_read_iterator_ptr odb_create_read_iterator(oda_ptr, const char*, int*); int odb_read_iterator_destroy(oda_read_iterator_ptr); int odb_read_iterator_get_no_of_columns(oda_read_iterator_ptr, int*); int odb_read_iterator_get_column_size_doubles(oda_read_iterator_ptr, int, int*); int odb_read_iterator_get_column_offset(oda_read_iterator_ptr, int, int*); int odb_read_iterator_get_column_type(oda_read_iterator_ptr, int, int*); int odb_read_iterator_get_column_name(oda_read_iterator_ptr, int, char**, int*); int odb_read_iterator_get_bitfield(oda_read_iterator_ptr, int, char**, char**, int*, int*); int odb_read_iterator_get_next_row(oda_read_iterator_ptr, int, double*, int*); int odb_read_iterator_get_missing_value(oda_read_iterator_ptr, int, double*); int odb_read_iterator_get_row_buffer_size_doubles(oda_read_iterator_ptr, int*); oda_ptr odb_select_create(const char*, int*); int odb_select_destroy(oda_ptr); oda_select_iterator_ptr odb_create_select_iterator(oda_ptr, const char*, int*); oda_select_iterator_ptr odb_create_select_iterator_from_file(oda_ptr, const char*, const char*, int*); int odb_select_iterator_destroy(oda_select_iterator_ptr); int odb_select_iterator_get_no_of_columns(oda_select_iterator_ptr, int*); int odb_select_iterator_get_column_size_doubles(oda_read_iterator_ptr, int, int*); int odb_select_iterator_get_column_offset(oda_read_iterator_ptr, int, int*); int odb_select_iterator_get_column_type(oda_select_iterator_ptr, int, int*); int odb_select_iterator_get_column_name(oda_select_iterator_ptr, int, char**, int*); int odb_select_iterator_get_bitfield(oda_select_iterator_ptr, int, char**, char**, int*, int*); int odb_select_iterator_get_next_row(oda_select_iterator_ptr, int, double*, int*); int odb_select_iterator_get_row_buffer_size_doubles(oda_read_iterator_ptr, int*); int odb_select_iterator_get_missing_value(oda_select_iterator_ptr ri, int index, double* value); oda_writer_ptr odb_writer_create(const char*, int*); int odb_writer_destroy(oda_writer_ptr); int odb_write_iterator_destroy(oda_write_iterator_ptr); oda_write_iterator_ptr odb_create_write_iterator(oda_writer_ptr, const char*, int*); oda_write_iterator_ptr odb_create_append_iterator(oda_writer_ptr, const char*, int*); int odb_write_iterator_set_no_of_columns(oda_write_iterator_ptr, int); int odb_write_iterator_set_column(oda_write_iterator_ptr, int, int, const char*); int odb_write_iterator_set_bitfield(oda_write_iterator_ptr, int, int, const char*, const char*, const char*); int odb_write_iterator_set_column_size_doubles(oda_write_iterator_ptr, int, int); int odb_write_iterator_get_row_buffer_size_doubles(oda_write_iterator_ptr, int*); int odb_write_iterator_get_column_offset(oda_write_iterator_ptr, int, int*); int odb_write_iterator_set_missing_value(oda_write_iterator_ptr, int, double); int odb_write_iterator_write_header(oda_write_iterator_ptr); int odb_write_iterator_set_next_row(oda_write_iterator_ptr, double*, int); // FIXME: This needs to be changed: return error code like all the rest of the functions double odb_count(const char*); int get_blocks_offsets(const char* fileName, size_t* numberOfBlocks, off_t** offsets, size_t** sizes); int release_blocks_offsets(off_t**); int release_blocks_sizes(size_t**); #if defined(__cplusplus) || defined(c_plusplus) || defined(SWIGPYTHON) } #endif #endif odc-1.6.3/src/odc/core/0000775000175000017500000000000015146027420014737 5ustar alastairalastairodc-1.6.3/src/odc/core/ThreadSharedDataHandle.h0000664000175000017500000000441415146027420021357 0ustar alastairalastair/* * (C) Copyright 1996-2018 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Simon Smart /// @date Dec 2018 #ifndef odc_core_ThreadSharedDataHandle_H #define odc_core_ThreadSharedDataHandle_H #include #include #include "eckit/io/DataHandle.h" namespace eckit { class DataHandle; } namespace odc { namespace core { //---------------------------------------------------------------------------------------------------------------------- class ThreadSharedDataHandle : public eckit::DataHandle { public: // methods ThreadSharedDataHandle(eckit::DataHandle& dh); ThreadSharedDataHandle(eckit::DataHandle* dh); ~ThreadSharedDataHandle() override; ThreadSharedDataHandle(const ThreadSharedDataHandle&); ThreadSharedDataHandle& operator=(const ThreadSharedDataHandle&); ThreadSharedDataHandle(ThreadSharedDataHandle&&); ThreadSharedDataHandle& operator=(ThreadSharedDataHandle&&); bool operator!=(const ThreadSharedDataHandle& other); bool operator==(const ThreadSharedDataHandle& other); void print(std::ostream& s) const override; eckit::Length openForRead() override; void openForWrite(const eckit::Length&) override; void openForAppend(const eckit::Length&) override; long read(void*, long) override; long write(const void*, long) override; void close() override; eckit::Length estimate() override; eckit::Offset position() override; eckit::Offset seek(const eckit::Offset&) override; std::string title() const override; private: // members struct Internal { Internal(eckit::DataHandle* dh, bool owned); ~Internal(); std::mutex m_; eckit::DataHandle* dh_; bool owned_; }; std::shared_ptr internal_; eckit::Offset position_; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace core } // namespace odc #endif odc-1.6.3/src/odc/core/DecodeTarget.h0000664000175000017500000000263315146027420017446 0ustar alastairalastair/* * (C) Copyright 1996-2018 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Simon Smart /// @date Dec 2018 #ifndef odc_core_DecodeTarget_H #define odc_core_DecodeTarget_H #include #include "odc/api/StridedData.h" namespace odc { namespace core { //---------------------------------------------------------------------------------------------------------------------- class DecodeTarget { public: // methods DecodeTarget(const std::vector& columns, const std::vector& facades); DecodeTarget(const std::vector& columns, std::vector&& facades); ~DecodeTarget(); const std::vector& columns() const; std::vector& dataFacades(); DecodeTarget slice(size_t rowOffset, size_t nrows); private: // members std::vector columns_; std::vector columnFacades_; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace core } // namespace odc #endif odc-1.6.3/src/odc/core/Table.h0000664000175000017500000000503415146027420016141 0ustar alastairalastair/* * (C) Copyright 1996-2018 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Simon Smart /// @date Dec 2018 #ifndef odc_core_Table_H #define odc_core_Table_H #include #include "eckit/io/Buffer.h" #include "odc/core/MetaData.h" #include "odc/core/Span.h" #include "odc/core/ThreadSharedDataHandle.h" namespace eckit { class DataHandle; } namespace odc { namespace core { class DecodeTarget; //---------------------------------------------------------------------------------------------------------------------- class Table { public: // methods // Construct a table. This is a static function rather than a constructor // so that we can return false for no-more-data rather than throwing an // exception static std::unique_ptr readTable(ThreadSharedDataHandle& dh); eckit::Offset startPosition() const; eckit::Offset nextPosition() const; eckit::Length encodedDataSize() const; size_t rowCount() const; size_t columnCount() const; int32_t byteOrder() const; bool otherByteOrder() const; const MetaData& columns() const; const Properties& properties() const; eckit::Buffer readEncodedData(bool includeHeader = false); void decode(DecodeTarget& target); Span span(const std::vector& columns, bool onlyConstant = false); Span decodeSpan(const std::vector& columns); private: // methods Table(const ThreadSharedDataHandle& dh); /// Lookups used for decoding. Memoised for efficiency const std::map& columnLookup(); const std::map& simpleColumnLookup(); private: // members ThreadSharedDataHandle dh_; eckit::Offset startPosition_; eckit::Offset dataPosition_; eckit::Length headerSize_; eckit::Length dataSize_; eckit::Offset nextPosition_; int32_t byteOrder_; MetaData metadata_; Properties properties_; // Lookups. Memoised for efficiency std::map columnLookup_; std::map simpleColumnLookup_; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace core } // namespace odc #endif odc-1.6.3/src/odc/core/Header.cc0000664000175000017500000001554015146027420016443 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "odc/core/Header.h" #include "eckit/io/Buffer.h" #include "eckit/io/DataHandle.h" #include "eckit/log/Log.h" #include "eckit/types/FixedString.h" #include "eckit/utils/MD5.h" #include "odc/LibOdc.h" #include "odc/ODBAPISettings.h" #include "odc/core/DataStream.h" #include "odc/core/Exceptions.h" #include "odc/core/MetaData.h" using namespace eckit; namespace odc { namespace core { //---------------------------------------------------------------------------------------------------------------------- Header::Header(MetaData& md, Properties& props) : md_(md), props_(props), dataSize_(0), rowsNumber_(0), byteOrder_(BYTE_ORDER_INDICATOR) {} Header::~Header() {} bool Header::readMagic(DataHandle& dh) { eckit::FixedString<5> magic; long bytesRead = dh.read(&magic, sizeof(magic)); if (bytesRead == 0 || bytesRead == EOF) return false; if (bytesRead != sizeof(magic)) throw ODBIncomplete(dh.title(), Here()); if (magic != "\xff\xffODA") throw ODBInvalid(dh.title(), "Incorrect MAGIC", Here()); return true; } template void Header::load(DataHandle& dh) { // There must be at least 56 bytes available to read the basic header. constexpr size_t basic_header_size = 12 + 32 + 4; char basicBuffer[basic_header_size]; if (dh.read(basicBuffer, sizeof(basicBuffer)) != sizeof(basicBuffer)) throw ODBIncomplete(dh.title(), Here()); DataStream ds1(basicBuffer, sizeof(basicBuffer)); int32_t formatVersionMajor; ds1.read(formatVersionMajor); ASSERT("File format version not supported" && formatVersionMajor <= FORMAT_VERSION_NUMBER_MAJOR); int32_t formatVersionMinor; ds1.read(formatVersionMinor); ASSERT("File format version not supported" && formatVersionMinor <= FORMAT_VERSION_NUMBER_MINOR && formatVersionMinor > 3); std::string headerDigest; ds1.read(headerDigest); int32_t headerSize; ds1.read(headerSize); // Read the remaining header data eckit::Buffer buffer(headerSize); if (dh.read(buffer, headerSize) != headerSize) throw ODBIncomplete(dh.title(), Here()); // Calculate the MD5 MD5 md5; md5.add(buffer.data(), buffer.size()); std::string actualHeaderDigest = md5.digest(); if (headerDigest != actualHeaderDigest) throw ODBInvalid(dh.title(), "Header digest incorrect", Here()); DataStream ds2(buffer, buffer.size()); // 0 means we don't know offset of next header. int64_t nextFrameOffset; ds2.read(nextFrameOffset); dataSize_ = nextFrameOffset; md_.dataSize(dataSize_); // Reserved, not used yet. int64_t prevFrameOffset; ds2.read(prevFrameOffset); ASSERT(prevFrameOffset == 0); // TODO: increase file format version int64_t numberOfRows; ds2.read(numberOfRows); rowsNumber_ = numberOfRows; md_.rowsNumber(rowsNumber_); LOG_DEBUG_LIB(LibOdc) << "Header::load: numberOfRows = " << numberOfRows << std::endl; // Flags -> ODAFlags Flags flags; ds2.read(flags); ds2.read(props_); md_.load(ds2); } void Header::loadAfterMagic(DataHandle& dh) { if (dh.read(&byteOrder_, sizeof(byteOrder_)) != sizeof(byteOrder_)) { throw ODBIncomplete(dh.title(), Here()); } if (byteOrder_ != BYTE_ORDER_INDICATOR) { load(dh); } else { load(dh); } } namespace { template std::pair serializeHeaderInternal(size_t dataSize, size_t rowsNumber, const Properties& properties, const MetaData& columns) { // Serialise the variable size part of the header first. Use the configured buffer size // but allow expansion if needed. constexpr size_t initial_header_size = 9 + 8 + 4 + 32 + 4; eckit::Buffer buffer(ODBAPISettings::instance().headerBufferSize()); bool serialised = false; char* variableHeaderStart = 0; int32_t variableHeaderSize; while (!serialised) { try { variableHeaderStart = buffer + initial_header_size; DataStream ds(variableHeaderStart, buffer.size() - initial_header_size); ds.write(static_cast(dataSize)); // Reserved: nextFrameOffset ds.write(static_cast(0)); // Reserved: prevFrameOffset ds.write(static_cast(rowsNumber)); // num. rows Flags flags(0); // n.b. flags unused ds.write(flags); ds.write(properties); columns.save(ds); serialised = true; variableHeaderSize = ds.position(); } catch (ODBEndOfDataStream& e) { buffer = eckit::Buffer(buffer.size() * 2); } } // Calculate MD5 of the variable portion of header data MD5 md5; md5.add(variableHeaderStart, variableHeaderSize); std::string headerDigest = md5.digest(); // Now Serialise everything into the final buffer DataStream ds(buffer.data(), initial_header_size); // Header. ds.write(static_cast(ODA_MAGIC_NUMBER)); // MAGIC ds.write('O'); // MAGIC ds.write('D'); // MAGIC ds.write('A'); // MAGIC ds.write(static_cast(BYTE_ORDER_INDICATOR)); ds.write(static_cast(FORMAT_VERSION_NUMBER_MAJOR)); ds.write(static_cast(FORMAT_VERSION_NUMBER_MINOR)); ds.write(headerDigest); // MD5 ds.write(static_cast(variableHeaderSize)); // How much header data follows ASSERT(ds.position() == eckit::Offset(initial_header_size)); return std::make_pair(std::move(buffer), variableHeaderSize + ds.position()); } } // namespace std::pair Header::serializeHeader(size_t dataSize, size_t rowsNumber, const Properties& properties, const MetaData& columns) { return serializeHeaderInternal(dataSize, rowsNumber, properties, columns); } std::pair Header::serializeHeaderOtherByteOrder(size_t dataSize, size_t rowsNumber, const Properties& properties, const MetaData& columns) { return serializeHeaderInternal(dataSize, rowsNumber, properties, columns); } //---------------------------------------------------------------------------------------------------------------------- } // namespace core } // namespace odc odc-1.6.3/src/odc/core/Codec.cc0000664000175000017500000000653515146027420016274 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "odc/core/Codec.h" #include "eckit/exception/Exceptions.h" #include "odc/core/CodecFactory.h" using namespace eckit; namespace odc { namespace core { //---------------------------------------------------------------------------------------------------------------------- Codec::Codec(const std::string& name, api::ColumnType type) : name_(name), hasMissing_(false), missingValue_(odc::MDI::realMDI()), min_(missingValue_), max_(missingValue_), type_(type) {} std::unique_ptr Codec::clone() { auto c = CodecFactory::instance().build(name_, type_); c->hasMissing_ = hasMissing_; c->missingValue_ = missingValue_; c->min_ = min_; c->max_ = max_; return c; } Codec::~Codec() {} void Codec::setDataStream(GeneralDataStream& ds) { if (ds.isOther()) { setDataStream(ds.other()); } else { setDataStream(ds.same()); } } void Codec::setDataStream(DataStream&) { throw eckit::SeriousBug("Mismatched byte order between DataStream and Codec", Here()); } void Codec::setDataStream(DataStream&) { throw eckit::SeriousBug("Mismatched byte order between DataStream and Codec", Here()); } void Codec::load(GeneralDataStream& ds) { if (ds.isOther()) { load(ds.other()); } else { load(ds.same()); } } void Codec::load(DataStream&) { throw eckit::SeriousBug("Mismatched byte order between DataStream and Codec", Here()); } void Codec::load(DataStream&) { throw eckit::SeriousBug("Mismatched byte order between DataStream and Codec", Here()); } void Codec::save(GeneralDataStream& ds) { if (ds.isOther()) { save(ds.other()); } else { save(ds.same()); } } void Codec::save(DataStream&) { throw eckit::SeriousBug("Mismatched byte order between DataStream and Codec", Here()); } void Codec::save(DataStream&) { throw eckit::SeriousBug("Mismatched byte order between DataStream and Codec", Here()); } void Codec::missingValue(double v) { ASSERT("Cannot change missing value after encoding of column data started" && (min_ == missingValue_) && (max_ == missingValue_)); min_ = max_ = missingValue_ = v; } void Codec::gatherStats(const double& v) { if (v == missingValue_) { hasMissing_ = 1; } else { if (v < min_ || min_ == missingValue_) min_ = v; if (v > max_ || max_ == missingValue_) max_ = v; } } void Codec::print(std::ostream& s) const { s << name_ << ", range=<" << std::fixed << min_ << "," << max_ << ">" << ", hasMissing=" << (hasMissing_ ? "true" : "false"); if (hasMissing_) { s << ", missingValue=" << missingValue_; } } //---------------------------------------------------------------------------------------------------------------------- } // namespace core } // namespace odc odc-1.6.3/src/odc/core/DecodeTarget.cc0000664000175000017500000000315215146027420017601 0ustar alastairalastair/* * (C) Copyright 1996-2018 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "odc/core/DecodeTarget.h" namespace odc { namespace core { //---------------------------------------------------------------------------------------------------------------------- DecodeTarget::DecodeTarget(const std::vector& columns, const std::vector& facades) : columns_(columns), columnFacades_(facades) {} DecodeTarget::DecodeTarget(const std::vector& columns, std::vector&& facades) : columns_(columns), columnFacades_(std::move(facades)) {} DecodeTarget::~DecodeTarget() {} const std::vector& DecodeTarget::columns() const { return columns_; } std::vector& DecodeTarget::dataFacades() { return columnFacades_; } DecodeTarget DecodeTarget::slice(size_t rowOffset, size_t nrows) { std::vector newFacades; newFacades.reserve(columnFacades_.size()); for (auto& facade : columnFacades_) { newFacades.emplace_back(facade.slice(rowOffset, nrows)); } return DecodeTarget(columns_, std::move(newFacades)); } //---------------------------------------------------------------------------------------------------------------------- } // namespace core } // namespace odc odc-1.6.3/src/odc/core/Encoder.cc0000664000175000017500000001011015146027420016616 0ustar alastairalastair/* * (C) Copyright 1996-2018 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "odc/core/Encoder.h" #include "odc/LibOdc.h" #include "odc/codec/CodecOptimizer.h" #include "odc/core/Header.h" using namespace eckit; namespace odc { namespace core { //---------------------------------------------------------------------------------------------------------------------- void encodeFrame(eckit::DataHandle& out, const std::vector& columns, const std::vector& data, const std::map& properties) { ASSERT(columns.size() == data.size()); ASSERT(columns.size() > 0); MetaData md; size_t ncols = columns.size(); size_t nrows = data[0].nelem(); // Construct the default codecs md.setSize(ncols); for (size_t i = 0; i < ncols; ++i) { md[i]->name(columns[i].name); md[i]->type(columns[i].type); ASSERT(columns[i].decodedSize % sizeof(double) == 0); md[i]->dataSizeDoubles(columns[i].decodedSize / sizeof(double)); if (!columns[i].bitfield.empty()) { eckit::sql::BitfieldDef bf; for (const auto& bit : columns[i].bitfield) { bf.first.push_back(bit.name); bf.second.push_back(bit.size); } md[i]->bitfieldDef(bf); } } // Gather statistics over all the columns size_t maxRowSize = sizeof(uint16_t); // all rows contain a marker for (size_t col = 0; col < ncols; ++col) { ASSERT(data[col].nelem() == nrows); Codec& coder(md[col]->coder()); for (const char* d : data[col]) { coder.gatherStats(*reinterpret_cast(d)); } maxRowSize += data[col].dataSize(); } // Optimise the codecs codec::CodecOptimizer().setOptimalCodecs(md); // TODO: Sort the columns into the optimal order for encoding // std::sort(md.begin(), md.end(), [](Column* a, Column* b) { ASSERT(false); }); // TODO: Sort the data columns as well. // ASSERT(false); // Encode the data const std::vector& sortedData(data); std::vector coders; for (const auto& col : md) coders.push_back(&col->coder()); Buffer encodedBuffer(maxRowSize * nrows); DataStream encodedStream(encodedBuffer); for (size_t row = 0; row < nrows; row++) { size_t startCol = 0; if (row != 0) { for (; startCol < ncols; ++startCol) { if (sortedData[startCol].isNewValue(row)) break; } } // Write the marker uint8_t marker[2]{static_cast((startCol / 256) % 256), static_cast(startCol % 256)}; encodedStream.writeBytes(marker, sizeof(marker)); // n.b. raw write // Write the updated values char* p = encodedStream.get(); for (size_t col = startCol; col < ncols; col++) { p = coders[col]->encode(p, *reinterpret_cast(sortedData[col].get(row))); } encodedStream.set(p); } // Encode the header Properties props{properties}; props["encoder"] = std::string("odc version ") + LibOdc::instance().version(); std::pair encodedHeader = Header::serializeHeader(encodedStream.position(), nrows, props, md); // And output the data ASSERT(out.write(encodedHeader.first, encodedHeader.second) == long(encodedHeader.second)); ASSERT(out.write(encodedBuffer, encodedStream.position()) == encodedStream.position()); } //---------------------------------------------------------------------------------------------------------------------- } // namespace core } // namespace odc odc-1.6.3/src/odc/core/Column.cc0000664000175000017500000000742215146027420016510 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "odc/core/Column.h" #include "eckit/utils/StringTools.h" using namespace eckit; using namespace odc::api; namespace odc { namespace core { //---------------------------------------------------------------------------------------------------------------------- Column::Column(MetaData& owner) : owner_(owner), name_(), type_(IGNORE), bitfieldDef_() {} Column::Column(const Column& o) : owner_(o.owner_), name_(o.name_), type_(o.type_), bitfieldDef_(o.bitfieldDef_) { *this = o; } Column::~Column() {} Column& Column::operator=(const Column& other) { name(other.name()); // type >(other.type(), false); type_ = other.type_; if (type_ == BITFIELD) bitfieldDef(other.bitfieldDef()); if (other.coder_) { coder_ = other.coder_->clone(); } else { coder_.reset(); } // hasMissing(other.hasMissing()); // missingValue(other.missingValue()); return *this; } const char* Column::columnTypeName(ColumnType type) { switch (type) { case IGNORE: return "IGNORE"; case INTEGER: return "INTEGER"; case REAL: return "REAL"; case DOUBLE: return "DOUBLE"; case STRING: return "STRING"; case BITFIELD: return "BITFIELD"; default: return "UNKNOWN_TYPE"; } } ColumnType Column::type(const std::string& t) { std::string ut(StringTools::upper(t)); if (ut == "IGNORE") return IGNORE; if (ut == "INTEGER") return INTEGER; if (ut == "REAL") return REAL; if (ut == "DOUBLE") return DOUBLE; if (ut == "STRING") return STRING; if (ut == "BITFIELD") return BITFIELD; Log::error() << "Unknown type: '" << t << "'" << std::endl; ASSERT(0 && "Unknown type"); return IGNORE; } bool Column::isConstant() { // FIXME return coder().name() == "constant" || coder().name() == "constant_string" || coder().name() == "long_constant_string"; } // Column::Column(DataHandle *dataHandle) : dataHandle_(dataHandle), name_(), type_(IGNORE/*?*/), coder_(0) {} /// return true if names and types are the same; do not compare codecs. bool Column::operator==(const Column& other) const { return equals(other); } bool Column::equals(const Column& other, bool compareDataSizes) const { if (name() == other.name() && type() == other.type()) { if (compareDataSizes) { return (dataSizeDoubles() == other.dataSizeDoubles()); } else { return true; } } return false; } void Column::print(std::ostream& s) const { s << "name: " << name_ << ", "; s << "type: " << columnTypeName(ColumnType(type_)); if (type_ == BITFIELD) { eckit::sql::FieldNames names = bitfieldDef_.first; eckit::sql::Sizes sizes = bitfieldDef_.second; ASSERT(names.size() == sizes.size()); s << " ["; for (size_t i = 0; i < names.size(); ++i) s << names[i] << ":" << sizes[i] << (i != names.size() - 1 ? ";" : ""); s << "] "; } s << ", "; s << "codec: "; if (coder_) s << *coder_; else s << "NONE"; } //---------------------------------------------------------------------------------------------------------------------- } // namespace core } // namespace odc odc-1.6.3/src/odc/core/CodecFactory.cc0000664000175000017500000000315615146027420017620 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "odc/core/CodecFactory.h" #include "eckit/exception/Exceptions.h" namespace odc { namespace core { //---------------------------------------------------------------------------------------------------------------------- CodecFactory::CodecFactory() {} CodecFactory::~CodecFactory() {} CodecFactory& CodecFactory::instance() { static CodecFactory theInstance; return theInstance; } void CodecFactory::enregister(const std::string& name, CodecBuilderBase& builder) { std::lock_guard lock(m_); ASSERT(builders_.find(name) == builders_.end()); builders_.emplace(name, builder); } void CodecFactory::deregister(const std::string& name, CodecBuilderBase& builder) { std::lock_guard lock(m_); auto it = builders_.find(name); ASSERT(it != builders_.end()); builders_.erase(it); } CodecBuilderBase::CodecBuilderBase(const std::string& name) : name_(name) { CodecFactory::instance().enregister(name, *this); } CodecBuilderBase::~CodecBuilderBase() { CodecFactory::instance().deregister(name_, *this); } //---------------------------------------------------------------------------------------------------------------------- } // namespace core } // namespace odc odc-1.6.3/src/odc/core/Exceptions.cc0000664000175000017500000000320615146027420017370 0ustar alastairalastair/* * (C) Copyright 1996-2018 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "odc/core/Exceptions.h" namespace odc { namespace core { //---------------------------------------------------------------------------------------------------------------------- ODBDecodeError::ODBDecodeError(const std::string& s, const eckit::CodeLocation& loc) : Exception(std::string("ODB decode failure: ") + s, loc) {} ODBIncomplete::ODBIncomplete(const std::string& s, const eckit::CodeLocation& loc) : ODBDecodeError(std::string("Unexpected end of file: ") + s, loc) {} ODBInvalid::ODBInvalid(const std::string& file, const std::string& reason, const eckit::CodeLocation&) : ODBDecodeError(std::string("Invalid ODB (") + file + ") -- " + reason, Here()) {} ODBEndOfDataStream::ODBEndOfDataStream(const std::string& s, const eckit::CodeLocation& loc) : ODBDecodeError(s, loc) {} AmbiguousColumnException::AmbiguousColumnException(const std::string& columnName) : UserError("Ambiguous column name: '" + columnName + "'") {} ColumnNotFoundException::ColumnNotFoundException(const std::string& columnName) : UserError("Column '" + columnName + "' not found.") {} //---------------------------------------------------------------------------------------------------------------------- } // namespace core } // namespace odc odc-1.6.3/src/odc/core/Span.h0000664000175000017500000000473015146027420016015 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef odc_core_Span_H #define odc_core_Span_H #include #include #include #include "eckit/io/Length.h" #include "eckit/io/Offset.h" #include "odc/api/ColumnType.h" namespace odc { namespace core { //---------------------------------------------------------------------------------------------------------------------- // Describe the data extent, and the keys, spanned by an ODB class Span { public: // methods Span(eckit::Offset start, eckit::Length length); ~Span(); eckit::Offset offset() const { return start_; } eckit::Length length() const { return length_; } void extend(const Span& other); void extend(eckit::Length length); // Add the values _as_a_double_. Not nice, but compatible with old decoding api void addValue(const std::string& column, api::ColumnType t, double val); void addValues(const std::string& column, const std::set& vals); void addValues(const std::string& column, const std::set& vals); void addValues(const std::string& column, const std::set& vals); bool operator==(const Span& rhs) const; template void visit(T& visitor) { for (const auto& kv : integerValues_) visitor(kv.first, kv.second); for (const auto& kv : realValues_) visitor(kv.first, kv.second); for (const auto& kv : stringValues_) visitor(kv.first, kv.second); } const std::set& getIntegerValues(const std::string& column) const; const std::set& getRealValues(const std::string& column) const; const std::set& getStringValues(const std::string& column) const; private: // members eckit::Offset start_; eckit::Length length_; std::map> integerValues_; std::map> realValues_; std::map> stringValues_; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace core } // namespace odc #endif odc-1.6.3/src/odc/core/TablesReader.cc0000664000175000017500000000677515146027420017622 0ustar alastairalastair/* * (C) Copyright 1996-2018 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #include "odc/core/TablesReader.h" #include "eckit/exception/Exceptions.h" using namespace eckit; namespace odc { namespace core { //---------------------------------------------------------------------------------------------------------------------- ReadTablesIterator::ReadTablesIterator(TablesReader& owner, long pos) : owner_(owner), pos_(pos) { // Ensure the first table is loaded if (pos_ != -1) { if (!owner_.get().ensureTable(0)) pos_ = -1; } } bool ReadTablesIterator::operator!=(const ReadTablesIterator& other) { return !(*this == other); } bool ReadTablesIterator::operator==(const ReadTablesIterator& other) { return (&owner_.get() == &other.owner_.get()) && (pos_ == other.pos_); } ReadTablesIterator& ReadTablesIterator::operator++() { ++pos_; if (!owner_.get().ensureTable(pos_)) { pos_ = -1; } return *this; } ReadTablesIterator ReadTablesIterator::operator++(int) { auto copy = *this; ++(*this); return copy; } Table* ReadTablesIterator::operator->() { ASSERT(pos_ != -1); return &owner_.get().getTable(pos_); } const Table* ReadTablesIterator::operator->() const { ASSERT(pos_ != -1); return &owner_.get().getTable(pos_); } Table& ReadTablesIterator::operator*() { ASSERT(pos_ != -1); return owner_.get().getTable(pos_); } const Table& ReadTablesIterator::operator*() const { ASSERT(pos_ != -1); return owner_.get().getTable(pos_); } //---------------------------------------------------------------------------------------------------------------------- TablesReader::TablesReader(DataHandle& dh) : dh_(dh) {} TablesReader::TablesReader(DataHandle* dh) : dh_(dh) {} TablesReader::TablesReader(const PathName& path) : TablesReader(path.fileHandle()) {} TablesReader::iterator TablesReader::begin() { return ReadTablesIterator(*this); } TablesReader::iterator TablesReader::end() { return ReadTablesIterator(*this, -1); } bool TablesReader::ensureTable(long idx) { // We only read-ahead by a maximum of one table std::lock_guard lock(m_); ASSERT(idx >= 0); ASSERT(idx <= long(tables_.size())); if (idx == long(tables_.size())) { // n.b. Some DataHandles don't implement estimate() --> accept "0" Offset nextPosition = (tables_.empty() ? Offset(0) : tables_.back()->nextPosition()); ASSERT(nextPosition <= dh_.estimate() || dh_.estimate() == Length(0)); // If the table has been truncated, this is an error, and we cannot read on. Offset pos = dh_.seek(nextPosition); if (pos < nextPosition) { throw ODBIncomplete(dh_.title(), Here()); } std::unique_ptr
tbl(Table::readTable(dh_)); if (!tbl) return false; tables_.emplace_back(std::move(tbl)); } return true; } Table& TablesReader::getTable(long idx) { ASSERT(idx >= 0); ASSERT(idx < long(tables_.size())); return *tables_[idx]; } //---------------------------------------------------------------------------------------------------------------------- } // namespace core } // namespace odc odc-1.6.3/src/odc/core/TablesReader.h0000664000175000017500000000566515146027420017461 0ustar alastairalastair/* * (C) Copyright 1996-2018 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Simon Smart /// @date Dec 2018 #ifndef odc_core_ReadTablesIterator_H #define odc_core_ReadTablesIterator_H #include #include #include "odc/core/Table.h" #include "odc/core/ThreadSharedDataHandle.h" namespace eckit { class DataHandle; } namespace odc { namespace core { //---------------------------------------------------------------------------------------------------------------------- class TablesReader; class ReadTablesIterator { public: // methods ReadTablesIterator(TablesReader& owner, long pos = 0); // Functionality to work as an iterator bool operator!=(const ReadTablesIterator& other); bool operator==(const ReadTablesIterator& other); ReadTablesIterator& operator++(); ReadTablesIterator operator++(int); Table* operator->(); const Table* operator->() const; Table& operator*(); const Table& operator*() const; private: // methods friend std::ostream& operator<<(std::ostream& os, const ReadTablesIterator& rti) { os << "ReadTablesIterator(" << &rti.owner_ << ", " << rti.pos_ << ")"; return os; } private: // members std::reference_wrapper owner_; long pos_; }; //---------------------------------------------------------------------------------------------------------------------- // TablesReader builds (in a thread safe way) a list of tables that can be queried by // the iterator. This is done lazily. As a result: // // i) Iterators can be instantiated multiple times, but the table structure will only be // read once // // ii) If data is encoded on the fly, it is all read in one pass. This means that for // straightforward workflows, this will work on streaming data that is larger than memory. class TablesReader { public: // types using iterator = ReadTablesIterator; public: // methods TablesReader(eckit::DataHandle& dh); TablesReader(eckit::DataHandle* dh); // n.b. takes ownership TablesReader(const eckit::PathName& path); iterator begin(); iterator end(); private: // members friend class ReadTablesIterator; bool ensureTable(long idx); Table& getTable(long idx); private: // members std::mutex m_; // Vector of pointers --> objects don't move if vector changes after returning reference to object std::vector> tables_; ThreadSharedDataHandle dh_; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace core } // namespace odc #endif odc-1.6.3/src/odc/core/Codec.h0000664000175000017500000001167015146027420016132 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ #ifndef odc_core_Codec_H #define odc_core_Codec_H #include #include #include "odc/MDI.h" #include "odc/api/ColumnType.h" #include "odc/core/CodecFactory.h" #include "odc/core/DataStream.h" namespace eckit { class DataHandle; } namespace odc { namespace core { //---------------------------------------------------------------------------------------------------------------------- class Codec { public: Codec(const std::string& name, api::ColumnType type); virtual ~Codec(); /// Creates a clone of this codec. NOTE: the clone is not really usefull for coding/decoding, but has the same /// stats/missing values as the original codec, which can be useful sometimes. virtual std::unique_ptr clone(); const std::string& name() const { return name_; } char* encode(char* p, const double& d) { return reinterpret_cast(encode(reinterpret_cast(p), d)); } virtual unsigned char* encode(unsigned char* p, const double& d) = 0; virtual void decode(double* out) = 0; virtual void skip() = 0; void setDataStream(GeneralDataStream& ds); virtual void setDataStream(DataStream& ds); virtual void setDataStream(DataStream& ds); virtual void clearDataStream() = 0; void load(GeneralDataStream& ds); virtual void load(DataStream& ds); virtual void load(DataStream& ds); void save(GeneralDataStream& ds); virtual void save(DataStream& ds); virtual void save(DataStream& ds); void resetStats() { min_ = max_ = missingValue_; hasMissing_ = false; } virtual void gatherStats(const double& v); void hasMissing(bool h) { hasMissing_ = h; } int32_t hasMissing() const { return hasMissing_; } void min(double m) { min_ = m; } double min() const { return min_; } void max(double m) { max_ = m; } double max() const { return max_; } virtual void missingValue(double v); double rawMissingValue() const { return missingValue_; } virtual double missingValue() const { return missingValue_; } // Some special functions for string handling inside the CodecOptimizer virtual size_t numStrings() const { NOTIMP; } virtual void copyStrings(Codec& rhs) { NOTIMP; } virtual size_t dataSizeDoubles() const { return 1; } virtual void dataSizeDoubles(size_t count) { if (count != 1) throw eckit::SeriousBug("Data size cannot be changed from 1x8 bytes", Here()); } private: // methods virtual void print(std::ostream& s) const; friend std::ostream& operator<<(std::ostream& s, const Codec& p) { p.print(s); return s; } protected: std::string name_; int32_t hasMissing_; double missingValue_; double min_; double max_; api::ColumnType type_; private: Codec(const Codec&); Codec& operator=(const Codec&); }; // template // Codec* Codec::findCodec(const std::string& name, bool differentByteOrder) //{ // return AbstractCodecFactory::getCodec(name, differentByteOrder); // } /// We need somewhere to distinguish the behaviour for SameByteOrder vs OtherByteOrder. That /// somewhere is here. template class DataStreamCodec : public Codec { public: // methods DataStreamCodec(const std::string& name, api::ColumnType type) : Codec(name, type), ds_(0) {} using Codec::setDataStream; void setDataStream(DataStream& ds) override { ds_ = &ds; } void clearDataStream() override { ds_ = 0; } protected: // methods using Codec::load; void load(DataStream& ds) override { // n.b. name read by the CodecFactory. ds.read(hasMissing_); ds.read(min_); ds.read(max_); ds.read(missingValue_); } using Codec::save; void save(DataStream& ds) override { // n.b. Name is written by the _column_ not the codec. // ds.write(name_); ds.write(hasMissing_); ds.write(min_); ds.write(max_); ds.write(missingValue_); } protected: // n.b. ds_ MUST be initialised before it is used. DataStream& ds() { ASSERT(ds_); return *ds_; } DataStream* ds_; }; //---------------------------------------------------------------------------------------------------------------------- } // namespace core } // namespace odc #endif odc-1.6.3/src/odc/core/CodecFactory.h0000664000175000017500000001271215146027420017460 0ustar alastairalastair/* * (C) Copyright 1996-2012 ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ /// @author Simon Smart /// @date January 2019 #ifndef odc_core_CodecFactory_H #define odc_core_CodecFactory_H #include #include #include #include #include #include #include #include "odc/ODBAPISettings.h" #include "odc/api/ColumnType.h" #include "odc/core/Exceptions.h" namespace odc { namespace core { //---------------------------------------------------------------------------------------------------------------------- class CodecBuilderBase; class Codec; template class DataStream; struct SameByteOrder; struct OtherByteOrder; class CodecFactory { public: // methods CodecFactory(); CodecFactory(const CodecFactory&) = delete; CodecFactory& operator=(const CodecFactory&) = delete; ~CodecFactory(); static CodecFactory& instance(); void enregister(const std::string& name, CodecBuilderBase& builder); void deregister(const std::string& name, CodecBuilderBase& builder); template std::unique_ptr build(const std::string& name, api::ColumnType type) const; template std::unique_ptr load(DataStream& ds, api::ColumnType type) const; private: // members mutable std::mutex m_; std::map> builders_; }; //---------------------------------------------------------------------------------------------------------------------- class CodecBuilderBase { protected: // methods CodecBuilderBase(const std::string& name); ~CodecBuilderBase(); public: // methods virtual std::unique_ptr make(const SameByteOrder&, api::ColumnType) const = 0; virtual std::unique_ptr make(const OtherByteOrder&, api::ColumnType) const = 0; private: // members std::string name_; }; //---------------------------------------------------------------------------------------------------------------------- template